Skip to content

Commit ca622e5

Browse files
stephantulstephantul
authored
feat: new integration test (#355)
* feat: new integration test * remove all
1 parent cbeea06 commit ca622e5

8 files changed

Lines changed: 702 additions & 0 deletions

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,6 @@ test-integration:
2727

2828
test-integration-update:
2929
uv run python -m tests.integration.update_distill_baseline
30+
31+
test-integration-pretrained-update:
32+
uv run python -m tests.integration.update_pretrained_baseline

‎tests/integration/data/pretrained/minishlab___potion-base-32m_baseline.json‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
{
2+
"metrics": {
3+
"base_model_name": null,
4+
"config": {
5+
"apply_pca": 512,
6+
"apply_zipf": true,
7+
"architectures": [
8+
"StaticModel"
9+
],
10+
"hidden_dim": 512,
11+
"model_type": "model2vec",
12+
"normalize": true,
13+
"seq_length": 1000000,
14+
"tokenizer_name": "baai/bge-base-en-v1.5"
15+
},
16+
"embedding_dim": 512,
17+
"embedding_dtype": "float32",
18+
"embedding_mean": -0.000142,
19+
"embedding_rank": 512,
20+
"embedding_row_norm_mean": 109.413377,
21+
"embedding_rows": 63091,
22+
"embedding_std": 4.876269,
23+
"encoding_speed": {
24+
"sentences_per_second": 119227.47,
25+
"tokens_per_second": 1341309.0
26+
},
27+
"first_tokens": [
28+
"[PAD]",
29+
"[UNK]",
30+
"[CLS]",
31+
"[SEP]",
32+
"[MASK]",
33+
"!",
34+
"\"",
35+
"#",
36+
"$",
37+
"%"
38+
],
39+
"full_vocab_size": 63091,
40+
"has_token_mapping": false,
41+
"has_weights": false,
42+
"language": null,
43+
"last_tokens": [
44+
"nti",
45+
"corporeal",
46+
"downgrading",
47+
"symbolise",
48+
"emptive",
49+
"daters",
50+
"cousteau",
51+
"crichton",
52+
"60k",
53+
"longines"
54+
],
55+
"median_token_length": 7,
56+
"mteb_sts_scores": {
57+
"BIOSSES": 0.775566,
58+
"SICK-R": 0.656684,
59+
"STS12": 0.627234,
60+
"STS13": 0.775936,
61+
"STS14": 0.729035,
62+
"STS15": 0.807592,
63+
"STS16": 0.768566,
64+
"STSBenchmark": 0.768385
65+
},
66+
"normalize": true,
67+
"token_order_hash": "ba90854d713d66960327011c1d592b5f16c6dd7fe26676b103490b62753dc7c9",
68+
"tokenizer_type": "WordPiece",
69+
"unk_token_id": 1,
70+
"vocabulary_quantization": null
71+
},
72+
"model": "minishlab/potion-base-32m"
73+
}

‎tests/integration/data/pretrained/minishlab___potion-base-8m_baseline.json‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
{
2+
"metrics": {
3+
"base_model_name": null,
4+
"config": {
5+
"apply_pca": 256,
6+
"apply_zipf": true,
7+
"architectures": [
8+
"StaticModel"
9+
],
10+
"hidden_dim": 256,
11+
"model_type": "model2vec",
12+
"normalize": true,
13+
"seq_length": 1000000,
14+
"tokenizer_name": "baai/bge-base-en-v1.5"
15+
},
16+
"embedding_dim": 256,
17+
"embedding_dtype": "float32",
18+
"embedding_mean": -0.000426,
19+
"embedding_rank": 256,
20+
"embedding_row_norm_mean": 94.661816,
21+
"embedding_rows": 29528,
22+
"embedding_std": 5.989001,
23+
"encoding_speed": {
24+
"sentences_per_second": 133169.33,
25+
"tokens_per_second": 1614678.17
26+
},
27+
"first_tokens": [
28+
"[PAD]",
29+
"[UNK]",
30+
"[CLS]",
31+
"[SEP]",
32+
"[MASK]",
33+
"!",
34+
"\"",
35+
"#",
36+
"$",
37+
"%"
38+
],
39+
"full_vocab_size": 29528,
40+
"has_token_mapping": false,
41+
"has_weights": false,
42+
"language": null,
43+
"last_tokens": [
44+
"##\uff01",
45+
"##\uff08",
46+
"##\uff09",
47+
"##\uff0c",
48+
"##\uff0d",
49+
"##\uff0e",
50+
"##\uff0f",
51+
"##\uff1a",
52+
"##\uff1f",
53+
"##\uff5e"
54+
],
55+
"median_token_length": 6,
56+
"mteb_sts_scores": {
57+
"BIOSSES": 0.758578,
58+
"SICK-R": 0.646753,
59+
"STS12": 0.622476,
60+
"STS13": 0.77276,
61+
"STS14": 0.719141,
62+
"STS15": 0.797535,
63+
"STS16": 0.762634,
64+
"STSBenchmark": 0.754052
65+
},
66+
"normalize": true,
67+
"token_order_hash": "efd6dc3c71fd5fe1c1f97b63e7caf527b1e2196f7af18e1e1581e43e82880a8a",
68+
"tokenizer_type": "WordPiece",
69+
"unk_token_id": 1,
70+
"vocabulary_quantization": null
71+
},
72+
"model": "minishlab/potion-base-8m"
73+
}

‎tests/integration/data/pretrained/minishlab___potion-multilingual-128m_baseline.json‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
{
2+
"metrics": {
3+
"base_model_name": null,
4+
"config": {
5+
"apply_pca": null,
6+
"apply_zipf": null,
7+
"architectures": [
8+
"StaticModel"
9+
],
10+
"hidden_dim": 256,
11+
"model_type": "model2vec",
12+
"normalize": true,
13+
"seq_length": 1000000,
14+
"sif_coefficient": null,
15+
"tokenizer_name": "BAAI/bge-m3"
16+
},
17+
"embedding_dim": 256,
18+
"embedding_dtype": "float32",
19+
"embedding_mean": -0.002286,
20+
"embedding_rank": 256,
21+
"embedding_row_norm_mean": 12.731157,
22+
"embedding_rows": 500353,
23+
"embedding_std": 0.857525,
24+
"encoding_speed": {
25+
"sentences_per_second": 117518.51,
26+
"tokens_per_second": 1498360.99
27+
},
28+
"first_tokens": [
29+
"[PAD]",
30+
"[UNK]",
31+
",",
32+
".",
33+
"\u2581",
34+
"s",
35+
"\u2581de",
36+
"-",
37+
"\u2581a",
38+
"a"
39+
],
40+
"full_vocab_size": 500353,
41+
"has_token_mapping": false,
42+
"has_weights": false,
43+
"language": [
44+
"en",
45+
"multilingual",
46+
"af",
47+
"am",
48+
"ar",
49+
"az",
50+
"be",
51+
"bg",
52+
"bn",
53+
"ca",
54+
"ceb",
55+
"co",
56+
"cs",
57+
"cy",
58+
"da",
59+
"de",
60+
"el",
61+
"eo",
62+
"es",
63+
"et",
64+
"eu",
65+
"fa",
66+
"fi",
67+
"fil",
68+
"fr",
69+
"fy",
70+
"ga",
71+
"gd",
72+
"gl",
73+
"gu",
74+
"ha",
75+
"haw",
76+
"hi",
77+
"hmn",
78+
"ht",
79+
"hu",
80+
"hy",
81+
"id",
82+
"ig",
83+
"is",
84+
"it",
85+
"iw",
86+
"ja",
87+
"jv",
88+
"ka",
89+
"kk",
90+
"km",
91+
"kn",
92+
"ko",
93+
"ku",
94+
"ky",
95+
"la",
96+
"lb",
97+
"lo",
98+
"lt",
99+
"lv",
100+
"mg",
101+
"mi",
102+
"mk",
103+
"ml",
104+
"mn",
105+
"mr",
106+
"ms",
107+
"mt",
108+
"my",
109+
"ne",
110+
"nl",
111+
"no",
112+
"ny",
113+
"pa",
114+
"pl",
115+
"ps",
116+
"pt",
117+
"ro",
118+
"ru",
119+
"sd",
120+
"si",
121+
"sk",
122+
"sl",
123+
"sm",
124+
"sn",
125+
"so",
126+
"sq",
127+
"sr",
128+
"st",
129+
"su",
130+
"sv",
131+
"sw",
132+
"ta",
133+
"te",
134+
"tg",
135+
"th",
136+
"tr",
137+
"uk",
138+
"ur",
139+
"uz",
140+
"vi",
141+
"xh",
142+
"yi",
143+
"yo",
144+
"zh",
145+
"zu"
146+
],
147+
"last_tokens": [
148+
"\u2581sansar",
149+
"\u2581dprk",
150+
"\u2581angeline",
151+
"\u2581\u05de\u05d0\u05dc\u05d4",
152+
"\u2581klinikos",
153+
"\u2581dzikir",
154+
"\u2581laivo",
155+
"\u2581ncel",
156+
"\u2581t38",
157+
"\u2581lugi"
158+
],
159+
"median_token_length": 7,
160+
"mteb_sts_scores": {
161+
"BIOSSES": 0.708733,
162+
"SICK-R": 0.608512,
163+
"STS12": 0.643476,
164+
"STS13": 0.742303,
165+
"STS14": 0.697261,
166+
"STS15": 0.793331,
167+
"STS16": 0.742115,
168+
"STSBenchmark": 0.717519
169+
},
170+
"normalize": true,
171+
"token_order_hash": "0b2eac292630557804d59ed065289d80954e3d56d793d0f7c4460fd19ec7cf22",
172+
"tokenizer_type": "Unigram",
173+
"unk_token_id": null,
174+
"vocabulary_quantization": null
175+
},
176+
"model": "minishlab/potion-multilingual-128m"
177+
}

0 commit comments

Comments
 (0)