File size: 12,678 Bytes
cac76f6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
09d4173
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
<svg xmlns="http://www.w3.org/2000/svg" width="2600" height="1840" viewBox="0 0 2600 1840" role="img" aria-label="Standard One v2 benchmark accuracy and quality vertical bars with horizontal latency bars">
<rect width="2600" height="1840" fill="#F4F7FB"/>
<text x="76" y="78" fill="#246EE5" font-family="Arial" font-size="30" font-weight="700">STANDARD ONE</text>
<text x="74" y="147" fill="#17233B" font-family="Arial" font-size="64" font-weight="700">v2 benchmark snapshot</text>
<text x="77" y="204" fill="#65748C" font-family="Arial" font-size="30">Merged BF16 checkpoints  ·  Served endpoint  ·  Public and held-out suites</text>
<rect x="2140" y="71" width="386" height="79" rx="39" fill="#FFFFFF" stroke="#DFE6F0" stroke-width="2"/>
<text x="2333" y="119" fill="#246EE5" font-family="Arial" font-size="27" font-weight="700" text-anchor="middle">PUBLIC + HELD-OUT</text>
<rect x="70" y="248" width="2460" height="942" rx="28" fill="#FFFFFF" stroke="#DFE6F0" stroke-width="2"/>
<text x="126" y="325" fill="#17233B" font-family="Arial" font-size="48" font-weight="700">Decision accuracy</text>
<text x="127" y="374" fill="#65748C" font-family="Arial" font-size="28">Same items and option order  ·  Higher is better  ·  Percent correct</text>
<rect x="1756" y="312" width="24" height="24" rx="6" fill="#EB7145"/>
<text x="1794" y="334" fill="#17233B" font-family="Arial" font-size="30" font-weight="700">3B</text>
<rect x="1907" y="312" width="24" height="24" rx="6" fill="#246EE5"/>
<text x="1945" y="334" fill="#17233B" font-family="Arial" font-size="30" font-weight="700">8B</text>
<rect x="2058" y="312" width="24" height="24" rx="6" fill="#818B9D"/>
<text x="2096" y="334" fill="#17233B" font-family="Arial" font-size="30" font-weight="700">Jev 1.13</text>
<rect x="163" y="414" width="927" height="48" rx="24" fill="#EDF4FF"/>
<text x="626" y="447" fill="#246EE5" font-family="Arial" font-size="25" font-weight="700" text-anchor="middle">PUBLIC JEVBENCH</text>
<rect x="1174" y="414" width="1268" height="48" rx="24" fill="#EFF2F7"/>
<text x="1808" y="447" fill="#65748C" font-family="Arial" font-size="25" font-weight="700" text-anchor="middle">HELD-OUT DECISION SUITES</text>
<text x="124" y="992" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">0</text>
<line x1="137" y1="869" x2="2463" y2="869" stroke="#DFE6F0" stroke-width="2"/>
<text x="124" y="869" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">25</text>
<line x1="137" y1="746" x2="2463" y2="746" stroke="#DFE6F0" stroke-width="2"/>
<text x="124" y="746" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">50</text>
<line x1="137" y1="624" x2="2463" y2="624" stroke="#DFE6F0" stroke-width="2"/>
<text x="124" y="624" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">75</text>
<line x1="137" y1="501" x2="2463" y2="501" stroke="#DFE6F0" stroke-width="2"/>
<text x="124" y="501" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">100</text>
<line x1="137" y1="992" x2="2463" y2="992" stroke="#AEBCCD" stroke-width="3"/>
<line x1="1134" y1="484" x2="1134" y2="1149" stroke="#D5DEE9" stroke-width="3"/>
<rect x="189" y="501" width="70" height="491" rx="9" fill="#EB7145"/>
<rect x="268" y="501" width="70" height="491" rx="9" fill="#246EE5"/>
<rect x="347" y="501" width="70" height="491" rx="9" fill="#818B9D"/>
<text x="303" y="1052" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">JevBench</text>
<text x="303" y="1091" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">public easy</text>
<rect x="521" y="556" width="70" height="436" rx="9" fill="#EB7145"/>
<rect x="600" y="535" width="70" height="457" rx="9" fill="#246EE5"/>
<rect x="679" y="508" width="70" height="484" rx="9" fill="#818B9D"/>
<text x="635" y="1052" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">JevBench</text>
<text x="635" y="1091" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">public standard</text>
<rect x="854" y="762" width="70" height="230" rx="9" fill="#EB7145"/>
<rect x="933" y="722" width="70" height="270" rx="9" fill="#246EE5"/>
<rect x="1012" y="638" width="70" height="354" rx="9" fill="#818B9D"/>
<text x="968" y="1052" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">JevBench</text>
<text x="968" y="1091" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">public hard</text>
<rect x="1186" y="559" width="70" height="433" rx="9" fill="#EB7145"/>
<rect x="1265" y="553" width="70" height="439" rx="9" fill="#246EE5"/>
<rect x="1344" y="548" width="70" height="444" rx="9" fill="#818B9D"/>
<text x="1300" y="1052" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">Judge</text>
<text x="1300" y="1091" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">proxy</text>
<rect x="1518" y="547" width="70" height="445" rx="9" fill="#EB7145"/>
<rect x="1597" y="546" width="70" height="446" rx="9" fill="#246EE5"/>
<rect x="1676" y="566" width="70" height="426" rx="9" fill="#818B9D"/>
<text x="1632" y="1052" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">Realistic</text>
<text x="1632" y="1091" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">transfer</text>
<rect x="1851" y="605" width="70" height="387" rx="9" fill="#EB7145"/>
<rect x="1930" y="593" width="70" height="399" rx="9" fill="#246EE5"/>
<rect x="2009" y="634" width="70" height="358" rx="9" fill="#818B9D"/>
<text x="1965" y="1052" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">Stated distribution</text>
<text x="1965" y="1091" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">argmax</text>
<rect x="2183" y="773" width="70" height="219" rx="9" fill="#EB7145"/>
<rect x="2262" y="732" width="70" height="260" rx="9" fill="#246EE5"/>
<rect x="2341" y="723" width="70" height="269" rx="9" fill="#818B9D"/>
<text x="2297" y="1052" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">Hard</text>
<text x="2297" y="1091" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">proxy</text>
<rect x="70" y="1220" width="1140" height="488" rx="28" fill="#FFFFFF" stroke="#DFE6F0" stroke-width="2"/>
<text x="126" y="1289" fill="#17233B" font-family="Arial" font-size="42" font-weight="700">Probability quality</text>
<text x="127" y="1333" fill="#65748C" font-family="Arial" font-size="28">Lower is better</text>
<text x="164" y="1586" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">0.00</text>
<line x1="178" y1="1504" x2="1115" y2="1504" stroke="#DFE6F0" stroke-width="2"/>
<text x="164" y="1504" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">0.10</text>
<line x1="178" y1="1422" x2="1115" y2="1422" stroke="#DFE6F0" stroke-width="2"/>
<text x="164" y="1422" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">0.20</text>
<line x1="178" y1="1381" x2="1115" y2="1381" stroke="#DFE6F0" stroke-width="2"/>
<text x="164" y="1381" fill="#65748C" font-family="Arial" font-size="24" text-anchor="end" dominant-baseline="middle">0.25</text>
<line x1="178" y1="1586" x2="1115" y2="1586" stroke="#AEBCCD" stroke-width="3"/>
<rect x="210" y="1402" width="94" height="184" rx="9" fill="#EB7145"/>
<rect x="323" y="1435" width="94" height="151" rx="9" fill="#246EE5"/>
<rect x="436" y="1505" width="94" height="81" rx="9" fill="#818B9D"/>
<text x="379" y="1639" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">Hard-tier ECE</text>
<rect x="729" y="1490" width="94" height="96" rx="9" fill="#EB7145"/>
<rect x="842" y="1497" width="94" height="89" rx="9" fill="#246EE5"/>
<rect x="955" y="1429" width="94" height="157" rx="9" fill="#818B9D"/>
<text x="898" y="1639" fill="#17233B" font-family="Arial" font-size="30" font-weight="700" text-anchor="middle">Distribution TV</text>
<text x="127" y="1686" fill="#65748C" font-family="Arial" font-size="24">Fitted Standard One temperatures  ·  Jev raw</text>
<rect x="1240" y="1220" width="1290" height="488" rx="28" fill="#FFFFFF" stroke="#DFE6F0" stroke-width="2"/>
<text x="1296" y="1289" fill="#17233B" font-family="Arial" font-size="42" font-weight="700">Speed on one H200</text>
<text x="1297" y="1333" fill="#65748C" font-family="Arial" font-size="28">Serial decision latency (ms)  ·  Lower is faster</text>
<text x="1297" y="1383" fill="#65748C" font-family="Arial" font-size="25" font-weight="700">MODEL</text>
<text x="1787" y="1385" fill="#17233B" font-family="Arial" font-size="27" font-weight="700" text-anchor="middle">P50</text>
<line x1="1613" y1="1412" x2="1613" y2="1617" stroke="#AEBCCD" stroke-width="2"/>
<text x="1613" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">0</text>
<line x1="1700" y1="1412" x2="1700" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="1700" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">20</text>
<line x1="1788" y1="1412" x2="1788" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="1788" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">40</text>
<line x1="1875" y1="1412" x2="1875" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="1875" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">60</text>
<line x1="1962" y1="1412" x2="1962" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="1962" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">80</text>
<text x="2264" y="1385" fill="#17233B" font-family="Arial" font-size="27" font-weight="700" text-anchor="middle">P95</text>
<line x1="2090" y1="1412" x2="2090" y2="1617" stroke="#AEBCCD" stroke-width="2"/>
<text x="2090" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">0</text>
<line x1="2177" y1="1412" x2="2177" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="2177" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">20</text>
<line x1="2264" y1="1412" x2="2264" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="2264" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">40</text>
<line x1="2352" y1="1412" x2="2352" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="2352" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">60</text>
<line x1="2439" y1="1412" x2="2439" y2="1617" stroke="#DFE6F0" stroke-width="2"/>
<text x="2439" y="1652" fill="#65748C" font-family="Arial" font-size="24" text-anchor="middle">80</text>
<text x="1297" y="1462" fill="#EB7145" font-family="Arial" font-size="26" font-weight="700">Standard One 3B</text>
<rect x="1613" y="1437" width="99" height="29" rx="8" fill="#EB7145"/>
<rect x="2090" y="1437" width="145" height="29" rx="8" fill="#EB7145"/>
<text x="1297" y="1514" fill="#246EE5" font-family="Arial" font-size="26" font-weight="700">Standard One 8B</text>
<rect x="1613" y="1489" width="113" height="29" rx="8" fill="#246EE5"/>
<rect x="2090" y="1489" width="183" height="29" rx="8" fill="#246EE5"/>
<text x="1297" y="1566" fill="#818B9D" font-family="Arial" font-size="26" font-weight="700">Qwen3-8B*</text>
<rect x="1613" y="1541" width="124" height="29" rx="8" fill="#818B9D"/>
<rect x="2090" y="1541" width="250" height="29" rx="8" fill="#818B9D"/>
<text x="1297" y="1618" fill="#596579" font-family="Arial" font-size="26" font-weight="700">Qwen3.5-4B*</text>
<rect x="1613" y="1593" width="213" height="29" rx="8" fill="#596579"/>
<rect x="2090" y="1593" width="317" height="29" rx="8" fill="#596579"/>
<text x="1297" y="1694" fill="#65748C" font-family="Arial" font-size="24">* Untuned Qwen checkpoints are shown for speed only</text>
<text x="78" y="1760" fill="#65748C" font-family="Arial" font-size="27">Public JevBench tiers are separate from held-out suites. Jev accuracy uses its hosted endpoint.</text>
<text x="78" y="1805" fill="#65748C" font-family="Arial" font-size="27">Measured results from StandardThinking model cards; no official composite leaderboard score is implied.</text>
</svg>