docs/assets/05-gain-over-base.svg

<?xml version="1.0" encoding="utf-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="782.135469pt" height="331.201594pt" viewBox="0 0 782.135469 331.201594" xmlns="http://www.w3.org/2000/svg" version="1.1">
 <metadata>
  <rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
   <cc:Work>
    <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
    <dc:date>2026-09-26T12:04:17.170054</dc:date>
    <dc:format>image/svg+xml</dc:format>
    <dc:creator>
     <cc:Agent>
      <dc:title>Matplotlib v3.10.9, https://matplotlib.org/</dc:title>
     </cc:Agent>
    </dc:creator>
   </cc:Work>
  </rdf:RDF>
 </metadata>
 <defs>
  <style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>
 </defs>
 <g id="figure_1">
  <g id="patch_1">
   <path d="M 0 331.201594 
L 782.135469 331.201594 
L 782.135469 0 
L 0 0 
z
" style="fill: #fcfcfb"/>
  </g>
  <g id="axes_1">
   <g id="patch_2">
    <path d="M 180.624219 270.254125 
L 710.724219 270.254125 
L 710.724219 26.318125 
L 180.624219 26.318125 
z
" style="fill: #fcfcfb"/>
   </g>
   <g id="matplotlib.axis_1">
    <g id="xtick_1">
     <g id="line2d_1">
      <path d="M 196.215395 270.254125 
L 196.215395 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #e6e5e1; stroke-linecap: square"/>
     </g>
     <g id="line2d_2"/>
     <g id="text_1">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="196.215395" y="282.112406" transform="rotate(-0 196.215395 282.112406)">−10</text>
     </g>
    </g>
    <g id="xtick_2">
     <g id="line2d_3">
      <path d="M 274.171278 270.254125 
L 274.171278 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #e6e5e1; stroke-linecap: square"/>
     </g>
     <g id="line2d_4"/>
     <g id="text_2">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="274.171278" y="282.112406" transform="rotate(-0 274.171278 282.112406)">0</text>
     </g>
    </g>
    <g id="xtick_3">
     <g id="line2d_5">
      <path d="M 352.12716 270.254125 
L 352.12716 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #e6e5e1; stroke-linecap: square"/>
     </g>
     <g id="line2d_6"/>
     <g id="text_3">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="352.12716" y="282.112406" transform="rotate(-0 352.12716 282.112406)">10</text>
     </g>
    </g>
    <g id="xtick_4">
     <g id="line2d_7">
      <path d="M 430.083042 270.254125 
L 430.083042 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #e6e5e1; stroke-linecap: square"/>
     </g>
     <g id="line2d_8"/>
     <g id="text_4">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="430.083042" y="282.112406" transform="rotate(-0 430.083042 282.112406)">20</text>
     </g>
    </g>
    <g id="xtick_5">
     <g id="line2d_9">
      <path d="M 508.038925 270.254125 
L 508.038925 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #e6e5e1; stroke-linecap: square"/>
     </g>
     <g id="line2d_10"/>
     <g id="text_5">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="508.038925" y="282.112406" transform="rotate(-0 508.038925 282.112406)">30</text>
     </g>
    </g>
    <g id="xtick_6">
     <g id="line2d_11">
      <path d="M 585.994807 270.254125 
L 585.994807 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #e6e5e1; stroke-linecap: square"/>
     </g>
     <g id="line2d_12"/>
     <g id="text_6">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="585.994807" y="282.112406" transform="rotate(-0 585.994807 282.112406)">40</text>
     </g>
    </g>
    <g id="xtick_7">
     <g id="line2d_13">
      <path d="M 663.950689 270.254125 
L 663.950689 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #e6e5e1; stroke-linecap: square"/>
     </g>
     <g id="line2d_14"/>
     <g id="text_7">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="663.950689" y="282.112406" transform="rotate(-0 663.950689 282.112406)">50</text>
     </g>
    </g>
    <g id="text_8">
     <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: middle" x="445.674219" y="296.758344" transform="rotate(-0 445.674219 296.758344)">accuracy change vs the untuned base model (percentage points)</text>
    </g>
   </g>
   <g id="matplotlib.axis_2">
    <g id="ytick_1">
     <g id="line2d_15"/>
     <g id="text_9">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #0b0b0b" x="177.124219" y="57.177766" transform="rotate(-0 177.124219 57.177766)">Distribution probability (1,036)</text>
     </g>
    </g>
    <g id="ytick_2">
     <g id="line2d_16"/>
     <g id="text_10">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #0b0b0b" x="177.124219" y="95.292766" transform="rotate(-0 177.124219 95.292766)">Realistic transfer set (600)</text>
     </g>
    </g>
    <g id="ytick_3">
     <g id="line2d_17"/>
     <g id="text_11">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #0b0b0b" x="177.124219" y="133.407766" transform="rotate(-0 177.124219 133.407766)">JevBench standard (public, 72)</text>
     </g>
    </g>
    <g id="ytick_4">
     <g id="line2d_18"/>
     <g id="text_12">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #0b0b0b" x="177.124219" y="171.522766" transform="rotate(-0 177.124219 171.522766)">Judge proxy (600)</text>
     </g>
    </g>
    <g id="ytick_5">
     <g id="line2d_19"/>
     <g id="text_13">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #0b0b0b" x="177.124219" y="209.637766" transform="rotate(-0 177.124219 209.637766)">JevBench easy (public, 48)</text>
     </g>
    </g>
    <g id="ytick_6">
     <g id="line2d_20"/>
     <g id="text_14">
      <text style="font-size: 11px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #0b0b0b" x="177.124219" y="247.752766" transform="rotate(-0 177.124219 247.752766)">JevBench hard (public, 111)</text>
     </g>
    </g>
   </g>
   <g id="patch_3">
    <path d="M 274.561057 51.855175 
L 637.913425 51.855175 
Q 638.303204 51.855175 638.303204 49.949425 
L 638.303204 41.564125 
Q 638.303204 39.658375 637.913425 39.658375 
L 274.561057 39.658375 
Q 274.171278 39.658375 274.171278 41.564125 
L 274.171278 49.949425 
Q 274.171278 51.855175 274.561057 51.855175 
z
" clip-path="url(#p025886b573)" style="fill: #2a78d6"/>
   </g>
   <g id="patch_4">
    <path d="M 274.561057 66.338875 
L 645.553101 66.338875 
Q 645.942881 66.338875 645.942881 64.433125 
L 645.942881 56.047825 
Q 645.942881 54.142075 645.553101 54.142075 
L 274.561057 54.142075 
Q 274.171278 54.142075 274.171278 56.047825 
L 274.171278 64.433125 
Q 274.171278 66.338875 274.561057 66.338875 
z
" clip-path="url(#p025886b573)" style="fill: #eb6834"/>
   </g>
   <g id="patch_5">
    <path d="M 274.561057 89.970175 
L 407.631748 89.970175 
Q 408.021528 89.970175 408.021528 88.064425 
L 408.021528 79.679125 
Q 408.021528 77.773375 407.631748 77.773375 
L 274.561057 77.773375 
Q 274.171278 77.773375 274.171278 79.679125 
L 274.171278 88.064425 
Q 274.171278 89.970175 274.561057 89.970175 
z
" clip-path="url(#p025886b573)" style="fill: #2a78d6"/>
   </g>
   <g id="patch_6">
    <path d="M 274.561057 104.453875 
L 476.466792 104.453875 
Q 476.856572 104.453875 476.856572 102.548125 
L 476.856572 94.162825 
Q 476.856572 92.257075 476.466792 92.257075 
L 274.561057 92.257075 
Q 274.171278 92.257075 274.171278 94.162825 
L 274.171278 102.548125 
Q 274.171278 104.453875 274.561057 104.453875 
z
" clip-path="url(#p025886b573)" style="fill: #eb6834"/>
   </g>
   <g id="patch_7">
    <path d="M 274.561057 128.085175 
L 414.491866 128.085175 
Q 414.881645 128.085175 414.881645 126.179425 
L 414.881645 117.794125 
Q 414.881645 115.888375 414.491866 115.888375 
L 274.561057 115.888375 
Q 274.171278 115.888375 274.171278 117.794125 
L 274.171278 126.179425 
Q 274.171278 128.085175 274.561057 128.085175 
z
" clip-path="url(#p025886b573)" style="fill: #2a78d6"/>
   </g>
   <g id="patch_8">
    <path d="M 274.561057 142.568875 
L 447.077425 142.568875 
Q 447.467204 142.568875 447.467204 140.663125 
L 447.467204 132.277825 
Q 447.467204 130.372075 447.077425 130.372075 
L 274.561057 130.372075 
Q 274.171278 130.372075 274.171278 132.277825 
L 274.171278 140.663125 
Q 274.171278 142.568875 274.561057 142.568875 
z
" clip-path="url(#p025886b573)" style="fill: #eb6834"/>
   </g>
   <g id="patch_9">
    <path d="M 274.561057 166.200175 
L 347.839586 166.200175 
Q 348.229366 166.200175 348.229366 164.294425 
L 348.229366 155.909125 
Q 348.229366 154.003375 347.839586 154.003375 
L 274.561057 154.003375 
Q 274.171278 154.003375 274.171278 155.909125 
L 274.171278 164.294425 
Q 274.171278 166.200175 274.561057 166.200175 
z
" clip-path="url(#p025886b573)" style="fill: #2a78d6"/>
   </g>
   <g id="patch_10">
    <path d="M 274.561057 180.683875 
L 467.345954 180.683875 
Q 467.735733 180.683875 467.735733 178.778125 
L 467.735733 170.392825 
Q 467.735733 168.487075 467.345954 168.487075 
L 274.561057 168.487075 
Q 274.171278 168.487075 274.171278 170.392825 
L 274.171278 178.778125 
Q 274.171278 180.683875 274.561057 180.683875 
z
" clip-path="url(#p025886b573)" style="fill: #eb6834"/>
   </g>
   <g id="patch_11">
    <path d="M 274.561057 204.315175 
L 273.781498 204.315175 
Q 274.171278 204.315175 274.171278 202.409425 
L 274.171278 194.024125 
Q 274.171278 192.118375 273.781498 192.118375 
L 274.561057 192.118375 
Q 274.171278 192.118375 274.171278 194.024125 
L 274.171278 202.409425 
Q 274.171278 204.315175 274.561057 204.315175 
z
" clip-path="url(#p025886b573)" style="fill: #2a78d6"/>
   </g>
   <g id="patch_12">
    <path d="M 274.561057 218.798875 
L 289.996322 218.798875 
Q 290.386101 218.798875 290.386101 216.893125 
L 290.386101 208.507825 
Q 290.386101 206.602075 289.996322 206.602075 
L 274.561057 206.602075 
Q 274.171278 206.602075 274.171278 208.507825 
L 274.171278 216.893125 
Q 274.171278 218.798875 274.561057 218.798875 
z
" clip-path="url(#p025886b573)" style="fill: #eb6834"/>
   </g>
   <g id="patch_13">
    <path d="M 211.338836 242.430175 
L 273.781498 242.430175 
Q 274.171278 242.430175 274.171278 240.524425 
L 274.171278 232.139125 
Q 274.171278 230.233375 273.781498 230.233375 
L 211.338836 230.233375 
Q 210.949057 230.233375 210.949057 232.139125 
L 210.949057 240.524425 
Q 210.949057 242.430175 211.338836 242.430175 
z
" clip-path="url(#p025886b573)" style="fill: #2a78d6"/>
   </g>
   <g id="patch_14">
    <path d="M 274.561057 256.913875 
L 287.813557 256.913875 
Q 288.203336 256.913875 288.203336 255.008125 
L 288.203336 246.622825 
Q 288.203336 244.717075 287.813557 244.717075 
L 274.561057 244.717075 
Q 274.171278 244.717075 274.171278 246.622825 
L 274.171278 255.008125 
Q 274.171278 256.913875 274.561057 256.913875 
z
" clip-path="url(#p025886b573)" style="fill: #eb6834"/>
   </g>
   <g id="PathCollection_1"/>
   <g id="PathCollection_2"/>
   <g id="line2d_21">
    <path d="M 274.171278 270.254125 
L 274.171278 26.318125 
" clip-path="url(#p025886b573)" style="fill: none; stroke: #52514e; stroke-linecap: square"/>
   </g>
   <g id="patch_15">
    <path d="M 180.624219 270.254125 
L 710.724219 270.254125 
" style="fill: none; stroke: #e6e5e1; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
   </g>
   <g id="text_15">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="644.539675" y="48.240212" transform="rotate(-0 644.539675 48.240212)">+46.7</text>
   </g>
   <g id="text_16">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="652.179351" y="62.723912" transform="rotate(-0 652.179351 62.723912)">+47.7</text>
   </g>
   <g id="text_17">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="414.257998" y="86.355212" transform="rotate(-0 414.257998 86.355212)">+17.2</text>
   </g>
   <g id="text_18">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="483.093042" y="100.838912" transform="rotate(-0 483.093042 100.838912)">+26.0</text>
   </g>
   <g id="text_19">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="421.118116" y="124.470212" transform="rotate(-0 421.118116 124.470212)">+18.0</text>
   </g>
   <g id="text_20">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="453.703675" y="138.953912" transform="rotate(-0 453.703675 138.953912)">+22.2</text>
   </g>
   <g id="text_21">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="354.465836" y="162.585212" transform="rotate(-0 354.465836 162.585212)">+9.5</text>
   </g>
   <g id="text_22">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="473.972204" y="177.068912" transform="rotate(-0 473.972204 177.068912)">+24.8</text>
   </g>
   <g id="text_23">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="280.407748" y="200.700212" transform="rotate(-0 280.407748 200.700212)">+0.0</text>
   </g>
   <g id="text_24">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="296.622572" y="215.183912" transform="rotate(-0 296.622572 215.183912)">+2.1</text>
   </g>
   <g id="text_25">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #0b0b0b" x="204.712586" y="238.815212" transform="rotate(-0 204.712586 238.815212)">-8.1</text>
   </g>
   <g id="text_26">
    <text style="font-size: 9px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="294.439807" y="253.298912" transform="rotate(-0 294.439807 253.298912)">+1.8</text>
   </g>
   <g id="text_27">
    <text style="font-weight: 700; font-size: 12px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="180.624219" y="16.318125" transform="rotate(-0 180.624219 16.318125)">What fine-tuning changed — same prompt, same offline runner, no system prompt, T=1</text>
   </g>
   <g id="legend_1">
    <g id="PathCollection_3">
     <defs>
      <path id="m37eeb27dce" d="M -4.472136 4.472136 
L 4.472136 4.472136 
L 4.472136 -4.472136 
L -4.472136 -4.472136 
z
" style="stroke: #2a78d6"/>
     </defs>
     <g>
      <use xlink:href="#m37eeb27dce" x="522.118047" y="243.290453" style="fill: #2a78d6; stroke: #2a78d6"/>
     </g>
    </g>
    <g id="text_28">
     <text style="font-size: 9.5px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="539.218047" y="245.784203" transform="rotate(-0 539.218047 245.784203)">Standard One 8B vs Ministral 3 8B</text>
    </g>
    <g id="PathCollection_4">
     <defs>
      <path id="m3fef64febf" d="M -4.472136 4.472136 
L 4.472136 4.472136 
L 4.472136 -4.472136 
L -4.472136 -4.472136 
z
" style="stroke: #eb6834"/>
     </defs>
     <g>
      <use xlink:href="#m3fef64febf" x="522.118047" y="257.234672" style="fill: #eb6834; stroke: #eb6834"/>
     </g>
    </g>
    <g id="text_29">
     <text style="font-size: 9.5px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #0b0b0b" x="539.218047" y="259.728422" transform="rotate(-0 539.218047 259.728422)">Standard One 3B vs Ministral 3 3B</text>
    </g>
   </g>
  </g>
  <g id="text_30">
   <text style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #52514e" transform="translate(101.964219 312.715719)">Public JevBench tiers are the 231 public items only; the other suites are held-out proxies built from public data. On the public hard tier the 8B base is ahead by</text>
   <text style="font-size: 8.5px; font-family: 'DejaVu Sans'; fill: #52514e" transform="translate(101.964219 322.233859)">9 of 111 items and the 3B base is behind by 2 of 111 items — see "Honest limits".</text>
  </g>
 </g>
 <defs>
  <clipPath id="p025886b573">
   <rect x="180.624219" y="26.318125" width="530.1" height="243.936"/>
  </clipPath>
 </defs>
</svg>