{ "active_parameters": 213932136, "completed_steps": 150450, "curve": [ { "dev": { "bits_per_byte": null, "nll": 1.7360164661501172, "targets": 24576 }, "grad_norm": 0.7202328023449567, "lr": 0.0002999334300626917, "step": 2000, "tokens": 24576000, "tokens_per_second": 40166.98948851087, "train_loss": 2.6591188952326776, "train_seconds": 611.8457049669996 }, { "dev": { "bits_per_byte": null, "nll": 1.5625182097512738, "targets": 24576 }, "grad_norm": 0.5783463389457808, "lr": 0.0002996374199002856, "step": 4000, "tokens": 49152000, "tokens_per_second": 40201.31696950577, "train_loss": 1.8227927624583244, "train_seconds": 1222.6465127319998 }, { "dev": { "bits_per_byte": null, "nll": 1.477370273765148, "targets": 24576 }, "grad_norm": 0.5094491846089862, "lr": 0.0002991050524310322, "step": 6000, "tokens": 73728000, "tokens_per_second": 40171.86403882233, "train_loss": 1.63224257000031, "train_seconds": 1835.3143864260026 }, { "dev": { "bits_per_byte": null, "nll": 1.4288524719776223, "targets": 24576 }, "grad_norm": 0.5059446461804917, "lr": 0.000298337262231228, "step": 8000, "tokens": 98304000, "tokens_per_second": 40194.18108252052, "train_loss": 1.6033206125497819, "train_seconds": 2445.7271513550018 }, { "dev": { "bits_per_byte": null, "nll": 1.3819708243422564, "targets": 24576 }, "grad_norm": 0.47802248086296967, "lr": 0.0002973353971640749, "step": 10000, "tokens": 122880000, "tokens_per_second": 40208.42046991102, "train_loss": 1.5544776976704597, "train_seconds": 3056.076278648007 }, { "dev": { "bits_per_byte": null, "nll": 1.3623778258431063, "targets": 24576 }, "grad_norm": 0.48993144611638473, "lr": 0.00029610121601349225, "step": 12000, "tokens": 147456000, "tokens_per_second": 40183.74573257091, "train_loss": 1.5037044615745545, "train_seconds": 3669.543426373008 }, { "dev": { "bits_per_byte": null, "nll": 1.3404339306344106, "targets": 24576 }, "grad_norm": 0.49791784672085204, "lr": 0.00029463688539655536, "step": 14000, "tokens": 172032000, "tokens_per_second": 40197.937361236334, "train_loss": 1.4850671073794366, "train_seconds": 4279.622570034995 }, { "dev": { "bits_per_byte": null, "nll": 1.3281894634687312, "targets": 24576 }, "grad_norm": 0.47890257468376807, "lr": 0.0002929449759599774, "step": 16000, "tokens": 196608000, "tokens_per_second": 40206.29043549813, "train_loss": 1.4604398087859154, "train_seconds": 4889.981091775003 }, { "dev": { "bits_per_byte": null, "nll": 1.2998799417171185, "targets": 24576 }, "grad_norm": 0.5167052195634401, "lr": 0.0002910284578673145, "step": 18000, "tokens": 221184000, "tokens_per_second": 40191.08839182736, "train_loss": 1.4284975415184384, "train_seconds": 5503.309535777004 }, { "dev": { "bits_per_byte": null, "nll": 1.288409102639295, "targets": 24576 }, "grad_norm": 0.4958051608066708, "lr": 0.000288890695584814, "step": 20000, "tokens": 245760000, "tokens_per_second": 40202.63918961572, "train_loss": 1.426745513856411, "train_seconds": 6113.031506237019 }, { "dev": { "bits_per_byte": null, "nll": 1.2735154572788059, "targets": 24576 }, "grad_norm": 0.5115055193244372, "lr": 0.0002865354419750618, "step": 22000, "tokens": 270336000, "tokens_per_second": 40211.73981461001, "train_loss": 1.4078242859244348, "train_seconds": 6722.812821488009 }, { "dev": { "bits_per_byte": null, "nll": 1.2704267208753557, "targets": 24576 }, "grad_norm": 0.5037998412396649, "lr": 0.00028396683170879447, "step": 24000, "tokens": 294912000, "tokens_per_second": 40209.580568723846, "train_loss": 1.386985070215252, "train_seconds": 7334.371456473011 }, { "dev": { "bits_per_byte": null, "nll": 1.2584130347651168, "targets": 24576 }, "grad_norm": 0.49683861869198326, "lr": 0.0002811893740064452, "step": 26000, "tokens": 319488000, "tokens_per_second": 40214.855718314444, "train_loss": 1.380577064871788, "train_seconds": 7944.526824561015 }, { "dev": { "bits_per_byte": null, "nll": 1.2534944205809422, "targets": 24576 }, "grad_norm": 0.501904258872761, "lr": 0.00027820794472216295, "step": 28000, "tokens": 344064000, "tokens_per_second": 40220.42508168015, "train_loss": 1.3711389843225479, "train_seconds": 8554.459563798007 }, { "dev": { "bits_per_byte": null, "nll": 1.2449243161164303, "targets": 24576 }, "grad_norm": 0.5125356080315538, "lr": 0.00027502777778420245, "step": 30000, "tokens": 368640000, "tokens_per_second": 40217.54015840415, "train_loss": 1.356677065899901, "train_seconds": 9166.149857699995 }, { "dev": { "bits_per_byte": null, "nll": 1.2305863491960698, "targets": 24576 }, "grad_norm": 0.5152298540288645, "lr": 0.0002716544560067128, "step": 32000, "tokens": 393216000, "tokens_per_second": 40219.65885044309, "train_loss": 1.3521544378995896, "train_seconds": 9776.711469935008 }, { "dev": { "bits_per_byte": null, "nll": 1.2286374940952989, "targets": 24576 }, "grad_norm": 0.5342970305999114, "lr": 0.00026809390128905165, "step": 34000, "tokens": 417792000, "tokens_per_second": 40221.41475168067, "train_loss": 1.343793150305748, "train_seconds": 10387.302450184012 }, { "dev": { "bits_per_byte": null, "nll": 1.218398080040968, "targets": 24576 }, "grad_norm": 0.5435414210190175, "lr": 0.0002643523642198339, "step": 36000, "tokens": 442368000, "tokens_per_second": 40218.382446010306, "train_loss": 1.3311349999314173, "train_seconds": 10999.149470863 }, { "dev": { "bits_per_byte": null, "nll": 1.2167425829778884, "targets": 24576 }, "grad_norm": 0.5405571161770136, "lr": 0.0002604364131039617, "step": 38000, "tokens": 466944000, "tokens_per_second": 40220.535693362785, "train_loss": 1.3245998654961586, "train_seconds": 11609.591765756997 }, { "dev": { "bits_per_byte": null, "nll": 1.2120968502909004, "targets": 24576 }, "grad_norm": 0.5447408177099538, "lr": 0.0002563529224319012, "step": 40000, "tokens": 491520000, "tokens_per_second": 40222.32279604792, "train_loss": 1.3186722019314765, "train_seconds": 12220.079941486989 }, { "dev": { "bits_per_byte": null, "nll": 1.2030824138605445, "targets": 24576 }, "grad_norm": 0.534390925566071, "lr": 0.0002521090608114473, "step": 42000, "tokens": 516096000, "tokens_per_second": 40220.67822980544, "train_loss": 1.3058949114262373, "train_seconds": 12831.608583306988 }, { "dev": { "bits_per_byte": null, "nll": 1.202116820819657, "targets": 24576 }, "grad_norm": 0.5548195480272012, "lr": 0.0002477122783831627, "step": 44000, "tokens": 540672000, "tokens_per_second": 40224.4795968712, "train_loss": 1.3047680519223213, "train_seconds": 13441.367182834985 }, { "dev": { "bits_per_byte": null, "nll": 1.1903526639122652, "targets": 24576 }, "grad_norm": 0.5836491723152043, "lr": 0.00024317029374158302, "step": 46000, "tokens": 565248000, "tokens_per_second": 40227.56645453644, "train_loss": 1.2981867902874946, "train_seconds": 14051.26011385303 }, { "dev": { "bits_per_byte": null, "nll": 1.1898730456430762, "targets": 24576 }, "grad_norm": 0.5702460263325784, "lr": 0.00023849108038514897, "step": 48000, "tokens": 589824000, "tokens_per_second": 40222.61214698756, "train_loss": 1.283541103606282, "train_seconds": 14663.990440118008 }, { "dev": { "bits_per_byte": null, "nll": 1.1831069678506654, "targets": 24576 }, "grad_norm": 0.5578569474663888, "lr": 0.00023368285271865155, "step": 50000, "tokens": 614400000, "tokens_per_second": 40223.48942093315, "train_loss": 1.2842901448607444, "train_seconds": 15274.65689439299 }, { "dev": { "bits_per_byte": null, "nll": 1.1844256588057818, "targets": 24576 }, "grad_norm": 0.5913718431509871, "lr": 0.00022875405163276448, "step": 52000, "tokens": 638976000, "tokens_per_second": 40224.731002705645, "train_loss": 1.2799920817613601, "train_seconds": 15885.152841843004 }, { "dev": { "bits_per_byte": null, "nll": 1.173831523364078, "targets": 24576 }, "grad_norm": 0.5844868823162954, "lr": 0.00022371332968597793, "step": 54000, "tokens": 663552000, "tokens_per_second": 40162.7098624142, "train_loss": 1.2714697028573696, "train_seconds": 16521.594341446 }, { "dev": { "bits_per_byte": null, "nll": 1.1719023542482427, "targets": 24576 }, "grad_norm": 0.5696322420893345, "lr": 0.00021856953591494743, "step": 56000, "tokens": 688128000, "tokens_per_second": 40149.40583977224, "train_loss": 1.2678369101285933, "train_seconds": 17139.18265057702 }, { "dev": { "bits_per_byte": null, "nll": 1.1704315858061074, "targets": 24576 }, "grad_norm": 0.57470837951865, "lr": 0.0002133317002999236, "step": 58000, "tokens": 712704000, "tokens_per_second": 40136.51323113992, "train_loss": 1.2606788657307624, "train_seconds": 17756.998369431065 }, { "dev": { "bits_per_byte": null, "nll": 1.1686503184390804, "targets": 24576 }, "grad_norm": 0.593138530484418, "lr": 0.00020800901791253362, "step": 60000, "tokens": 737280000, "tokens_per_second": 40121.34269525105, "train_loss": 1.251968766509054, "train_seconds": 18376.254394080086 }, { "dev": { "bits_per_byte": null, "nll": 1.1647448938377487, "targets": 24576 }, "grad_norm": 0.5961724355617762, "lr": 0.00020261083277374302, "step": 62000, "tokens": 761856000, "tokens_per_second": 40111.51973881791, "train_loss": 1.2475054085254669, "train_seconds": 18993.446395468138 }, { "dev": { "bits_per_byte": null, "nll": 1.1538009448481634, "targets": 24576 }, "grad_norm": 0.6139201341371189, "lr": 0.00019714662145033538, "step": 64000, "tokens": 786432000, "tokens_per_second": 40101.124340143266, "train_loss": 1.2433507589697839, "train_seconds": 19611.22070616712 }, { "dev": { "bits_per_byte": null, "nll": 1.1535151990586223, "targets": 24576 }, "grad_norm": 0.5897288431574661, "lr": 0.00019162597641870705, "step": 66000, "tokens": 811008000, "tokens_per_second": 40045.33174463077, "train_loss": 1.2397141224744606, "train_seconds": 20252.24825634611 }, { "dev": { "bits_per_byte": null, "nll": 1.1488343902246647, "targets": 24576 }, "grad_norm": 0.5967022131431149, "lr": 0.00018605858922518073, "step": 68000, "tokens": 835584000, "tokens_per_second": 40032.074306387156, "train_loss": 1.2318142200112343, "train_seconds": 20872.86293497616 }, { "dev": { "bits_per_byte": null, "nll": 1.1467947373682532, "targets": 24576 }, "grad_norm": 0.593558180090472, "lr": 0.00018045423347240125, "step": 70000, "tokens": 860160000, "tokens_per_second": 40017.78950500684, "train_loss": 1.2265688494443894, "train_seconds": 21494.44061352716 }, { "dev": { "bits_per_byte": null, "nll": 1.1424939681100101, "targets": 24576 }, "grad_norm": 0.6226438935129511, "lr": 0.00017482274766168076, "step": 72000, "tokens": 884736000, "tokens_per_second": 40004.60630142482, "train_loss": 1.2211514333132152, "train_seconds": 22115.853192848164 }, { "dev": { "bits_per_byte": null, "nll": 1.138869294531795, "targets": 24576 }, "grad_norm": 0.6580544595327715, "lr": 0.00016917401792141323, "step": 74000, "tokens": 909312000, "tokens_per_second": 39994.88224609538, "train_loss": 1.2172966094017028, "train_seconds": 22735.708894074174 }, { "dev": { "bits_per_byte": null, "nll": 1.1361407364744054, "targets": 24576 }, "grad_norm": 0.6581408191022303, "lr": 0.0001635179606518795, "step": 76000, "tokens": 933888000, "tokens_per_second": 39982.28630693697, "train_loss": 1.2028349896272024, "train_seconds": 23357.543708999186 }, { "dev": { "bits_per_byte": null, "nll": 1.1283157926086334, "targets": 24576 }, "grad_norm": 0.6406093455937347, "lr": 0.00015786450511690985, "step": 78000, "tokens": 958464000, "tokens_per_second": 39975.012302253905, "train_loss": 1.2053093385100364, "train_seconds": 23976.57798709318 }, { "dev": { "bits_per_byte": null, "nll": 1.1241916415925264, "targets": 24576 }, "grad_norm": 0.6612391196222913, "lr": 0.0001522235760129639, "step": 80000, "tokens": 983040000, "tokens_per_second": 39967.50170342419, "train_loss": 1.2011327757537364, "train_seconds": 24595.983188906168 }, { "dev": { "bits_per_byte": null, "nll": 1.1256335623562965, "targets": 24576 }, "grad_norm": 0.655634705150764, "lr": 0.00014660507604622917, "step": 82000, "tokens": 1007616000, "tokens_per_second": 39958.25809611626, "train_loss": 1.1896413449461132, "train_seconds": 25216.71484218015 }, { "dev": { "bits_per_byte": null, "nll": 1.1171926651397166, "targets": 24576 }, "grad_norm": 0.6478550144640661, "lr": 0.00014101886854832373, "step": 84000, "tokens": 1032192000, "tokens_per_second": 39951.77745971354, "train_loss": 1.1885131983757018, "train_seconds": 25835.94687472513 }, { "dev": { "bits_per_byte": null, "nll": 1.1190596509464787, "targets": 24576 }, "grad_norm": 0.6525248430353242, "lr": 0.0001354747601611207, "step": 86000, "tokens": 1056768000, "tokens_per_second": 39944.85267133271, "train_loss": 1.1814602708220483, "train_seconds": 26455.67399372116 }, { "dev": { "bits_per_byte": null, "nll": 1.1132559779381475, "targets": 24576 }, "grad_norm": 0.6476653683967294, "lr": 0.00012998248362109288, "step": 88000, "tokens": 1081344000, "tokens_per_second": 39937.50266405014, "train_loss": 1.1786426500296492, "train_seconds": 27075.904297175173 }, { "dev": { "bits_per_byte": null, "nll": 1.106964013007292, "targets": 24576 }, "grad_norm": 0.6835598310502361, "lr": 0.00012455168067339873, "step": 90000, "tokens": 1105920000, "tokens_per_second": 39933.73752238102, "train_loss": 1.170840959995985, "train_seconds": 27693.876621996194 }, { "dev": { "bits_per_byte": null, "nll": 1.104686529892949, "targets": 24576 }, "grad_norm": 0.6933287492892446, "lr": 0.0001191918851457043, "step": 92000, "tokens": 1130496000, "tokens_per_second": 39929.781570080464, "train_loss": 1.1664400331377982, "train_seconds": 28312.10078161522 }, { "dev": { "bits_per_byte": null, "nll": 1.0999679531370654, "targets": 24576 }, "grad_norm": 0.6747424570363748, "lr": 0.00011391250621145569, "step": 94000, "tokens": 1155072000, "tokens_per_second": 39923.34487769781, "train_loss": 1.1599219765007143, "train_seconds": 28932.245119703195 }, { "dev": { "bits_per_byte": null, "nll": 1.100811435482363, "targets": 24576 }, "grad_norm": 0.6676343560462272, "lr": 0.00010872281187198309, "step": 96000, "tokens": 1179648000, "tokens_per_second": 39919.22416257468, "train_loss": 1.1568616698682308, "train_seconds": 29550.874916701185 }, { "dev": { "bits_per_byte": null, "nll": 1.0970658371507978, "targets": 24576 }, "grad_norm": 0.6882154253552107, "lr": 0.00010363191268643382, "step": 98000, "tokens": 1204224000, "tokens_per_second": 39915.20102095341, "train_loss": 1.1486421208679676, "train_seconds": 30169.55869438926 }, { "dev": { "bits_per_byte": null, "nll": 1.0939033420323818, "targets": 24576 }, "grad_norm": 0.6833644373335422, "lr": 0.00009864874577809651, "step": 100000, "tokens": 1228800000, "tokens_per_second": 39909.817635011124, "train_loss": 1.1436737758131907, "train_seconds": 30789.416560050326 }, { "dev": { "bits_per_byte": null, "nll": 1.0885084713756228, "targets": 24576 }, "grad_norm": 0.6943333407110701, "lr": 0.00009378205914519423, "step": 102000, "tokens": 1253376000, "tokens_per_second": 39906.16482699491, "train_loss": 1.1405141915082933, "train_seconds": 31408.079564492295 }, { "dev": { "bits_per_byte": null, "nll": 1.0915803708139011, "targets": 24576 }, "grad_norm": 0.7230128053029767, "lr": 0.00008904039630368757, "step": 104000, "tokens": 1277952000, "tokens_per_second": 39902.007685613935, "train_loss": 1.1335780620574951, "train_seconds": 32027.260634826307 }, { "dev": { "bits_per_byte": null, "nll": 1.085777908168964, "targets": 24576 }, "grad_norm": 0.724568985381462, "lr": 0.00008443208128904915, "step": 106000, "tokens": 1302528000, "tokens_per_second": 39895.930202232856, "train_loss": 1.1290085162970889, "train_seconds": 32648.142138746305 }, { "dev": { "bits_per_byte": null, "nll": 1.0808364136402595, "targets": 24576 }, "grad_norm": 0.7391718987111651, "lr": 0.00007996520404333804, "step": 108000, "tokens": 1327104000, "tokens_per_second": 39891.81819665909, "train_loss": 1.1206585954129695, "train_seconds": 33267.57365276331 }, { "dev": { "bits_per_byte": null, "nll": 1.075356114801764, "targets": 24576 }, "grad_norm": 0.7285795209339059, "lr": 0.00007564760621322716, "step": 110000, "tokens": 1351680000, "tokens_per_second": 39888.94134339505, "train_loss": 1.11664033934474, "train_seconds": 33886.08357298046 }, { "dev": { "bits_per_byte": null, "nll": 1.0732655222608407, "targets": 24576 }, "grad_norm": 0.7370814509679531, "lr": 0.00007148686738391594, "step": 112000, "tokens": 1376256000, "tokens_per_second": 39884.86442838094, "train_loss": 1.1117175451797052, "train_seconds": 34505.72089749151 }, { "dev": { "bits_per_byte": null, "nll": 1.068521210136628, "targets": 24576 }, "grad_norm": 0.74495214231759, "lr": 0.00006749029177309461, "step": 114000, "tokens": 1400832000, "tokens_per_second": 39882.702115155415, "train_loss": 1.107140950500965, "train_seconds": 35123.798682328605 }, { "dev": { "bits_per_byte": null, "nll": 1.0669093107996581, "targets": 24576 }, "grad_norm": 0.7429760408093904, "lr": 0.0000636648954083184, "step": 116000, "tokens": 1425408000, "tokens_per_second": 39880.437402952986, "train_loss": 1.1043814161121845, "train_seconds": 35742.035263997735 }, { "dev": { "bits_per_byte": null, "nll": 1.06352839285792, "targets": 24576 }, "grad_norm": 0.753663374761045, "lr": 0.00006001739381030259, "step": 118000, "tokens": 1449984000, "tokens_per_second": 39877.09569434923, "train_loss": 1.096616155991757, "train_seconds": 36361.324082221705 }, { "dev": { "bits_per_byte": null, "nll": 1.059793703530472, "targets": 24576 }, "grad_norm": 0.7704402763444791, "lr": 0.000056554190203760464, "step": 120000, "tokens": 1474560000, "tokens_per_second": 39875.70181749022, "train_loss": 1.093344551116228, "train_seconds": 36978.91028348574 }, { "dev": { "bits_per_byte": null, "nll": 1.0545657625460578, "targets": 24576 }, "grad_norm": 0.770193637862138, "lr": 0.00005328136427647946, "step": 122000, "tokens": 1499136000, "tokens_per_second": 39874.39392150098, "train_loss": 1.0894255455732345, "train_seconds": 37596.45859323367 }, { "dev": { "bits_per_byte": null, "nll": 1.0510872257678383, "targets": 24576 }, "grad_norm": 0.7929503340062495, "lr": 0.00005020466150637006, "step": 124000, "tokens": 1523712000, "tokens_per_second": 39871.62226090378, "train_loss": 1.0822365283966064, "train_seconds": 38215.45032779064 }, { "dev": { "bits_per_byte": null, "nll": 1.0504651216027896, "targets": 24576 }, "grad_norm": 0.802954836084796, "lr": 0.0000473294830752229, "step": 126000, "tokens": 1548288000, "tokens_per_second": 39870.3578627684, "train_loss": 1.07673549503088, "train_seconds": 38833.06002241372 }, { "dev": { "bits_per_byte": null, "nll": 1.0493203217937563, "targets": 24576 }, "grad_norm": 0.7580745283341429, "lr": 0.000044660876386881586, "step": 128000, "tokens": 1572864000, "tokens_per_second": 39868.85020501433, "train_loss": 1.0750941784083843, "train_seconds": 39450.949598796804 }, { "dev": { "bits_per_byte": null, "nll": 1.045435637481224, "targets": 24576 }, "grad_norm": 0.7754103804601181, "lr": 0.00004220352620647569, "step": 130000, "tokens": 1597440000, "tokens_per_second": 39865.39312597222, "train_loss": 1.072553506293821, "train_seconds": 40070.84528057171 }, { "dev": { "bits_per_byte": null, "nll": 1.0441179985200633, "targets": 24576 }, "grad_norm": 0.8122855416949406, "lr": 0.00003996174643627, "step": 132000, "tokens": 1622016000, "tokens_per_second": 39863.778706117286, "train_loss": 1.0693979724645615, "train_seconds": 40688.96759531464 }, { "dev": { "bits_per_byte": null, "nll": 1.042489304855985, "targets": 24576 }, "grad_norm": 0.7651654635484804, "lr": 0.00003793947254256663, "step": 134000, "tokens": 1646592000, "tokens_per_second": 39860.67686826115, "train_loss": 1.0660356606467296, "train_seconds": 41308.6813714167 }, { "dev": { "bits_per_byte": null, "nll": 1.04162178599293, "targets": 24576 }, "grad_norm": 0.7961447866924479, "lr": 0.0000361402546469557, "step": 136000, "tokens": 1671168000, "tokens_per_second": 39859.148461034274, "train_loss": 1.0603021009266376, "train_seconds": 41926.83648607571 }, { "dev": { "bits_per_byte": null, "nll": 1.0385937039291362, "targets": 24576 }, "grad_norm": 0.820799219968492, "lr": 0.000034567251294042076, "step": 138000, "tokens": 1695744000, "tokens_per_second": 39857.62071143589, "train_loss": 1.0583040608465673, "train_seconds": 42545.03830715263 }, { "dev": { "bits_per_byte": null, "nll": 1.036851932944917, "targets": 24576 }, "grad_norm": 0.7704111480996634, "lr": 0.000033223223906589426, "step": 140000, "tokens": 1720320000, "tokens_per_second": 39853.75851886772, "train_loss": 1.057887106668204, "train_seconds": 43165.81582099865 }, { "dev": { "bits_per_byte": null, "nll": 1.0364988710288603, "targets": 24576 }, "grad_norm": 0.8115261183417851, "lr": 0.00003211053193781564, "step": 142000, "tokens": 1744896000, "tokens_per_second": 39848.667793489054, "train_loss": 1.0517295686304569, "train_seconds": 43788.06360711265 }, { "dev": { "bits_per_byte": null, "nll": 1.0345454902247837, "targets": 24576 }, "grad_norm": 0.7876409056413154, "lr": 0.00003123112872934953, "step": 144000, "tokens": 1769472000, "tokens_per_second": 39845.64161657429, "train_loss": 1.0489775944650173, "train_seconds": 44408.169330719626 }, { "dev": { "bits_per_byte": null, "nll": 1.0353213657639142, "targets": 24576 }, "grad_norm": 0.8414267591493471, "lr": 0.00003058655808212059, "step": 146000, "tokens": 1794048000, "tokens_per_second": 39842.03970391353, "train_loss": 1.0453252461221483, "train_seconds": 45029.019933027615 }, { "dev": { "bits_per_byte": null, "nll": 1.0357807584457464, "targets": 24576 }, "grad_norm": 0.8355712088117428, "lr": 0.000030177951546201323, "step": 148000, "tokens": 1818624000, "tokens_per_second": 39840.50131008946, "train_loss": 1.0446138744354247, "train_seconds": 45647.618383241585 }, { "dev": { "bits_per_byte": null, "nll": 1.0333022224068305, "targets": 24576 }, "grad_norm": 0.8683022559664173, "lr": 0.00003000602643436008, "step": 150000, "tokens": 1843200000, "tokens_per_second": 39839.01916177443, "train_loss": 1.0458105452656745, "train_seconds": 46266.19928857465 }, { "dev": { "bits_per_byte": null, "nll": 1.0331160305800633, "targets": 24576 }, "grad_norm": 0.8198040794457188, "lr": 0.000030000000029628566, "step": 150450, "tokens": 1848729600, "tokens_per_second": 39838.63395166432, "train_loss": 1.040829320881102, "train_seconds": 46405.44658843069 } ], "data_parallel": 1, "executable": { "bytes": 34479000, "path": "/bin/7c682fdb928a461a5d12489ae50817f9ba53cd85-sm120/geometric-stack", "sha256": "ae44391ad594de8b11db4df40b75837274a6e187c4d5913ffc2ece01dff5edae" }, "final": { "bits_per_byte": null, "nll": 1.032235825163159, "targets": 196608 }, "inputs": { "init": null, "lens": null, "merges": null, "tokenizer": { "bytes": 109457, "path": "/data/tokenizer.json", "sha256": "d36d3e8700a123e620012df77de195f244fbdb4d05d9e1aa7e77fa9407590f89" }, "train": [ { "bytes": 1105336084, "path": "/data/corpora/ts-train/tokens.u16", "sha256": "2ba5e64fe076bbb5ffd037df4ff90f697e793f96fda01ada921f1db2e533f0fc" }, { "bytes": 85555972, "path": "/data/corpora/td-train/tokens.u16", "sha256": "b4e24f1f202f2375dc830422c50be5b48d6c1be6a39aeaa7173a2fb84e9f50d3" }, { "bytes": 164496986, "path": "/data/chat-v0-p2/train/tokens.u16", "sha256": "b5d3bf977a524d9e47c0c75592d0170a183271ddf0cf4d7b560e3ece7a54f184" } ], "valid": { "bytes": 11159720, "path": "/data/corpora/ts-valid/tokens.u16", "sha256": "8ebf9db3033963aa5a3a34fe91f4715fe275c73f7c4e4ca7b98cf9a2d87af422" } }, "model_sha256": "e448de86ae30ab0785e2c8400dcf32e29b97e8ed03830b14183371bb1664edcc", "parameters": 213932136, "resumed_from": { "bytes": 28643, "path": "/runs/step8-chunks/c14/checkpoint/state.json", "sha256": "d2e55e9c330963d0be06dc400cb31fd46eb6b6bc704f43b6721988f85b7f0fd2" }, "samples": { "rows": [ { "greedy": ", there was a little girl named Lily. She loved to play with her toys and eat yummy food. One day, she found a big, red apple in the kitchen. Lily was very happy and wanted to eat it right away.\nBut, Lily's mom said, \"Wait, Lily! That apple is spoiled. It will make you sick.\" Lily was sad, but she listened to her mom. She put the apple on the table and went to play with her toys.\nLater, Lily's mom gave her a new apple. It was green and looked yummy. Lily took a bite and it was so good! She", "greedy_ids": [ 14, 403, 285, 261, 393, 477, 404, 407, 16, 341, 504, 268, 327, 331, 338, 624, 269, 711, 1036, 804, 16, 530, 329, 14, 394, 505, 261, 348, 14, 677, 1467, 315, 265, 1170, 16, 407, 285, 381, 378, 269, 409, 268, 711, 311, 1194, 575, 16, 201, 769, 14, 407, 386, 390, 326, 14, 319, 57, 4033, 14, 407, 3, 1354, 1467, 435, 2920, 16, 427, 619, 533, 351, 2198, 399, 407, 285, 496, 14, 412, 394, 1369, 268, 338, 390, 16, 341, 572, 265, 1467, 356, 265, 1288, 269, 430, 268, 327, 331, 338, 624, 16, 201, 2180, 14, 407, 386, 390, 765, 338, 261, 545, 1467, 16, 427, 285, 1074, 269, 506, 1036, 16, 407, 614, 261, 1741, 269, 311, 285, 391, 561, 3, 341 ], "prompt": "u don't have to be scared of the loud dog, I'll protect you\". The mole felt so safe with the little girl. She was very kind and the mole soon came to trust her. He leaned against her and she kept him safe. The mole had found his best friend.<|eos|>Once upon a time", "prompt_start": 0, "sampled": ", in a small town, there lived a girl named Amy. Amy was a very patient girl. She always waited for her turn and never got mad at her friends.\nOne day, Amy's mom gave her a big red ball. She told Amy to stay in the yard while she went to the store. Amy played with her ball, but she couldn't stop thinking about playing with her friends. She saw her friend Tim walking by her house and decided to go play with him.\nWhile playing with Tim, Amy's ball rolled away and got lost. She was very sad and started to cry. A kind boy named Tom saw", "sampled_ids": [ 14, 315, 261, 563, 1013, 14, 403, 630, 261, 477, 404, 830, 16, 830, 285, 261, 381, 2059, 477, 16, 341, 701, 1524, 388, 338, 882, 269, 856, 659, 1821, 452, 338, 411, 16, 201, 385, 329, 14, 830, 386, 390, 765, 338, 261, 348, 677, 481, 16, 341, 743, 830, 268, 963, 315, 265, 1235, 1001, 394, 430, 268, 265, 1123, 16, 830, 476, 331, 338, 481, 14, 412, 394, 1060, 488, 1275, 4006, 666, 586, 331, 338, 411, 16, 341, 384, 338, 360, 344, 1272, 860, 338, 603, 269, 709, 268, 480, 327, 331, 474, 16, 201, 1766, 586, 331, 344, 14, 830, 386, 481, 1759, 575, 269, 659, 1056, 16, 341, 285, 381, 496, 269, 549, 268, 1050, 16, 439, 803, 501, 404, 417, 384 ] }, { "greedy": "fly. It can fly and breathe fire.\"\nThe pupils were amazed. They had never seen a dragonfly before. They were not scared anymore. They were excited to learn about the dragonfly. They thanked Miss Amy for showing them the dragonfly. They knew it was a special day.<|eos|>Once upon a time, there was a little girl named Lily. She loved to play with her toys and eat yummy food. One day, she found a big, red apple in the kitchen. Lily was very happy and wanted to eat it right away.\nBut, Lily's mom said,", "greedy_ids": [ 72, 463, 16, 427, 469, 849, 269, 1120, 299, 260, 1306, 399, 201, 413, 1406, 82, 316, 85, 406, 2113, 16, 314, 366, 856, 1500, 261, 1958, 72, 463, 1092, 16, 314, 406, 367, 611, 858, 16, 314, 406, 726, 268, 620, 666, 265, 1958, 72, 463, 16, 314, 925, 3740, 830, 388, 3657, 495, 265, 1958, 72, 463, 16, 314, 676, 311, 285, 261, 854, 329, 16, 1, 431, 440, 261, 400, 14, 403, 285, 261, 393, 477, 404, 407, 16, 341, 504, 268, 327, 331, 338, 624, 269, 711, 1036, 804, 16, 530, 329, 14, 394, 505, 261, 348, 14, 677, 1467, 315, 265, 1170, 16, 407, 285, 381, 378, 269, 409, 268, 711, 311, 1194, 575, 16, 201, 769, 14, 407, 386, 390, 326, 14 ], "prompt": " Lisa and Tim looking at the bug. She walked over and said, \"What are you two looking at?\" Lisa said, \"We found a bug with big eyes!\" Miss Amy looked at the bug. She realized it was a special bug. She told her pupils, \"This bug is called a dragon", "prompt_start": 1859921, "sampled": "fly. It can fly very high and very fast.\"\nLisa and Tim were surprised. They didn't know that dragonflies could fly high. They thanked Miss Amy for telling them about the dragonfly. Now, they could all play with their new friend.<|eos|>One day, a little boy named Tim went to play outside. It was a mild day, not too hot and not too cold. He saw his friend, Sam, and they decided to play catch. They took turns throwing a ball to each other, laughing and having fun.\n\"Let's see who can throw the ball higher!\" said", "sampled_ids": [ 72, 463, 16, 427, 469, 849, 381, 901, 269, 381, 733, 399, 201, 46, 2037, 269, 344, 406, 900, 16, 314, 802, 488, 657, 383, 1958, 72, 3625, 468, 849, 901, 16, 314, 925, 3740, 830, 388, 1265, 300, 495, 666, 265, 1958, 72, 463, 16, 1137, 14, 364, 468, 434, 327, 331, 470, 545, 360, 16, 1, 385, 329, 14, 261, 393, 501, 404, 344, 430, 268, 327, 699, 16, 427, 285, 261, 3864, 329, 14, 367, 425, 1254, 269, 367, 425, 1214, 16, 318, 384, 343, 360, 14, 551, 14, 269, 364, 709, 268, 327, 1129, 16, 314, 614, 1579, 1779, 300, 261, 481, 268, 758, 557, 14, 1783, 269, 1121, 446, 16, 201, 4, 897, 386, 538, 660, 469, 1779, 265, 481, 1857, 415, 326 ] }, { "greedy": "Soon, the waiter came with a big box. He put the box on the table and opened it. Inside was a big cake with candles and frosting. It had candles and frosting and sprinkles. It also had a sign that said \"Happy Birthday, Tom and Lily!\"\nTom and Lily were very happy. They clapped their hands and shouted \"Yay, cake!\" They ran to the table and grabbed a slice each. They blew out the candles and made a wish. They wished for more fun and more love.\nThe waiter smiled and said \"Your cake", "greedy_ids": [ 2302, 14, 265, 1168, 284, 553, 331, 261, 348, 616, 16, 318, 572, 265, 616, 356, 265, 1288, 269, 905, 311, 16, 1664, 285, 261, 348, 885, 331, 2598, 956, 269, 493, 2082, 300, 16, 427, 366, 2598, 956, 269, 493, 2082, 300, 269, 1711, 749, 956, 16, 427, 1067, 366, 261, 1689, 383, 326, 319, 42, 3206, 376, 2035, 14, 417, 269, 407, 415, 201, 740, 269, 407, 406, 381, 378, 16, 314, 1478, 470, 1091, 269, 1494, 319, 59, 282, 14, 885, 415, 314, 594, 268, 265, 1288, 269, 1513, 261, 756, 552, 758, 16, 314, 1540, 486, 265, 2598, 956, 269, 578, 261, 1729, 16, 314, 1441, 388, 674, 446, 269, 674, 863, 16, 201, 413, 1168, 284, 565, 269, 326, 319, 536, 84, 885 ], "prompt": " be careful and not to make a mess.\nTom and Lily sat on their chairs and looked at the menu. They saw many things they liked, such as pizza, pasta, and ice cream. They told Mom and Dad what they wanted to eat. Mom and Dad nodded and called the waiter.\n", "prompt_start": 3719842, "sampled": "\"Can you order for us, please?\" Tom and Lily said together.\n\"Of course, we will be there soon,\" the waiter said. \"You are very polite and polite. What do you want to order?\"\nTom and Lily thought hard. They did not know what to order. Tom wanted pizza and ice cream. Lily wanted ice cream and cookies. They looked at the menu again.\n\"Can we order pizza and ice cream, please?\" Tom and Lily said together.\nThe waiter smiled and said: \"Sure, that sounds like a good order. What flavor do you want to order", "sampled_ids": [ 4, 1038, 351, 3624, 388, 824, 14, 1293, 478, 417, 269, 407, 326, 458, 16, 201, 4, 2181, 1984, 14, 365, 619, 324, 403, 1084, 638, 265, 1168, 284, 326, 16, 319, 536, 484, 381, 2599, 269, 2599, 16, 1601, 377, 351, 371, 268, 3624, 478, 201, 740, 269, 407, 585, 881, 16, 314, 454, 367, 657, 579, 268, 3624, 16, 417, 409, 2419, 269, 1264, 1450, 16, 407, 409, 1264, 1450, 269, 1665, 16, 314, 506, 452, 265, 4011, 87, 602, 16, 201, 4, 1038, 365, 3624, 2419, 269, 1264, 1450, 14, 1293, 478, 417, 269, 407, 326, 458, 16, 201, 413, 1168, 284, 565, 269, 326, 28, 319, 2166, 14, 383, 1995, 517, 261, 561, 3624, 16, 1601, 608, 1410, 306, 377, 351, 371, 268, 3624 ] } ], "seed": 1, "temperature": 0.8, "top_k": 40 }, "schema": "uor-r4.geometric-stack-run/1", "settings": { "batch": 32, "checkpoint_every": 1000000, "clip": 1.0, "config": { "arch": "geometric", "context": 384, "heads": 24, "mlp_hidden": 675, "pattern": "rrarrarrarrarrar", "read": "l2", "rotation": true, "seed": 1, "vocab_size": 4096, "width": 1536 }, "eval_every": 2000, "eval_windows": 64, "final_windows": 512, "init": null, "key_shift": "false", "lens": null, "lr": 0.0003, "merges": null, "min_lr": 0.1, "precision": "bf16", "qat": false, "qat_codec": null, "sample_tokens": 128, "steps": 150450, "tokenizer": "/data/tokenizer.json", "train": [ "/data/corpora/ts-train/tokens.u16", "/data/corpora/td-train/tokens.u16", "/data/chat-v0-p2/train/tokens.u16" ], "train_weights": [ 0.7, 0.1, 0.2 ], "transport_snap": null, "valid": "/data/corpora/ts-valid/tokens.u16", "warmup": 500, "weight_decay": 0.1 }, "step_seconds": { "first": 2.2208142459999998, "max_after_first": 0.379016785, "mean_after_first": 0.30926662861246446, "min_after_first": 0.305886248, "steps": 4927 }, "stopped_early": false, "target_visits": 1848729600, "tf32": false, "threads": "16", "tokens_per_second": 39838.63395166432, "train_seconds": 46405.44658843069 }