{ "active_parameters": 213932136, "completed_steps": 150000, "curve": [ { "dev": { "bits_per_byte": null, "nll": 2.9030416484550723, "targets": 24576 }, "step": 0, "tokens": 0 }, { "dev": { "bits_per_byte": null, "nll": 2.5523055809480346, "targets": 24576 }, "grad_norm": 1.0545403837101495, "lr": 0.00019975682322914177, "step": 4000, "tokens": 49152000, "tokens_per_second": 46806.65662814915, "train_loss": 2.3252306063473225, "train_seconds": 1050.1070476040022 }, { "dev": { "bits_per_byte": null, "nll": 2.4913074550555177, "targets": 24576 }, "grad_norm": 1.030744139549884, "lr": 0.00019888483875168944, "step": 8000, "tokens": 98304000, "tokens_per_second": 46838.538954778596, "train_loss": 2.2114218744337557, "train_seconds": 2098.784509374 }, { "dev": { "bits_per_byte": null, "nll": 2.4607102592082426, "targets": 24576 }, "grad_norm": 0.9020604931683426, "lr": 0.00019738521615714398, "step": 12000, "tokens": 147456000, "tokens_per_second": 46850.96059260544, "train_loss": 2.1620804379582403, "train_seconds": 3147.3420850899947 }, { "dev": { "bits_per_byte": null, "nll": 2.4214659397649716, "targets": 24576 }, "grad_norm": 0.9110601300097976, "lr": 0.00019526854465638546, "step": 16000, "tokens": 196608000, "tokens_per_second": 46853.7382880609, "train_loss": 2.1298196042478086, "train_seconds": 4196.207329097984 }, { "dev": { "bits_per_byte": null, "nll": 2.3989440543214307, "targets": 24576 }, "grad_norm": 0.8728188850669167, "lr": 0.0001925497705972265, "step": 20000, "tokens": 245760000, "tokens_per_second": 46857.53183907725, "train_loss": 2.1031029906570913, "train_seconds": 5244.83450908198 }, { "dev": { "bits_per_byte": null, "nll": 2.3877681926971976, "targets": 24576 }, "grad_norm": 0.8841449842811544, "lr": 0.00018924809192450505, "step": 24000, "tokens": 294912000, "tokens_per_second": 46859.61076428494, "train_loss": 2.086106381505728, "train_seconds": 6293.522186590025 }, { "dev": { "bits_per_byte": null, "nll": 2.3719195768016115, "targets": 24576 }, "grad_norm": 0.894962556631147, "lr": 0.00018538682261858605, "step": 28000, "tokens": 344064000, "tokens_per_second": 46861.61144046403, "train_loss": 2.066322033762932, "train_seconds": 7342.129078022013 }, { "dev": { "bits_per_byte": null, "nll": 2.3583353023906253, "targets": 24576 }, "grad_norm": 0.8866391587352401, "lr": 0.00018099322806950442, "step": 32000, "tokens": 393216000, "tokens_per_second": 46862.79992058527, "train_loss": 2.0493402725458143, "train_seconds": 8390.791857643002 }, { "dev": { "bits_per_byte": null, "nll": 2.34749530461174, "targets": 24576 }, "grad_norm": 0.9201245256818397, "lr": 0.00017609833254921453, "step": 36000, "tokens": 442368000, "tokens_per_second": 46863.160256637035, "train_loss": 2.0403972225785254, "train_seconds": 9439.568257399997 }, { "dev": { "bits_per_byte": null, "nll": 2.335180931759818, "targets": 24576 }, "grad_norm": 0.8469721411139691, "lr": 0.00017073670014143273, "step": 40000, "tokens": 491520000, "tokens_per_second": 46863.35894667588, "train_loss": 2.0266376483142374, "train_seconds": 10488.364706407043 }, { "dev": { "bits_per_byte": null, "nll": 2.3184834107242733, "targets": 24576 }, "grad_norm": 0.8941298671223565, "lr": 0.00016494619067598177, "step": 44000, "tokens": 540672000, "tokens_per_second": 46863.58203161271, "train_loss": 2.0138602941930293, "train_seconds": 11537.14625645303 }, { "dev": { "bits_per_byte": null, "nll": 2.314498261884701, "targets": 24576 }, "grad_norm": 0.882209571943328, "lr": 0.0001587676923910466, "step": 48000, "tokens": 589824000, "tokens_per_second": 46864.37748656758, "train_loss": 2.00153856036067, "train_seconds": 12585.764105563063 }, { "dev": { "bits_per_byte": null, "nll": 2.303414721554431, "targets": 24576 }, "grad_norm": 0.871290039390319, "lr": 0.00015224483321107956, "step": 52000, "tokens": 638976000, "tokens_per_second": 46864.71210296379, "train_loss": 1.992524571210146, "train_seconds": 13634.480429457075 }, { "dev": { "bits_per_byte": null, "nll": 2.2966258138980553, "targets": 24576 }, "grad_norm": 0.8562851237311796, "lr": 0.00014542367267909468, "step": 56000, "tokens": 688128000, "tokens_per_second": 45568.8898896263, "train_loss": 1.983941870421171, "train_seconds": 15100.828694022048 }, { "dev": { "bits_per_byte": null, "nll": 2.291375207043408, "targets": 24576 }, "grad_norm": 0.915086840641239, "lr": 0.00013835237671869392, "step": 60000, "tokens": 737280000, "tokens_per_second": 39798.42363273046, "train_loss": 1.9730176376402377, "train_seconds": 18525.356853422116 }, { "dev": { "bits_per_byte": null, "nll": 2.275626207080917, "targets": 24576 }, "grad_norm": 0.9407370947267671, "lr": 0.00013108087752241226, "step": 64000, "tokens": 786432000, "tokens_per_second": 39486.86985354042, "train_loss": 1.9636255254149437, "train_seconds": 19916.291235971137 }, { "dev": { "bits_per_byte": null, "nll": 2.2725748816495313, "targets": 24576 }, "grad_norm": 0.9474793497819595, "lr": 0.00012366052096799557, "step": 68000, "tokens": 835584000, "tokens_per_second": 39856.32303515696, "train_loss": 1.9539809826612473, "train_seconds": 20964.90434561507 }, { "dev": { "bits_per_byte": null, "nll": 2.2620197626138516, "targets": 24576 }, "grad_norm": 0.9346774208293191, "lr": 0.00011614370405229254, "step": 72000, "tokens": 884736000, "tokens_per_second": 40190.768314598026, "train_loss": 1.943806702464819, "train_seconds": 22013.41345541403 }, { "dev": { "bits_per_byte": null, "nll": 2.2484762282965414, "targets": 24576 }, "grad_norm": 0.932023935217344, "lr": 0.00010858350490293175, "step": 76000, "tokens": 933888000, "tokens_per_second": 40494.73956433892, "train_loss": 1.9345955960452557, "train_seconds": 23061.958418480965 }, { "dev": { "bits_per_byte": null, "nll": 2.2425845804625486, "targets": 24576 }, "grad_norm": 0.9758921038757217, "lr": 0.0001010333079803633, "step": 80000, "tokens": 983040000, "tokens_per_second": 40772.388173045736, "train_loss": 1.9195933798849583, "train_seconds": 24110.434636003956 }, { "dev": { "bits_per_byte": null, "nll": 2.233739086815889, "targets": 24576 }, "grad_norm": 0.9482285472073453, "lr": 0.00009354642711680783, "step": 84000, "tokens": 1032192000, "tokens_per_second": 41026.889062110116, "train_loss": 1.9145210573375224, "train_seconds": 25158.914643451928 }, { "dev": { "bits_per_byte": null, "nll": 2.223805864646233, "targets": 24576 }, "grad_norm": 0.9275743205891211, "lr": 0.00008617572905392898, "step": 88000, "tokens": 1081344000, "tokens_per_second": 41260.94684435442, "train_loss": 1.903701186001301, "train_seconds": 26207.44511945092 }, { "dev": { "bits_per_byte": null, "nll": 2.2169806678866166, "targets": 24576 }, "grad_norm": 1.0070880977426901, "lr": 0.0000789732601375245, "step": 92000, "tokens": 1130496000, "tokens_per_second": 41477.135184684776, "train_loss": 1.894820238173008, "train_seconds": 27255.884355711965 }, { "dev": { "bits_per_byte": null, "nll": 2.2083875170919876, "targets": 24576 }, "grad_norm": 1.014914219900583, "lr": 0.00007198987880523799, "step": 96000, "tokens": 1179648000, "tokens_per_second": 41677.255140334404, "train_loss": 1.8844552681446076, "train_seconds": 28304.359200910058 }, { "dev": { "bits_per_byte": null, "nll": 2.200075198501621, "targets": 24576 }, "grad_norm": 0.9940838151149737, "lr": 0.00006527489646238776, "step": 100000, "tokens": 1228800000, "tokens_per_second": 41862.882710518614, "train_loss": 1.8750371419787406, "train_seconds": 29352.971425717114 }, { "dev": { "bits_per_byte": null, "nll": 2.18731408059489, "targets": 24576 }, "grad_norm": 1.0063372319480717, "lr": 0.000058875729281778746, "step": 104000, "tokens": 1277952000, "tokens_per_second": 42035.75330216055, "train_loss": 1.86530925360322, "train_seconds": 30401.5486724801 }, { "dev": { "bits_per_byte": null, "nll": 2.182616288567267, "targets": 24576 }, "grad_norm": 1.0063333820365328, "lr": 0.00005283756338622555, "step": 108000, "tokens": 1327104000, "tokens_per_second": 42197.15140210186, "train_loss": 1.855627230644226, "train_seconds": 31450.08503900802 }, { "dev": { "bits_per_byte": null, "nll": 2.1749686576465064, "targets": 24576 }, "grad_norm": 1.0554865318172735, "lr": 0.00004720303577801807, "step": 112000, "tokens": 1376256000, "tokens_per_second": 42347.78123689638, "train_loss": 1.8480984157919884, "train_seconds": 32498.89273539811 }, { "dev": { "bits_per_byte": null, "nll": 2.1692987409236464, "targets": 24576 }, "grad_norm": 1.0331397226418841, "lr": 0.000042011933268364866, "step": 116000, "tokens": 1425408000, "tokens_per_second": 42489.06546017219, "train_loss": 1.8391764216423034, "train_seconds": 33547.64301267415 }, { "dev": { "bits_per_byte": null, "nll": 2.161684334605468, "targets": 24576 }, "grad_norm": 1.0525274692674875, "lr": 0.00003730091153275069, "step": 120000, "tokens": 1474560000, "tokens_per_second": 42621.87874832813, "train_loss": 1.8334151634275913, "train_seconds": 34596.31633572325 }, { "dev": { "bits_per_byte": null, "nll": 2.154300959904038, "targets": 24576 }, "grad_norm": 1.0591538977784967, "lr": 0.0000331032362760275, "step": 124000, "tokens": 1523712000, "tokens_per_second": 42746.36262305084, "train_loss": 1.8259893079400062, "train_seconds": 35645.418849704496 }, { "dev": { "bits_per_byte": null, "nll": 2.1510941852404897, "targets": 24576 }, "grad_norm": 1.1107467981292782, "lr": 0.00002944854833493756, "step": 128000, "tokens": 1572864000, "tokens_per_second": 42864.019477765905, "train_loss": 1.8219300164580345, "train_seconds": 36694.27223958462 }, { "dev": { "bits_per_byte": null, "nll": 2.147425336745802, "targets": 24576 }, "grad_norm": 1.0390858468322515, "lr": 0.00002636265437673842, "step": 132000, "tokens": 1622016000, "tokens_per_second": 42975.00937725712, "train_loss": 1.8122640961110592, "train_seconds": 37743.237837625464 }, { "dev": { "bits_per_byte": null, "nll": 2.1407593697144387, "targets": 24576 }, "grad_norm": 1.0958744125982507, "lr": 0.00002386734467185946, "step": 136000, "tokens": 1671168000, "tokens_per_second": 43079.97623148053, "train_loss": 1.8093835397064686, "train_seconds": 38792.22195992765 }, { "dev": { "bits_per_byte": null, "nll": 2.140339879902316, "targets": 24576 }, "grad_norm": 1.0444245768296487, "lr": 0.00002198023922734275, "step": 140000, "tokens": 1720320000, "tokens_per_second": 43179.59708889657, "train_loss": 1.8025973540842533, "train_seconds": 39841.03873082161 }, { "dev": { "bits_per_byte": null, "nll": 2.1363886775093928, "targets": 24576 }, "grad_norm": 1.0430003529307572, "lr": 0.000020714663367558396, "step": 144000, "tokens": 1769472000, "tokens_per_second": 43273.96260091018, "train_loss": 1.8014133936464787, "train_seconds": 40889.992356807714 }, { "dev": { "bits_per_byte": null, "nll": 2.133301656815748, "targets": 24576 }, "grad_norm": 1.152501292693433, "lr": 0.000020079553640750505, "step": 148000, "tokens": 1818624000, "tokens_per_second": 43363.7590745239, "train_loss": 1.795748849272728, "train_seconds": 41938.79956012478 }, { "dev": { "bits_per_byte": null, "nll": 2.1322520364301014, "targets": 24576 }, "grad_norm": 1.1306620740321143, "lr": 0.000020000000019871466, "step": 150000, "tokens": 1843200000, "tokens_per_second": 43406.85716410915, "train_loss": 1.792580764055252, "train_seconds": 42463.33691083365 } ], "data_parallel": 1, "executable": { "bytes": 34788112, "path": "/target/release/examples/geometric-stack", "sha256": "94bc6ec8913fe1abb1f529672c676aeb9dd4b0b329a683f80a6bbc3e3d155ec3" }, "final": { "bits_per_byte": null, "nll": 2.0732471856125962, "targets": 196608 }, "inputs": { "init": { "bytes": 855748328, "path": "/preroot/model/model.safetensors", "sha256": "e448de86ae30ab0785e2c8400dcf32e29b97e8ed03830b14183371bb1664edcc" }, "lens": null, "merges": null, "tokenizer": { "bytes": 109457, "path": "/in/tokenizer.json", "sha256": "d36d3e8700a123e620012df77de195f244fbdb4d05d9e1aa7e77fa9407590f89" }, "train": [ { "bytes": 2815995020, "path": "/smollm-tok/fw0/tokens.u16", "sha256": "bcc853fec8cba6a38b1877fe5bd23cc8b251591f1858ddba223f3a1c0a0c3663" }, { "bytes": 2827283732, "path": "/smollm-tok/fw1/tokens.u16", "sha256": "d3d27a2ba38d3f8c4d9bc1bd273becfe3d22a10a2514c34f05ceb913cc4bdcf6" }, { "bytes": 998312182, "path": "/smollm-tok/cos0/tokens.u16", "sha256": "0093b878832960d944e8f9bc03df7ce5502bcad7da0e2457f08e22686379999a" }, { "bytes": 997625570, "path": "/smollm-tok/cos1/tokens.u16", "sha256": "47b147e6d85ccfda437f60417c2febdfdc25035d4a2099a8e789e96f8da84e08" }, { "bytes": 998967134, "path": "/smollm-tok/cos2/tokens.u16", "sha256": "a997875790ff4b74a20fa220b60eb9b95dbb6b15ab231612541c001ed1fefb93" }, { "bytes": 998260804, "path": "/smollm-tok/cos3/tokens.u16", "sha256": "d10eab9ec3aed1a9d7e2a0bf95cd9046e5e0216e40240e41f4904f3f6578d34f" } ], "valid": { "bytes": 11292954, "path": "/smollm-tok/fw-valid/tokens.u16", "sha256": "d258868ba767f145a63800c83b824a32c3ed78c38b18fd64413c82341cebdf74" } }, "model_sha256": "2b425ccdb2b5eae4d3f8dab924ee09cb6d167cb7aa65188de7cf9d24ef6d6586", "parameters": 213932136, "resumed_from": null, "samples": null, "schema": "uor-r4.geometric-stack-run/1", "settings": { "batch": 32, "checkpoint_every": 4000, "clip": 1.0, "config": { "arch": "geometric", "context": 384, "heads": 24, "mlp_hidden": 675, "pattern": "rrarrarrarrarrar", "read": "l2", "rotation": true, "seed": 1, "vocab_size": 4096, "width": 1536 }, "eval_every": 4000, "eval_windows": 64, "final_windows": 512, "init": { "config": { "bytes": 200, "path": "/preroot/model/config.json", "sha256": "8f9788549fd6950a51ffbd660c5e908a7bb81ebc31321d3f2a4a16141bc85cff" }, "model": { "bytes": 855748328, "path": "/preroot/model/model.safetensors", "sha256": "e448de86ae30ab0785e2c8400dcf32e29b97e8ed03830b14183371bb1664edcc" } }, "key_shift": "false", "lens": null, "lr": 0.0002, "merges": null, "min_lr": 0.1, "precision": "bf16", "qat": false, "qat_codec": null, "sample_tokens": 0, "steps": 150000, "tokenizer": "/in/tokenizer.json", "train": [ "/smollm-tok/fw0/tokens.u16", "/smollm-tok/fw1/tokens.u16", "/smollm-tok/cos0/tokens.u16", "/smollm-tok/cos1/tokens.u16", "/smollm-tok/cos2/tokens.u16", "/smollm-tok/cos3/tokens.u16" ], "train_weights": [ 0.35, 0.35, 0.075, 0.075, 0.075, 0.075 ], "transport_snap": null, "valid": "/smollm-tok/fw-valid/tokens.u16", "warmup": 500, "weight_decay": 0.1 }, "step_seconds": { "first": 1.318258063, "max_after_first": 0.875829171, "mean_after_first": 0.2830820115652147, "min_after_first": 0.258904896, "steps": 150000 }, "stopped_early": false, "target_visits": 1843200000, "tf32": true, "threads": "16", "tokens_per_second": 43406.85716410915, "train_seconds": 42463.33691083365 }