{
  "architecture": {
    "attn_pdrop": 0.0,
    "bos_token_id": 128000,
    "compute_dtype": "bf16",
    "embd_pdrop": 0.0,
    "eos_token_id": 128001,
    "feed_forward_prime": "swiglu",
    "force_flash": false,
    "hidden_size": 2048,
    "initializer_range": 0.02,
    "intermediate_size": 4224,
    "mini_batch_size": 1024,
    "num_attention_heads": 32,
    "num_hidden_layers": 24,
    "output_size": 128256,
    "param_dtype": "fp32",
    "post_norm": true,
    "pre_norm": true,
    "prime": true,
    "prime_start": 18,
    "qk_norm": true,
    "resid_pdrop": 0.0,
    "rms_norm_eps": 1e-06,
    "rope_theta": 500000.0,
    "seq_modeling_block": "SWA",
    "sliding_window_size": 8192,
    "state_dtype": "fp32",
    "suffix_len": 6,
    "swa_attention_backend": "auto",
    "tie_word_embeddings": true,
    "vocab_size": 128256
  },
  "format": "e2e-routed-router-v7",
  "router": {
    "adapter_rank": 128,
    "adapter_silu": true,
    "cadence": 4,
    "dead_branch_skip": false,
    "exit_ce_sites_per_chunk": 0,
    "exit_ce_vocab_chunk_size": 4096,
    "exit_norm": true,
    "input_norm_eps": 0.0078125,
    "kv_policy": "calculate",
    "masked_exit_cost": 0.0,
    "mode": "exit_only",
    "sites": [
      {
        "end": 8,
        "schema": "binary",
        "start": 4
      },
      {
        "end": 12,
        "schema": "binary",
        "start": 8
      },
      {
        "end": 16,
        "schema": "binary",
        "start": 12
      },
      {
        "end": 20,
        "schema": "binary",
        "start": 16
      },
      {
        "end": 24,
        "schema": "binary",
        "start": 20
      }
    ],
    "start_layer": 4,
    "temperature": 1.0,
    "ttt_only": false,
    "update_bias": 1.0
  },
  "training": {
    "accum_steps": 32,
    "data_seed": 42,
    "data_split": "train",
    "dataset_name": "books3",
    "dummy_dataset": false,
    "dummy_num_tokens": 1048576,
    "efficiency_coefficient": 0.5,
    "exit_ce_coefficient": 0.5,
    "gate_teacher_counterfactual": true,
    "global_batch_size": 32,
    "inner_learning_rate": 1.0,
    "inner_warmup_steps": 0,
    "mask_bos_targets": false,
    "mask_final_target": true,
    "model_seed": 42,
    "optimizer_inner": {
      "b1": null,
      "b2": null,
      "bf16_momentum": null,
      "clip_gradient": 1.0,
      "emb_wd": null,
      "end_lr": null,
      "init_lr": null,
      "lr": 1.0,
      "lr_decay_steps": null,
      "lr_warmup_steps": null,
      "optimizer_type": "sgd",
      "weight_decay": null
    },
    "optimizer_outer": {
      "b1": 0.9,
      "b2": 0.95,
      "bf16_momentum": false,
      "clip_gradient": 1.0,
      "emb_wd": true,
      "end_lr": 0.0001,
      "init_lr": 0.0,
      "lr": 0.001,
      "lr_decay_steps": 1024,
      "lr_warmup_steps": 256,
      "optimizer_type": "adamw",
      "weight_decay": 0.1
    },
    "pure_min_realized": false,
    "router_warmup_steps": 256,
    "seq_length": 8192,
    "spec_inner": [
      "language_model.**.suffix_blocks.feed_forward_prime.**"
    ],
    "spec_outer": [
      "language_model.**.router_storage.**"
    ],
    "target_cost": null,
    "target_cost_realized": true,
    "total_steps": 1024,
    "train_mode": "meta",
    "z_coefficient": 0.0
  }
}