{
  "benchmark": "Baihu-V1 vs google/gemma-4-E2B-it",
  "hardware": "1x NVIDIA GeForce RTX 4090 D (24 GB), driver 580.119.02",
  "held_out_transfer_hf_bf16": {
    "label": "321 held-out conversations, HF bf16 checkpoints",
    "gemma_4_e2b_it_base": {
      "n": 321,
      "rouge_l": 0.2359,
      "plain_text_rate": 0.0156,
      "reply_len_ratio": 3.8645,
      "num_recall": 0.5795,
      "num_recall_n": 240,
      "reply_chars": 432.3,
      "gen_seconds": 203.9
    },
    "baihu_v1": {
      "n": 321,
      "rouge_l": 0.4321,
      "plain_text_rate": 1.0,
      "reply_len_ratio": 0.9678,
      "num_recall": 0.5282,
      "num_recall_n": 240,
      "reply_chars": 113.7,
      "gen_seconds": 81.7
    }
  },
  "held_out_transfer_gguf_q4_k_m": {
    "label": "321 held-out conversations, published Q4_K_M GGUFs",
    "gemma_4_e2b_it_base": {
      "n": 321,
      "rouge_l": 0.2345,
      "plain_text_rate": 0.0125,
      "reply_len_ratio": 3.8524,
      "num_recall": 0.57,
      "num_recall_n": 240,
      "reply_chars": 432.2,
      "gen_seconds": 192.1
    },
    "baihu_v1": {
      "n": 321,
      "rouge_l": 0.4194,
      "plain_text_rate": 1.0,
      "reply_len_ratio": 0.9592,
      "num_recall": 0.5121,
      "num_recall_n": 240,
      "reply_chars": 113.9,
      "gen_seconds": 62.1
    }
  },
  "held_out_perplexity_q4_k_m": {
    "corpus": "321 held-out conversations, 60 x 2048-token chunks (llama-perplexity)",
    "gemma_4_e2b_it_q4_k_m": 13.7372,
    "baihu_v1_q4_k_m": 5.2833,
    "note": "llama-perplexity reports a standard error of +/-0.26 / +/-0.07"
  },
  "mmlu_5shot": {
    "n": 500,
    "n_shot": 5,
    "gemma_4_e2b_it": 0.534,
    "baihu_v1": 0.526,
    "delta": -0.008,
    "by_subject": {
      "astronomy": {
        "base": 0.54,
        "baihu_v1": 0.52
      },
      "computer_security": {
        "base": 0.44,
        "baihu_v1": 0.5
      },
      "elementary_mathematics": {
        "base": 0.48,
        "baihu_v1": 0.48
      },
      "formal_logic": {
        "base": 0.58,
        "baihu_v1": 0.6
      },
      "high_school_biology": {
        "base": 0.54,
        "baihu_v1": 0.54
      },
      "high_school_mathematics": {
        "base": 0.44,
        "baihu_v1": 0.4
      },
      "high_school_physics": {
        "base": 0.38,
        "baihu_v1": 0.48
      },
      "high_school_world_history": {
        "base": 0.64,
        "baihu_v1": 0.64
      },
      "logical_fallacies": {
        "base": 0.64,
        "baihu_v1": 0.54
      },
      "world_religions": {
        "base": 0.66,
        "baihu_v1": 0.56
      }
    }
  },
  "llama_bench": [
    {
      "model": "gemma-4-E2B-it-Q4_K_M",
      "n_prompt": 512,
      "n_gen": 0,
      "avg_ts": 15094.8,
      "stddev_ts": 2731.5,
      "backends": "CUDA",
      "gpu": "NVIDIA GeForce RTX 4090 D",
      "n_gpu_layers": 99
    },
    {
      "model": "gemma-4-E2B-it-Q4_K_M",
      "n_prompt": 0,
      "n_gen": 128,
      "avg_ts": 307.8,
      "stddev_ts": 1.7,
      "backends": "CUDA",
      "gpu": "NVIDIA GeForce RTX 4090 D",
      "n_gpu_layers": 99
    },
    {
      "model": "Baihu-V1-Q4_K_M",
      "n_prompt": 512,
      "n_gen": 0,
      "avg_ts": 15643.3,
      "stddev_ts": 2944.9,
      "backends": "CUDA",
      "gpu": "NVIDIA GeForce RTX 4090 D",
      "n_gpu_layers": 99
    },
    {
      "model": "Baihu-V1-Q4_K_M",
      "n_prompt": 0,
      "n_gen": 128,
      "avg_ts": 307.1,
      "stddev_ts": 3.0,
      "backends": "CUDA",
      "gpu": "NVIDIA GeForce RTX 4090 D",
      "n_gpu_layers": 99
    }
  ],
  "training": {
    "base_model": "google/gemma-4-E2B-it",
    "method": "LoRA r=16 alpha=32 dropout=0.05 on language_model q/k/v/o/gate/up/down",
    "trainable_params": 24158208,
    "total_params": 5128455712,
    "train_conversations": 973,
    "val_conversations": 321,
    "epochs": 3,
    "optimiser_steps": 183,
    "batch": "4 x grad_accum 4",
    "learning_rate": 0.0002,
    "max_seq_len": 768,
    "precision": "bf16",
    "wall_time_minutes": 6.17,
    "train_loss": 1.165,
    "eval_loss": 1.536,
    "seed": 20260930
  }
}