| { |
| "benchmark": "Baihu-V1 vs google/gemma-4-E2B-it", |
| "hardware": "1x NVIDIA GeForce RTX 4090 D (24 GB), driver 580.119.02", |
| "held_out_transfer_hf_bf16": { |
| "label": "321 held-out conversations, HF bf16 checkpoints", |
| "gemma_4_e2b_it_base": { |
| "n": 321, |
| "rouge_l": 0.2359, |
| "plain_text_rate": 0.0156, |
| "reply_len_ratio": 3.8645, |
| "num_recall": 0.5795, |
| "num_recall_n": 240, |
| "reply_chars": 432.3, |
| "gen_seconds": 203.9 |
| }, |
| "baihu_v1": { |
| "n": 321, |
| "rouge_l": 0.4321, |
| "plain_text_rate": 1.0, |
| "reply_len_ratio": 0.9678, |
| "num_recall": 0.5282, |
| "num_recall_n": 240, |
| "reply_chars": 113.7, |
| "gen_seconds": 81.7 |
| } |
| }, |
| "held_out_transfer_gguf_q4_k_m": { |
| "label": "321 held-out conversations, published Q4_K_M GGUFs", |
| "gemma_4_e2b_it_base": { |
| "n": 321, |
| "rouge_l": 0.2345, |
| "plain_text_rate": 0.0125, |
| "reply_len_ratio": 3.8524, |
| "num_recall": 0.57, |
| "num_recall_n": 240, |
| "reply_chars": 432.2, |
| "gen_seconds": 192.1 |
| }, |
| "baihu_v1": { |
| "n": 321, |
| "rouge_l": 0.4194, |
| "plain_text_rate": 1.0, |
| "reply_len_ratio": 0.9592, |
| "num_recall": 0.5121, |
| "num_recall_n": 240, |
| "reply_chars": 113.9, |
| "gen_seconds": 62.1 |
| } |
| }, |
| "held_out_perplexity_q4_k_m": { |
| "corpus": "321 held-out conversations, 60 x 2048-token chunks (llama-perplexity)", |
| "gemma_4_e2b_it_q4_k_m": 13.7372, |
| "baihu_v1_q4_k_m": 5.2833, |
| "note": "llama-perplexity reports a standard error of +/-0.26 / +/-0.07" |
| }, |
| "mmlu_5shot": { |
| "n": 500, |
| "n_shot": 5, |
| "gemma_4_e2b_it": 0.534, |
| "baihu_v1": 0.526, |
| "delta": -0.008, |
| "by_subject": { |
| "astronomy": { |
| "base": 0.54, |
| "baihu_v1": 0.52 |
| }, |
| "computer_security": { |
| "base": 0.44, |
| "baihu_v1": 0.5 |
| }, |
| "elementary_mathematics": { |
| "base": 0.48, |
| "baihu_v1": 0.48 |
| }, |
| "formal_logic": { |
| "base": 0.58, |
| "baihu_v1": 0.6 |
| }, |
| "high_school_biology": { |
| "base": 0.54, |
| "baihu_v1": 0.54 |
| }, |
| "high_school_mathematics": { |
| "base": 0.44, |
| "baihu_v1": 0.4 |
| }, |
| "high_school_physics": { |
| "base": 0.38, |
| "baihu_v1": 0.48 |
| }, |
| "high_school_world_history": { |
| "base": 0.64, |
| "baihu_v1": 0.64 |
| }, |
| "logical_fallacies": { |
| "base": 0.64, |
| "baihu_v1": 0.54 |
| }, |
| "world_religions": { |
| "base": 0.66, |
| "baihu_v1": 0.56 |
| } |
| } |
| }, |
| "llama_bench": [ |
| { |
| "model": "gemma-4-E2B-it-Q4_K_M", |
| "n_prompt": 512, |
| "n_gen": 0, |
| "avg_ts": 15094.8, |
| "stddev_ts": 2731.5, |
| "backends": "CUDA", |
| "gpu": "NVIDIA GeForce RTX 4090 D", |
| "n_gpu_layers": 99 |
| }, |
| { |
| "model": "gemma-4-E2B-it-Q4_K_M", |
| "n_prompt": 0, |
| "n_gen": 128, |
| "avg_ts": 307.8, |
| "stddev_ts": 1.7, |
| "backends": "CUDA", |
| "gpu": "NVIDIA GeForce RTX 4090 D", |
| "n_gpu_layers": 99 |
| }, |
| { |
| "model": "Baihu-V1-Q4_K_M", |
| "n_prompt": 512, |
| "n_gen": 0, |
| "avg_ts": 15643.3, |
| "stddev_ts": 2944.9, |
| "backends": "CUDA", |
| "gpu": "NVIDIA GeForce RTX 4090 D", |
| "n_gpu_layers": 99 |
| }, |
| { |
| "model": "Baihu-V1-Q4_K_M", |
| "n_prompt": 0, |
| "n_gen": 128, |
| "avg_ts": 307.1, |
| "stddev_ts": 3.0, |
| "backends": "CUDA", |
| "gpu": "NVIDIA GeForce RTX 4090 D", |
| "n_gpu_layers": 99 |
| } |
| ], |
| "training": { |
| "base_model": "google/gemma-4-E2B-it", |
| "method": "LoRA r=16 alpha=32 dropout=0.05 on language_model q/k/v/o/gate/up/down", |
| "trainable_params": 24158208, |
| "total_params": 5128455712, |
| "train_conversations": 973, |
| "val_conversations": 321, |
| "epochs": 3, |
| "optimiser_steps": 183, |
| "batch": "4 x grad_accum 4", |
| "learning_rate": 0.0002, |
| "max_seq_len": 768, |
| "precision": "bf16", |
| "wall_time_minutes": 6.17, |
| "train_loss": 1.165, |
| "eval_loss": 1.536, |
| "seed": 20260930 |
| } |
| } |
