{
  "evaluation_dtype": "bfloat16",
  "base_validation": {
    "choice": {
      "n": 200,
      "nll": 1.8233506392094205,
      "brier": 0.9199781741252144,
      "hard_label_n": 200,
      "hard_label_accuracy": 0.135,
      "mae": null
    },
    "score": {
      "n": 200,
      "nll": 1.2868162845555702,
      "brier": 0.755653917889163,
      "hard_label_n": 200,
      "hard_label_accuracy": 0.355,
      "mae": 0.833484931158023
    },
    "noul": {
      "n": 200,
      "nll": 0.5812086463426769,
      "brier": 0.3886810789112994,
      "hard_label_n": 200,
      "hard_label_accuracy": 0.685,
      "mae": 0.36346207647013273
    }
  },
  "decide_validation": {
    "choice": {
      "n": 200,
      "nll": 0.8005431699930687,
      "brier": 0.40420523643337797,
      "hard_label_n": 200,
      "hard_label_accuracy": 0.74,
      "mae": null
    },
    "score": {
      "n": 200,
      "nll": 0.7330622738796232,
      "brier": 0.438596593023593,
      "hard_label_n": 200,
      "hard_label_accuracy": 0.635,
      "mae": 0.5985458431827447
    },
    "noul": {
      "n": 200,
      "nll": 0.2804184128570066,
      "brier": 0.18566711737282862,
      "hard_label_n": 200,
      "hard_label_accuracy": 0.865,
      "mae": 0.17999652788487253
    }
  },
  "decide_test_uncalibrated": {
    "choice": {
      "n": 300,
      "nll": 0.8291260863610138,
      "brier": 0.43582428224932407,
      "hard_label_n": 300,
      "hard_label_accuracy": 0.68,
      "mae": null
    },
    "score": {
      "n": 300,
      "nll": 0.7432575400592191,
      "brier": 0.4429180549709099,
      "hard_label_n": 300,
      "hard_label_accuracy": 0.63,
      "mae": 0.5857725157580695
    },
    "noul": {
      "n": 300,
      "nll": 0.1734100737580112,
      "brier": 0.11055894969304025,
      "hard_label_n": 300,
      "hard_label_accuracy": 0.93,
      "mae": 0.13128673404692287
    }
  },
  "decide_test_calibrated": {
    "choice": {
      "n": 300,
      "nll": 0.804358669731068,
      "brier": 0.42891276915451876,
      "hard_label_n": 300,
      "hard_label_accuracy": 0.68,
      "mae": null
    },
    "score": {
      "n": 300,
      "nll": 0.7420130093340176,
      "brier": 0.44234142225539114,
      "hard_label_n": 300,
      "hard_label_accuracy": 0.63,
      "mae": 0.5816908331595515
    },
    "noul": {
      "n": 300,
      "nll": 0.14012702863107165,
      "brier": 0.09082276054150111,
      "hard_label_n": 300,
      "hard_label_accuracy": 0.93,
      "mae": 0.09262523968667083
    }
  }
}