Rebuild the Kalshi transfer set; correct a contamination claim that was wrong. The old set had no builder and its prompts depended on an API field present in no archive we kept. Elections is worse than a constant on BOTH sides of the freeze, so the old 'contamination-free subset' framing attributed to contamination what was category composition.
c58c7b9 verified | """Rebuild the rebuilt set's prompts to the shipped construction, and prove it. | |
| uv run python scripts/kalshi_rebuild_prompts.py | |
| The shipped eval set's prompt is | |
| title with a trailing "?" removed + " -- resolves YES if: " + yes_sub_title | |
| (with an em dash, not two hyphens). `yes_sub_title` is in no archived source we | |
| hold, so it was re-fetched from the live API by `kalshi_subtitles.py` | |
| (docs/10 2026-08-30g). | |
| ## The check that makes this trustworthy | |
| 157 question ids appear in both the shipped 778 and the rebuilt 1,259. If the | |
| construction rule is right, it reproduces all 157 shipped texts **exactly**. | |
| This script refuses to write unless it does. A rule that reproduces most of them | |
| is a rule that is wrong somewhere, and "most" is how the prompt defect survived | |
| the first run. | |
| ## Rows whose subtitle could not be fetched | |
| DROPPED AND COUNTED, never defaulted to the bare title. Defaulting would | |
| silently reintroduce the exact defect the reproduction gate caught, on a subset, | |
| where it would be invisible. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| SEP = " — resolves YES if: " # em dash | |
| def build(title: str, sub: str) -> str: | |
| """The construction is CONDITIONAL, and the condition had to be recovered. | |
| When `yes_sub_title` already appears in the title it adds nothing, and the | |
| shipped set keeps the title unchanged, question mark and all. When it does | |
| not, the title's trailing "?" is dropped and the subtitle is appended. | |
| That is not a guess. On the 157 questions shared with the shipped set the | |
| split is perfect: 92 of 92 unsuffixed rows have the subtitle inside the | |
| title, and 0 of 65 suffixed rows do. | |
| """ | |
| if sub.lower() in title.lower(): | |
| return title | |
| return title.rstrip().rstrip("?").rstrip() + SEP + sub | |
| def main() -> int: | |
| subs: dict[str, dict] = {} | |
| for line in open("data/kalshi/subtitles.jsonl", encoding="utf-8"): | |
| if line.strip(): | |
| r = json.loads(line) | |
| subs[r["ticker"]] = r | |
| ev = json.load(open("data/kalshi_eval_rebuilt.json", encoding="utf-8")) | |
| old = {q["question_id"]: q["text"] | |
| for q in json.load(open("data/kalshi_score_778.json", encoding="utf-8"))["questions"]} | |
| kept, dropped = [], {"no_subtitle": 0, "not_found": 0, "no_title": 0} | |
| for q in ev["questions"]: | |
| t = q["question_id"].split(":", 1)[1] | |
| rec = subs.get(t) | |
| if rec is None or rec.get("status") == "not_found": | |
| dropped["not_found"] += 1 | |
| continue | |
| sub, title = rec.get("yes_sub_title"), rec.get("title") | |
| if not title: | |
| dropped["no_title"] += 1 | |
| continue | |
| if sub is None: | |
| dropped["no_subtitle"] += 1 | |
| continue | |
| q = dict(q) | |
| q["text"] = build(title, sub) | |
| kept.append(q) | |
| shared = [q for q in kept if q["question_id"] in old] | |
| exact = sum(1 for q in shared if q["text"] == old[q["question_id"]]) | |
| print(f"rebuilt {len(kept):,} rows; dropped {sum(dropped.values())} {dropped}") | |
| print(f"shared with the shipped set: {len(shared)}") | |
| print(f"prompts reproduced EXACTLY: {exact} / {len(shared)}") | |
| if shared and exact != len(shared): | |
| print("\nREFUSING to write. The construction rule does not reproduce every") | |
| print("shipped prompt, so it is wrong somewhere. Examples:") | |
| for q in shared: | |
| if q["text"] != old[q["question_id"]]: | |
| print(f" id : {q['question_id']}") | |
| print(f" shipped : {old[q['question_id']]}") | |
| print(f" rebuilt : {q['text']}") | |
| break | |
| return 1 | |
| ev["questions"] = sorted(kept, key=lambda q: q["question_id"]) | |
| ev["n"] = len(kept) | |
| ev["prompt_construction"] = ( | |
| "if yes_sub_title appears in title: title unchanged; " | |
| "else title (trailing '?' removed) + ' — resolves YES if: ' + yes_sub_title. " | |
| "yes_sub_title re-fetched from the live API; it is in no archived source.") | |
| ev["dropped_prompt_rebuild"] = dropped | |
| import hashlib | |
| ev["content_hash"] = hashlib.sha256( | |
| json.dumps([q["question_id"] for q in ev["questions"]], sort_keys=True).encode() | |
| ).hexdigest() | |
| with open("data/kalshi_eval_rebuilt.json", "w", encoding="utf-8") as h: | |
| json.dump(ev, h, indent=1, sort_keys=True) | |
| print(f"\nwrote data/kalshi_eval_rebuilt.json n={ev['n']:,} hash {ev['content_hash'][:16]}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |