build_dpo_dataset

  1"""Build marola's DPO preference dataset (MIP-0025 §4.3, Layer 3) from `core/llm/Reviewer.scala`'s
  2own historical reject/revise decisions.
  3
  4Real historical decisions are already captured, once, as the DSPy-compiled `review_prompt.json`
  5demos (core/src/main/resources/review_prompt.json) — bootstrap examples where a real Ollama
  6reviewer graded a real draft summary. Wherever a demo's verdict wasn't "approve", the reviewer's
  7own `final_summary` is a real correction of a real flawed draft: that is exactly a (chosen,
  8rejected) preference pair, with no invented text on either side. An "approve" demo carries no
  9preference signal (the draft was fine as-is) and yields no pair — one pair per reject/revise
 10decision, never more, never fabricated when there isn't one.
 11
 12Run:  python build_dpo_dataset.py             (or `just finetune-dpo-dataset`)
 13Self-test:  python build_dpo_dataset.py --self-test   (or `just quality-other`)
 14
 15review_prompt.json comes from the app's resources tarball (MIP-0070 §5.4), unpacked into
 16`.tmp/resources` by `just resources-fetch`; `--resources DIR` overrides it, as in build_dataset.py.
 17"""
 18
 19from __future__ import annotations
 20
 21import argparse
 22import json
 23import sys
 24from pathlib import Path
 25
 26sys.path.insert(0, str(Path(__file__).resolve().parent))
 27import build_dataset as bd  # reuse INPUT_FIELDS/render_inputs — same conditions shape
 28
 29RESOURCES = bd.RESOURCES
 30OUT = Path(__file__).resolve().parent / "data"
 31
 32DPO_INPUT_FIELDS = bd.INPUT_FIELDS
 33
 34
 35def review_records(path: Path) -> list[dict]:
 36    """Real Reviewer decisions: one dict per review_prompt.json demo, in ReviewResult's own shape
 37    (Reviewer.scala's `ReviewResult(finalSummary, score, verdict)`) plus the draft it reviewed."""
 38    doc = json.loads(path.read_text(encoding="utf-8"))
 39    records = []
 40    for demo in doc.get("demos", []):
 41        if "review_json" not in demo or "summary" not in demo:
 42            continue
 43        review = json.loads(demo["review_json"])
 44        records.append(
 45            {
 46                "conditions": {k: demo[k] for k in DPO_INPUT_FIELDS if k in demo},
 47                "draft_summary": demo["summary"],
 48                "score": review.get("score"),
 49                "verdict": review.get("verdict", "approve"),
 50                "final_summary": review.get("final_summary", demo["summary"]),
 51            }
 52        )
 53    return records
 54
 55
 56def preference_pairs(records: list[dict]) -> list[dict]:
 57    """One (chosen, rejected) pair per real reject/revise record; none for "approve"."""
 58    pairs = []
 59    for r in records:
 60        if r["verdict"] == "approve":
 61            continue
 62        if r["final_summary"] == r["draft_summary"]:
 63            continue  # flagged but the text didn't actually change — no real preference signal
 64        prompt = bd.render_inputs(r["conditions"], DPO_INPUT_FIELDS)
 65        pairs.append(
 66            {"prompt": prompt, "chosen": r["final_summary"], "rejected": r["draft_summary"]}
 67        )
 68    return pairs
 69
 70
 71def main(resources: Path = RESOURCES) -> None:
 72    records = review_records(resources / "review_prompt.json")
 73    pairs = preference_pairs(records)
 74    OUT.mkdir(parents=True, exist_ok=True)
 75    with (OUT / "dpo_pairs.jsonl").open("w", encoding="utf-8") as f:
 76        for p in pairs:
 77            f.write(json.dumps(p, ensure_ascii=False) + "\n")
 78    print(
 79        f"wrote {len(pairs)} DPO preference pairs (from {len(records)} reviewer decisions) to {OUT}"
 80    )
 81
 82
 83# --- self-test: a small fixture of Reviewer decisions, not the real ones ---------------------
 84_FIXTURE_RECORDS = [
 85    {
 86        "conditions": {"beach_name": "Fixture Beach A", "score": "90"},
 87        "draft_summary": "Great conditions, go for it.",
 88        "score": 90,
 89        "verdict": "approve",
 90        "final_summary": "Great conditions, go for it.",
 91    },
 92    {
 93        "conditions": {"beach_name": "Fixture Beach B", "score": "40"},
 94        "draft_summary": "Nice and calm, a pleasant swim.",
 95        "score": 35,
 96        "verdict": "revise",
 97        "final_summary": "Calm but high jellyfish risk — worth a second look before swimming.",
 98    },
 99    {
100        "conditions": {"beach_name": "Fixture Beach C", "score": "5"},
101        "draft_summary": "Great day for a swim.",
102        "score": 5,
103        "verdict": "reject",
104        "final_summary": "Unsafe: strong riptide and rough seas — do not swim.",
105    },
106]
107
108
109def _self_test(resources: Path = RESOURCES) -> None:
110    # Fixture with reject/revise events: exactly one pair per non-approve event, nothing invented.
111    pairs = preference_pairs(_FIXTURE_RECORDS)
112    assert len(pairs) == 2, f"expected 2 pairs (one revise + one reject), got {len(pairs)}"
113    chosen_texts = {p["chosen"] for p in pairs}
114    assert _FIXTURE_RECORDS[1]["final_summary"] in chosen_texts
115    assert _FIXTURE_RECORDS[2]["final_summary"] in chosen_texts
116    for p in pairs:
117        assert p["chosen"] != p["rejected"], "chosen and rejected must differ"
118
119    # A fixture with only "approve" events: no reject/revise → zero pairs, none invented.
120    approve_only = [r for r in _FIXTURE_RECORDS if r["verdict"] == "approve"]
121    assert preference_pairs(approve_only) == [], "an all-approve fixture must yield zero pairs"
122
123    # Real source check: review_prompt.json's own demos really do have reject/revise decisions,
124    # and every generated pair's text matches one of them verbatim — no rewording, no invention.
125    real_records = review_records(resources / "review_prompt.json")
126    real_pairs = preference_pairs(real_records)
127    assert real_pairs, "no real reject/revise decisions found in review_prompt.json's demos"
128    real_texts = {
129        (r["final_summary"], r["draft_summary"])
130        for r in real_records
131        if r["verdict"] != "approve" and r["final_summary"] != r["draft_summary"]
132    }
133    generated_texts = {(p["chosen"], p["rejected"]) for p in real_pairs}
134    assert generated_texts <= real_texts, "a generated pair's text doesn't match a real decision"
135
136    print(
137        f"self-test OK: {len(pairs)} pairs from the fixture, {len(real_pairs)} from "
138        f"review_prompt.json's real reviewer decisions, all verbatim"
139    )
140
141
142def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
143    ap = argparse.ArgumentParser(
144        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
145    )
146    ap.add_argument(
147        "--resources",
148        type=Path,
149        default=RESOURCES,
150        help="dir with review_prompt.json (default: .tmp/resources, from `just resources-fetch`)",
151    )
152    ap.add_argument("--self-test", action="store_true")
153    return ap.parse_args(argv)
154
155
156if __name__ == "__main__":
157    args = parse_args()
158    if args.self_test:
159        _self_test(args.resources)
160    else:
161        main(args.resources)