build_dpo_dataset

  1"""Build marola's DPO preference dataset (MIP-0025 §4.3, Layer 3) from `core/llm/Reviewer.scala`'s
  2own historical reject/revise decisions.
  3
  4Real historical decisions are already captured, once, as the DSPy-compiled `review_prompt.json`
  5demos (core/src/main/resources/review_prompt.json) — bootstrap examples where a real Ollama
  6reviewer graded a real draft summary. Wherever a demo's verdict wasn't "approve", the reviewer's
  7own `final_summary` is a real correction of a real flawed draft: that is exactly a (chosen,
  8rejected) preference pair, with no invented text on either side. An "approve" demo carries no
  9preference signal (the draft was fine as-is) and yields no pair — one pair per reject/revise
 10decision, never more, never fabricated when there isn't one.
 11
 12Run:  python build_dpo_dataset.py             (or `just finetune-dpo-dataset`)
 13Self-test:  python build_dpo_dataset.py --self-test   (or `just quality-other`)
 14
 15`--resources DIR` overrides where review_prompt.json is read from — the same app -> ml contract
 16as build_dataset.py's own flag (MIP-0070 §5.4): once marola-ml is a separate repo, it points at
 17the unpacked resources tarball ci.yml publishes, not `../core`.
 18"""
 19
 20from __future__ import annotations
 21
 22import argparse
 23import json
 24import sys
 25from pathlib import Path
 26
 27sys.path.insert(0, str(Path(__file__).resolve().parent))
 28import build_dataset as bd  # reuse INPUT_FIELDS/render_inputs — same conditions shape
 29
 30RESOURCES = bd.RESOURCES
 31OUT = Path(__file__).resolve().parent / "data"
 32
 33DPO_INPUT_FIELDS = bd.INPUT_FIELDS
 34
 35
 36def review_records(path: Path) -> list[dict]:
 37    """Real Reviewer decisions: one dict per review_prompt.json demo, in ReviewResult's own shape
 38    (Reviewer.scala's `ReviewResult(finalSummary, score, verdict)`) plus the draft it reviewed."""
 39    doc = json.loads(path.read_text(encoding="utf-8"))
 40    records = []
 41    for demo in doc.get("demos", []):
 42        if "review_json" not in demo or "summary" not in demo:
 43            continue
 44        review = json.loads(demo["review_json"])
 45        records.append(
 46            {
 47                "conditions": {k: demo[k] for k in DPO_INPUT_FIELDS if k in demo},
 48                "draft_summary": demo["summary"],
 49                "score": review.get("score"),
 50                "verdict": review.get("verdict", "approve"),
 51                "final_summary": review.get("final_summary", demo["summary"]),
 52            }
 53        )
 54    return records
 55
 56
 57def preference_pairs(records: list[dict]) -> list[dict]:
 58    """One (chosen, rejected) pair per real reject/revise record; none for "approve"."""
 59    pairs = []
 60    for r in records:
 61        if r["verdict"] == "approve":
 62            continue
 63        if r["final_summary"] == r["draft_summary"]:
 64            continue  # flagged but the text didn't actually change — no real preference signal
 65        prompt = bd.render_inputs(r["conditions"], DPO_INPUT_FIELDS)
 66        pairs.append(
 67            {"prompt": prompt, "chosen": r["final_summary"], "rejected": r["draft_summary"]}
 68        )
 69    return pairs
 70
 71
 72def main(resources: Path = RESOURCES) -> None:
 73    records = review_records(resources / "review_prompt.json")
 74    pairs = preference_pairs(records)
 75    OUT.mkdir(parents=True, exist_ok=True)
 76    with (OUT / "dpo_pairs.jsonl").open("w", encoding="utf-8") as f:
 77        for p in pairs:
 78            f.write(json.dumps(p, ensure_ascii=False) + "\n")
 79    print(
 80        f"wrote {len(pairs)} DPO preference pairs (from {len(records)} reviewer decisions) to {OUT}"
 81    )
 82
 83
 84# --- self-test: a small fixture of Reviewer decisions, not the real ones ---------------------
 85_FIXTURE_RECORDS = [
 86    {
 87        "conditions": {"beach_name": "Fixture Beach A", "score": "90"},
 88        "draft_summary": "Great conditions, go for it.",
 89        "score": 90,
 90        "verdict": "approve",
 91        "final_summary": "Great conditions, go for it.",
 92    },
 93    {
 94        "conditions": {"beach_name": "Fixture Beach B", "score": "40"},
 95        "draft_summary": "Nice and calm, a pleasant swim.",
 96        "score": 35,
 97        "verdict": "revise",
 98        "final_summary": "Calm but high jellyfish risk — worth a second look before swimming.",
 99    },
100    {
101        "conditions": {"beach_name": "Fixture Beach C", "score": "5"},
102        "draft_summary": "Great day for a swim.",
103        "score": 5,
104        "verdict": "reject",
105        "final_summary": "Unsafe: strong riptide and rough seas — do not swim.",
106    },
107]
108
109
110def _self_test(resources: Path = RESOURCES) -> None:
111    # Fixture with reject/revise events: exactly one pair per non-approve event, nothing invented.
112    pairs = preference_pairs(_FIXTURE_RECORDS)
113    assert len(pairs) == 2, f"expected 2 pairs (one revise + one reject), got {len(pairs)}"
114    chosen_texts = {p["chosen"] for p in pairs}
115    assert _FIXTURE_RECORDS[1]["final_summary"] in chosen_texts
116    assert _FIXTURE_RECORDS[2]["final_summary"] in chosen_texts
117    for p in pairs:
118        assert p["chosen"] != p["rejected"], "chosen and rejected must differ"
119
120    # A fixture with only "approve" events: no reject/revise → zero pairs, none invented.
121    approve_only = [r for r in _FIXTURE_RECORDS if r["verdict"] == "approve"]
122    assert preference_pairs(approve_only) == [], "an all-approve fixture must yield zero pairs"
123
124    # Real source check: review_prompt.json's own demos really do have reject/revise decisions,
125    # and every generated pair's text matches one of them verbatim — no rewording, no invention.
126    real_records = review_records(resources / "review_prompt.json")
127    real_pairs = preference_pairs(real_records)
128    assert real_pairs, "no real reject/revise decisions found in review_prompt.json's demos"
129    real_texts = {
130        (r["final_summary"], r["draft_summary"])
131        for r in real_records
132        if r["verdict"] != "approve" and r["final_summary"] != r["draft_summary"]
133    }
134    generated_texts = {(p["chosen"], p["rejected"]) for p in real_pairs}
135    assert generated_texts <= real_texts, "a generated pair's text doesn't match a real decision"
136
137    print(
138        f"self-test OK: {len(pairs)} pairs from the fixture, {len(real_pairs)} from "
139        f"review_prompt.json's real reviewer decisions, all verbatim"
140    )
141
142
143def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
144    ap = argparse.ArgumentParser(
145        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
146    )
147    ap.add_argument(
148        "--resources",
149        type=Path,
150        default=RESOURCES,
151        help="dir with review_prompt.json (default: core/src/main/resources)",
152    )
153    ap.add_argument("--self-test", action="store_true")
154    return ap.parse_args(argv)
155
156
157if __name__ == "__main__":
158    args = parse_args()
159    if args.self_test:
160        _self_test(args.resources)
161    else:
162        main(args.resources)