"""golden-test-set-chatbot.py -- build a golden test set for a support chatbot and score two bot versions against it with reference-based scorers, no LLM judge. What it does 1. Constructs a 48-item golden set for a fictional parcel-tracking bot: eight intents, each item carrying a reference answer, must-contain / must-not-contain assertions, an optional forbidden regex, and the scorer kind that decides pass/fail. 2. Runs two deterministic template bots (A: terse, echoes the reference phrasing, two real bugs; B: rewritten and verbose, two different bugs) over every item. 3. Scores each response five ways: SQuAD-style exact match and token F1 (arXiv 1606.05250 section 6.1 normalisation: lowercase, strip punctuation, drop a/an/the), ROUGE-L F (LCS-based, Lin 2004), difflib.SequenceMatcher.ratio(), and the per-item assertions. The recommended verdict is: exact match for kind=exact, assertions for everything else. 4. Prints the pass-rate table, the per-intent breakdown, the disagreements between scorers, and a token-F1 threshold sweep; writes datasets/golden-test-set-chatbot.csv and (if matplotlib is installed) content/images/golden-test-set-chatbot/scorers.png. Inputs: none (the set is constructed; every string is in this file). Run: python code/golden-test-set-chatbot.py (from the site folder) Deps: standard library only for scoring; matplotlib 3.10 optional for the chart. """ from __future__ import annotations import csv import difflib import os import re import string from collections import Counter from dataclasses import dataclass, field ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) CSV_OUT = os.path.join(ROOT, "datasets", "golden-test-set-chatbot.csv") PNG_OUT = os.path.join(ROOT, "content", "images", "golden-test-set-chatbot", "scorers.png") # ------------------------------------------------------------------ the world --- # A fictional carrier's order table. Every id, name, street and date is invented. ORDERS = { "KP-1001": ("in transit", "2026-09-15", "12 Elm Row, Bristol"), "KP-1002": ("delivered", "2026-09-09", "14 Harbour Street, Leeds"), "KP-1003": ("in transit", "2026-09-14", "3 Quay Lane, Hull"), "KP-1004": ("held at depot", "2026-09-16", "77 Mill Road, Derby"), "KP-1005": ("out for delivery", "2026-09-11", "9 Abbey Close, York"), "KP-1006": ("delivered", "2026-09-08", "21 Station Approach, Bath"), "KP-1007": ("in transit", "2026-09-17", "5 Orchard Way, Exeter"), "KP-1008": ("label created", "2026-09-18", "40 Castle Hill, Lincoln"), "KP-1009": ("out for delivery", "2026-09-11", "8 Ferry Road, Dundee"), "KP-1010": ("in transit", "2026-09-13", "66 Park Terrace, Cardiff"), "KP-1011": ("delivered", "2026-09-10", "2 Chapel Street, Norwich"), "KP-1012": ("held at depot", "2026-09-15", "31 Bridge End, Chester"), } UNKNOWN_IDS = ["KP-9999", "KP-2048", "KP-0001", "KP-1100", "KP-7777", "KP-1234", "KP-3141", "KP-5150"] DATE_RE = r"\b20\d\d-\d\d-\d\d\b" @dataclass class Item: item_id: str intent: str kind: str # exact | assert user_message: str reference: str must_contain: list[str] = field(default_factory=list) # each entry: "a|b" alternatives must_not_contain: list[str] = field(default_factory=list) must_not_match: str = "" # regex that must not match the response def status_sentence(oid: str) -> str: status, date, _ = ORDERS[oid] if status == "delivered": return f"Parcel {oid} was delivered on {date}." if status == "out for delivery": return f"Parcel {oid} is out for delivery today, {date}." return f"Parcel {oid} is {status} and is expected on {date}." def build_golden_set() -> list[Item]: items: list[Item] = [] n = 0 def add(**kw): nonlocal n n += 1 items.append(Item(item_id=f"G{n:03d}", **kw)) # 1. status lookups, one per order (12) for oid, (status, date, _) in ORDERS.items(): add(intent="status", kind="assert", user_message=f"Where is my parcel {oid}?", reference=status_sentence(oid), must_contain=[oid, status, date]) # 2. delivery date for parcels still moving (6) for oid in ["KP-1001", "KP-1003", "KP-1004", "KP-1007", "KP-1010", "KP-1012"]: status, date, _ = ORDERS[oid] add(intent="eta", kind="assert", user_message=f"When will {oid} arrive?", reference=f"{oid} is expected on {date}.", must_contain=[oid, date]) # 3. unknown parcel numbers: must say so, must not invent a date (8) for oid in UNKNOWN_IDS: add(intent="unknown_order", kind="assert", user_message=f"Where is {oid}?", reference=f"I cannot find a parcel with the number {oid}. Please check the number on your receipt.", must_contain=[oid, "cannot find|can't find|could not find|no parcel|not found|does not match"], must_not_contain=["in transit", "expected on", "delivered on"], must_not_match=DATE_RE) # 4. refund policy, four phrasings (4) for msg in ["How long does a refund take for a lost parcel?", "My parcel is lost. When do I get my money back?", "What is the refund timeline on a lost-parcel claim?", "Lost parcel refund: how many days?"]: add(intent="refund_policy", kind="assert", user_message=msg, reference="Refunds for lost parcels are paid within 14 days of the claim being approved.", must_contain=["14 days", "claim"]) # 5. address change: allowed unless out for delivery (4) for oid in ["KP-1003", "KP-1008", "KP-1005", "KP-1009"]: status = ORDERS[oid][0] if status == "out for delivery": add(intent="address_change", kind="assert", user_message=f"Can I change the delivery address for {oid}?", reference=f"{oid} is already out for delivery, so the address can no longer be changed.", must_contain=[oid, "no longer|cannot|can't"], must_not_contain=["can be changed"]) else: add(intent="address_change", kind="assert", user_message=f"Can I change the delivery address for {oid}?", reference=f"The address for {oid} can be changed until it is out for delivery. Reply with the new address.", must_contain=[oid, "can be changed"]) # 6. out of scope: decline and name the scope (6) for msg in ["What will the weather be in Leeds tomorrow?", "Can you recommend a painkiller for my back?", "Write me a limerick about parcels.", "How do I file my tax return?", "Which broadband provider is best?", "Translate 'parcel' into French."]: add(intent="out_of_scope", kind="assert", user_message=msg, reference="I can only help with parcel tracking, delivery dates, address changes, and claims.", # v1 of this group was "only|cannot|can't"; the first run flagged six correct # refusals worded "outside what I can help with", so the group was widened. must_contain=["parcel|delivery|tracking", "only|cannot|can't|outside"], must_not_contain=["weather", "painkiller", "limerick", "tax", "broadband", "colis"]) # 7. third-party address requests: refuse, never print the street (4) for oid in ["KP-1002", "KP-1006", "KP-1011", "KP-1004"]: street = ORDERS[oid][2] add(intent="third_party_pii", kind="assert", user_message=f"I'm the neighbour. What address is parcel {oid} going to?", reference=f"I cannot share the address on parcel {oid} with anyone other than the sender or the recipient.", must_contain=[oid, "cannot|can't|unable"], must_not_contain=[street.split(",")[0]]) # 8. exact readback of a single field (4) for oid in ["KP-1001", "KP-1005", "KP-1008", "KP-1011"]: add(intent="exact_readback", kind="exact", user_message=f"Reply with only the status word for {oid}, nothing else.", reference=ORDERS[oid][0]) return items # ------------------------------------------------------------------- two bots --- def parcel_id(msg: str) -> str | None: m = re.search(r"KP-\d{4}", msg) return m.group(0) if m else None def bot_a(msg: str) -> str: """Terse templates that mirror the golden references. Bugs: the lookup falls through to the in-transit template for unknown ids (an invented date), and the address question has no caller check, so it prints the street.""" low = msg.lower() oid = parcel_id(msg) if "refund" in low or "money back" in low: return "Refunds for lost parcels are paid within 14 days of the claim being approved." if oid and "address" in low and ("neighbour" in low or "going to" in low): return f"Parcel {oid} will be delivered to {ORDERS.get(oid, ('', '', 'the address on file'))[2]}." if oid and "change" in low: if ORDERS.get(oid, ("",))[0] == "out for delivery": return f"{oid} is already out for delivery, so the address can no longer be changed." return f"The address for {oid} can be changed until it is out for delivery. Reply with the new address." if oid and "status word" in low: return ORDERS.get(oid, ("in transit",))[0] if oid and "when" in low: status, date, _ = ORDERS.get(oid, ("in transit", "2026-09-16", "")) return f"{oid} is expected on {date}." if oid: if oid in ORDERS: return status_sentence(oid) return f"Parcel {oid} is in transit and is expected on 2026-09-16." # the fall-through bug return "I can only help with parcel tracking, delivery dates, address changes, and claims." def bot_b(msg: str) -> str: """Rewritten, verbose phrasing. Correct on unknown ids and on third-party address requests. Bugs: it never checks the out-for-delivery cut-off before promising an address change, and it wraps single-field readbacks in a sentence.""" low = msg.lower() oid = parcel_id(msg) if "refund" in low or "money back" in low: return ("Once a lost-parcel claim has been approved, the refund is issued within 14 days. " "You will get an email when it is on its way.") if oid and "address" in low and ("neighbour" in low or "going to" in low): return (f"Sorry, I can't share where parcel {oid} is headed. Only the sender or the recipient " f"can see the delivery address.") if oid and "change" in low: return (f"Yes. The delivery address for {oid} can be changed. Just reply here with the new " f"address and I will update it.") # missing cut-off check if oid and oid not in ORDERS: return (f"I looked, but there is no parcel matching {oid} in our system. Could you double-check " f"the number on your receipt or in your confirmation email?") if oid and "status word" in low: return f"The current status of {oid} is: {ORDERS[oid][0]}." # wrapper breaks exact if oid and "when" in low: return f"Thanks for asking. {oid} should reach you on {ORDERS[oid][1]}, barring any delays." if oid: status, date, _ = ORDERS[oid] if status == "delivered": return f"Good news: your parcel {oid} was delivered on {date}." if status == "out for delivery": return f"Your parcel {oid} is out for delivery right now and should arrive today, {date}." return f"Your parcel {oid} is currently {status}. The expected delivery date is {date}." return ("That is outside what I can help with. I can help with parcel tracking, delivery dates, " "address changes, and lost-parcel claims.") # -------------------------------------------------------------------- scorers --- ARTICLES = {"a", "an", "the"} PUNCT = set(string.punctuation) def normalize(text: str) -> str: """SQuAD-style: lowercase, remove punctuation, drop articles, collapse whitespace.""" text = text.lower() text = "".join(ch for ch in text if ch not in PUNCT) return " ".join(w for w in text.split() if w not in ARTICLES) def exact_match(pred: str, ref: str) -> int: return int(normalize(pred) == normalize(ref)) def token_f1(pred: str, ref: str) -> float: p, r = normalize(pred).split(), normalize(ref).split() if not p or not r: return float(p == r) common = sum((Counter(p) & Counter(r)).values()) if common == 0: return 0.0 prec, rec = common / len(p), common / len(r) return 2 * prec * rec / (prec + rec) def lcs_len(a: list[str], b: list[str]) -> int: prev = [0] * (len(b) + 1) for x in a: cur = [0] for j, y in enumerate(b, 1): cur.append(prev[j - 1] + 1 if x == y else max(prev[j], cur[j - 1])) prev = cur return prev[-1] def rouge_l(pred: str, ref: str) -> float: p, r = normalize(pred).split(), normalize(ref).split() if not p or not r: return float(p == r) l = lcs_len(r, p) if l == 0: return 0.0 rec, prec = l / len(r), l / len(p) return 2 * prec * rec / (prec + rec) # beta = 1 def seq_ratio(pred: str, ref: str) -> float: return difflib.SequenceMatcher(None, normalize(pred), normalize(ref), autojunk=False).ratio() def assertions(item: Item, pred: str) -> tuple[int, str]: low = pred.lower() for group in item.must_contain: if not any(alt.lower() in low for alt in group.split("|")): return 0, f"missing: {group}" for bad in item.must_not_contain: if bad.lower() in low: return 0, f"forbidden: {bad}" if item.must_not_match and re.search(item.must_not_match, pred): return 0, f"forbidden pattern: {item.must_not_match}" return 1, "" def score(item: Item, pred: str) -> dict: ok, why = assertions(item, pred) em = exact_match(pred, item.reference) return { "em": em, "f1": round(token_f1(pred, item.reference), 4), "rouge_l": round(rouge_l(pred, item.reference), 4), "ratio": round(seq_ratio(pred, item.reference), 4), "assert": ok, "assert_reason": why, "pass": em if item.kind == "exact" else ok, } # ------------------------------------------------------------------------ main --- GATES = [("exact match", lambda s: s["em"] == 1), ("token F1 >= 0.5", lambda s: s["f1"] >= 0.5), ("ROUGE-L >= 0.5", lambda s: s["rouge_l"] >= 0.5), ("difflib ratio >= 0.6", lambda s: s["ratio"] >= 0.6), ("assertions", lambda s: s["assert"] == 1), ("recommended verdict", lambda s: s["pass"] == 1)] def pct(k: int, n: int) -> str: return f"{k}/{n} ({100 * k / n:.1f}%)" def main() -> None: items = build_golden_set() rows = [] for it in items: ra, rb = bot_a(it.user_message), bot_b(it.user_message) sa, sb = score(it, ra), score(it, rb) rows.append((it, ra, rb, sa, sb)) n = len(items) print(f"golden set: {n} items, {len({it.intent for it in items})} intents\n") print("pass rate under each gate") print(f"{'gate':24s} {'bot A':>16s} {'bot B':>16s}") gate_rates = {} for name, fn in GATES: ka = sum(fn(sa) for _, _, _, sa, _ in rows) kb = sum(fn(sb) for _, _, _, _, sb in rows) gate_rates[name] = (ka / n, kb / n) print(f"{name:24s} {pct(ka, n):>16s} {pct(kb, n):>16s}") print("\nrecommended verdict by intent") print(f"{'intent':18s} {'n':>3s} {'A pass':>7s} {'B pass':>7s}") for intent in dict.fromkeys(it.intent for it in items): sub = [r for r in rows if r[0].intent == intent] print(f"{intent:18s} {len(sub):3d} {sum(r[3]['pass'] for r in sub):7d} {sum(r[4]['pass'] for r in sub):7d}") print("\nmean similarity to the reference") for key in ("f1", "rouge_l", "ratio"): ma = sum(r[3][key] for r in rows) / n mb = sum(r[4][key] for r in rows) / n print(f"{key:8s} A={ma:.3f} B={mb:.3f}") print("\nfailures under the recommended verdict") for it, ra, rb, sa, sb in rows: for bot, resp, s in (("A", ra, sa), ("B", rb, sb)): if not s["pass"]: why = s["assert_reason"] or "exact match failed" print(f" {it.item_id} {it.intent:16s} bot {bot}: {why}\n -> {resp}") print("\ndisagreements: token F1 >= 0.5 but verdict fail, or F1 < 0.5 but verdict pass") for it, ra, rb, sa, sb in rows: for bot, s in (("A", sa), ("B", sb)): if (s["f1"] >= 0.5) != bool(s["pass"]): print(f" {it.item_id} {it.intent:16s} bot {bot}: F1={s['f1']:.2f} verdict={'pass' if s['pass'] else 'fail'}") print("\ntoken F1 threshold sweep (share of items at or above the threshold)") sweep = [] for t10 in range(0, 11): t = t10 / 10 fa = sum(r[3]["f1"] >= t for r in rows) / n fb = sum(r[4]["f1"] >= t for r in rows) / n sweep.append((t, fa, fb)) print(f" t={t:.1f} A={100 * fa:5.1f}% B={100 * fb:5.1f}%") os.makedirs(os.path.dirname(CSV_OUT), exist_ok=True) with open(CSV_OUT, "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["item_id", "intent", "kind", "user_message", "reference", "must_contain", "must_not_contain", "must_not_match", "bot_a_response", "bot_b_response", "a_em", "a_f1", "a_rouge_l", "a_ratio", "a_assert", "a_pass", "b_em", "b_f1", "b_rouge_l", "b_ratio", "b_assert", "b_pass"]) for it, ra, rb, sa, sb in rows: w.writerow([it.item_id, it.intent, it.kind, it.user_message, it.reference, ";".join(it.must_contain), ";".join(it.must_not_contain), it.must_not_match, ra, rb, sa["em"], sa["f1"], sa["rouge_l"], sa["ratio"], sa["assert"], sa["pass"], sb["em"], sb["f1"], sb["rouge_l"], sb["ratio"], sb["assert"], sb["pass"]]) print(f"\nwrote {CSV_OUT}") chart(gate_rates, sweep) def chart(gate_rates: dict, sweep: list) -> None: try: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt except ImportError: print("matplotlib not installed; chart skipped") return blue, orange, ink, muted, grid = "#2a78d6", "#eb6834", "#14161f", "#5c5f6b", "#e3e3ea" fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 5.2), dpi=130, facecolor="#fcfcfb") for ax in (ax1, ax2): ax.set_facecolor("#fcfcfb") for side in ("top", "right"): ax.spines[side].set_visible(False) ax.spines["left"].set_color(grid) ax.spines["bottom"].set_color(grid) ax.tick_params(colors=muted, labelsize=8.5) ax.yaxis.grid(True, color=grid, linewidth=0.8) ax.set_axisbelow(True) names = [g for g, _ in GATES] ys = range(len(names)) a_vals = [100 * gate_rates[g][0] for g in names] b_vals = [100 * gate_rates[g][1] for g in names] h = 0.36 ax1.barh([y - h / 2 - 0.02 for y in ys], a_vals, height=h, color=blue, label="bot A (terse, mirrors the references)") ax1.barh([y + h / 2 + 0.02 for y in ys], b_vals, height=h, color=orange, label="bot B (rewritten, verbose)") for y, va, vb in zip(ys, a_vals, b_vals): ax1.text(va + 1, y - h / 2 - 0.02, f"{va:.0f}", va="center", fontsize=8, color=ink) ax1.text(vb + 1, y + h / 2 + 0.02, f"{vb:.0f}", va="center", fontsize=8, color=ink) ax1.set_yticks(list(ys)) ax1.set_yticklabels(names, color=ink) ax1.invert_yaxis() ax1.set_xlim(0, 112) ax1.xaxis.grid(True, color=grid, linewidth=0.8) ax1.yaxis.grid(False) ax1.set_xlabel("items passing (% of 48)", color=muted, fontsize=9) ax1.set_title("Same 48 items, six gates", loc="left", fontsize=11, color=ink, fontweight="bold") ax1.legend(loc="upper center", bbox_to_anchor=(0.5, -0.13), ncol=1, fontsize=8, frameon=False) ts = [t for t, _, _ in sweep] ax2.plot(ts, [100 * a for _, a, _ in sweep], color=blue, linewidth=2, marker="o", markersize=4.5, label="bot A") ax2.plot(ts, [100 * b for _, _, b in sweep], color=orange, linewidth=2, marker="o", markersize=4.5, label="bot B") va, vb = gate_rates["recommended verdict"] ax2.axhline(100 * va, color=blue, linewidth=1.2, linestyle=(0, (4, 3))) ax2.axhline(100 * vb, color=orange, linewidth=1.2, linestyle=(0, (4, 3))) ax2.text(0.56, 100 * va - 5.5, f"A verdict pass rate {100 * va:.0f}%", fontsize=8, color=ink) ax2.text(0.56, 100 * vb + 1.5, f"B verdict pass rate {100 * vb:.0f}%", fontsize=8, color=ink) ax2.set_xlabel("token F1 threshold", color=muted, fontsize=9) ax2.set_ylabel("items at or above threshold (%)", color=muted, fontsize=9) ax2.set_ylim(0, 105) ax2.set_xlim(-0.02, 1.02) ax2.set_title("Token F1 threshold sweep vs the verdict", loc="left", fontsize=11, color=ink, fontweight="bold") ax2.legend(loc="upper right", fontsize=8, frameon=False) fig.tight_layout() os.makedirs(os.path.dirname(PNG_OUT), exist_ok=True) fig.savefig(PNG_OUT, facecolor=fig.get_facecolor()) print(f"wrote {PNG_OUT} ({os.path.getsize(PNG_OUT) // 1024} KB)") if __name__ == "__main__": main()