"""llm-as-judge-mt-bench-gpt4.py -- position bias in GPT-4's own published MT-Bench judgments. Companion to llm-as-judge-position-bias.py. That listing runs local rule-based judges; this one calls no model at all. It re-scores judgments the MT-Bench authors (Zheng et al., arXiv 2306.05685v4) published, where GPT-4 judged every pair twice, once in each order. What it does 1. Downloads five files at pinned revisions (honest User-Agent, robots.txt read first, requests at least 1 s apart, SHA-256 checked) into a cache folder: - lmsys/mt-bench Space, revision a4b674ca: data/mt_bench/model_judgment/gpt-4_pair.jsonl (4,800 records: 30 models against gpt-3.5-turbo, 80 questions, 2 turns, both orders), gpt-4_single.jsonl (GPT-4's 1-10 grade of each answer alone) and question.jsonl. The Space states no data licence, so this listing reads those files at run time and the shipped CSV holds only counts computed from them. - lmsys/mt_bench_human_judgments dataset, revision f7d2896d (CC BY 4.0): the gpt4_pair split (2,400 six-model judgments, both orders combined) and the human split (3,355 votes by experts and by the paper's authors). 2. Classes each pair of verdicts the way Shi et al. (arXiv 2406.07791v9, Figure 1b) do, from the letters chosen in the original and the swapped prompt: (A,B) (B,A) (C,C) consistent; (A,A) (C,A) (A,C) primacy-preferred; (B,B) (C,B) (B,C) recency-preferred. In the file, g1 shows model_1 as Assistant A and g2 shows model_2 as A (FastChat common.py, play_a_match_pair), and winners are already mapped back to model_1 / model_2 / tie. 3. Reports position consistency (PC), primacy and recency counts, preference fairness PF = (recency - primacy) / n (Shi et al.'s equation 3 min-max scaled with the extremes taken as every pair primacy or every pair recency, which reduces to this), all of them by category, turn and prompt; PC against the gap between GPT-4's single-answer grades of the two answers; PC against Shi et al.'s overall-win-rate quality gap; and how many outputs a first-token parser and a last-token parser read differently. 4. Reproduces Table 5 of the MT-Bench paper (GPT-4 pairwise vs expert agreement, setups S1 and S2, and expert vs expert), then splits it by what GPT-4 did: consistent winner, consistent tie, or inconsistent. 5. Writes datasets/llm-as-judge-mt-bench-gpt4-swaps.csv (counts per model, category and turn), datasets/llm-as-judge-mt-bench-gpt4-experts.csv (one row per judged item from the CC BY dataset) and, with matplotlib, content/images/llm-as-judge-position-bias/gpt4-swaps.png. Run: python code/llm-as-judge-mt-bench-gpt4.py [--cache DIR] [--json FILE] (from the site folder) Deps: pyarrow 24.0.0 (reads the dataset's parquet files); matplotlib 3.10 optional (chart). Download is about 70 MB on the first run; later runs use the cache. """ from __future__ import annotations import argparse import csv import hashlib import itertools import json import os import re import sys import time import urllib.error import urllib.parse import urllib.request import urllib.robotparser from collections import Counter, defaultdict ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) UA = ("llm-as-judge-mt-bench-gpt4/1.0 " "(+https://matrixcognition.com/code/llm-as-judge-mt-bench-gpt4.py)") SPACE = "https://huggingface.co/spaces/lmsys/mt-bench/resolve/a4b674ca573c24143824ac7f60d9173e7081e37d/" DSET = ("https://huggingface.co/datasets/lmsys/mt_bench_human_judgments/resolve/" "f7d2896d2cc5d80f8b55c2bbc722613555233c25/data/") FILES = { # cache name: (url, sha256) "space_gpt-4_pair.jsonl": (SPACE + "data/mt_bench/model_judgment/gpt-4_pair.jsonl", "d662c0b7d1d297f0494fcb4cc09fe8f054fa22d75deb4754a483a921984bc585"), "space_gpt-4_single.jsonl": (SPACE + "data/mt_bench/model_judgment/gpt-4_single.jsonl", "76c55033c6b2b1cc3f62513458f84748a23352495fd42b1062a7401de5ff9bd9"), "space_question.jsonl": (SPACE + "data/mt_bench/question.jsonl", None), "gpt4_pair.parquet": (DSET + "gpt4_pair-00000-of-00001-c0b431264a82ddc0.parquet", None), "human.parquet": (DSET + "human-00000-of-00001-25f4910818759289.parquet", None), } CATEGORIES = ["writing", "roleplay", "reasoning", "math", "coding", "extraction", "stem", "humanities"] SWAPS_CSV = os.path.join(ROOT, "datasets", "llm-as-judge-mt-bench-gpt4-swaps.csv") EXPERTS_CSV = os.path.join(ROOT, "datasets", "llm-as-judge-mt-bench-gpt4-experts.csv") PNG_OUT = os.path.join(ROOT, "content", "images", "llm-as-judge-position-bias", "gpt4-swaps.png") # ---------------------------------------------------------------- download _robots: dict[str, urllib.robotparser.RobotFileParser] = {} _last: dict[str, float] = {} def allowed(url: str) -> bool: p = urllib.parse.urlparse(url) host = f"{p.scheme}://{p.netloc}" if host not in _robots: rp = urllib.robotparser.RobotFileParser() req = urllib.request.Request(host + "/robots.txt", headers={"User-Agent": UA, "Accept": "text/plain"}) try: rp.parse(urllib.request.urlopen(req, timeout=30).read().decode("utf-8", "replace").splitlines()) except urllib.error.HTTPError as e: rp.parse([] if e.code in (404, 410) else ["User-agent: *", "Disallow: /"]) _robots[host] = rp return all(_robots[host].can_fetch(a, url) for a in ("ClaudeBot", "Claude-User", "anthropic-ai", "*")) def fetch(cache: str) -> None: os.makedirs(cache, exist_ok=True) for name, (url, sha) in FILES.items(): path = os.path.join(cache, name) if not os.path.exists(path): if not allowed(url): sys.exit(f"robots.txt disallows {url}; stopping") host = urllib.parse.urlparse(url).netloc wait = _last.get(host, 0) + 1.0 - time.time() if wait > 0: time.sleep(wait) print("downloading", name) data = urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=300).read() _last[host] = time.time() with open(path, "wb") as f: f.write(data) if sha: got = hashlib.sha256(open(path, "rb").read()).hexdigest() if got != sha: sys.exit(f"{name}: sha256 {got} does not match the pinned file") def read_parquet(path: str) -> list[dict]: import pyarrow.parquet as pq return pq.read_table(path, columns=["question_id", "model_a", "model_b", "winner", "judge", "turn"]).to_pylist() def read_jsonl(path: str) -> list[dict]: with open(path, encoding="utf-8") as f: return [json.loads(line) for line in f if line.strip()] # ---------------------------------------------------------------- the swap test LETTER_G1 = {"model_1": "A", "model_2": "B", "tie": "C"} # g1: model_1 shown first LETTER_G2 = {"model_2": "A", "model_1": "B", "tie": "C"} # g2: model_2 shown first CONSISTENT = {("A", "B"), ("B", "A"), ("C", "C")} PRIMACY = {("A", "A"), ("C", "A"), ("A", "C")} RECENCY = {("B", "B"), ("C", "B"), ("B", "C")} VERDICT_RE = re.compile(r"\[\[([ABC])\]\]") def letters(rec: dict) -> tuple[str, str] | None: if rec["g1_winner"] == "error" or rec["g2_winner"] == "error": return None return LETTER_G1[rec["g1_winner"]], LETTER_G2[rec["g2_winner"]] def position_class(pair: tuple[str, str] | None) -> str: if pair is None: return "error" return "consistent" if pair in CONSISTENT else "primacy" if pair in PRIMACY else "recency" def fastchat_parse(text: str) -> str: """FastChat's rule (common.py run_judge_pair): [[A]] anywhere wins, then [[B]], then [[C]].""" for tok, out in (("[[A]]", "A"), ("[[B]]", "B"), ("[[C]]", "C")): if tok in text: return out return "error" def last_token_parse(text: str) -> str: """The companion listing's rule: the last verdict token wins.""" found = VERDICT_RE.findall(text) return found[-1] if found else "error" def summarise(recs: list[dict]) -> dict: c = Counter(r["cls"] for r in recs) n = c["consistent"] + c["primacy"] + c["recency"] return {"valid": n, "error": c["error"], "consistent": c["consistent"], "primacy": c["primacy"], "recency": c["recency"], "pc": c["consistent"] / n if n else float("nan"), "pf": (c["recency"] - c["primacy"]) / n if n else float("nan")} def gap_bin(g: float) -> str: return "0-0.5" if g < 1 else "1-1.5" if g < 2 else "2-2.5" if g < 3 else "3-4.5" if g < 5 else "5+" GAP_BINS = ["0-0.5", "1-1.5", "2-2.5", "3-4.5", "5+"] def swap_analysis(pairs: list[dict], singles: list[dict], cat_of: dict) -> dict: for r in pairs: r["cat"] = cat_of[r["question_id"]] r["prompt"] = r["judge"][1].replace("-multi-turn", "") r["cls"] = position_class(letters(r)) out = {"records": len(pairs), "all": summarise(pairs), "by_turn": {t: summarise([r for r in pairs if r["turn"] == t]) for t in (1, 2)}, "by_category": {c: summarise([r for r in pairs if r["cat"] == c]) for c in CATEGORIES}, "by_prompt": {p: summarise([r for r in pairs if r["prompt"] == p]) for p in sorted({r["prompt"] for r in pairs})}, "by_category_turn": {f"{c}|{t}": summarise([r for r in pairs if r["cat"] == c and r["turn"] == t]) for c in CATEGORIES for t in (1, 2)}, "choice_pairs": Counter("".join(letters(r)) for r in pairs if letters(r)), "baselines": sorted({r["model_2"] for r in pairs}), "models": sorted({r["model_1"] for r in pairs})} # per (model, category) cells, both turns: Shi et al. average PC and PF over candidates and tasks cells = defaultdict(list) for r in pairs: if r["cls"] != "error": cells[(r["model_1"], r["cat"])].append(r) pcs = [sum(x["cls"] == "consistent" for x in v) / len(v) for v in cells.values()] mean = sum(pcs) / len(pcs) out["cells"] = {"n": len(cells), "pc_mean": mean, "pc_sd": (sum((p - mean) ** 2 for p in pcs) / (len(pcs) - 1)) ** 0.5} # Shi et al.'s answer quality gap: owr = (Cw + (Ct + CI)/2) / n, dq = |owr - 1/2|, per cell dq_bins = defaultdict(lambda: [0, 0]) for v in cells.values(): cw = sum(1 for x in v if x["cls"] == "consistent" and x["g1_winner"] == "model_1") owr = (cw + (len(v) - cw - sum(1 for x in v if x["cls"] == "consistent" and x["g1_winner"] == "model_2")) / 2) / len(v) b = min(int(abs(owr - 0.5) * 10), 4) dq_bins[b][0] += sum(x["cls"] == "consistent" for x in v) dq_bins[b][1] += len(v) out["dq_bins"] = {f"{b / 10:.1f}-{(b + 1) / 10:.1f}": {"pairs": n, "pc": k / n} for b, (k, n) in sorted(dq_bins.items())} # an outside measure of the gap: GPT-4's 1-10 grade of each answer judged alone score = {(s["question_id"], s["model"], s["turn"]): s["score"] for s in singles} by_gap = {b: Counter() for b in GAP_BINS} missing = 0 for r in pairs: if r["cls"] == "error": continue a = score.get((r["question_id"], r["model_1"], r["turn"])) b = score.get((r["question_id"], r["model_2"], r["turn"])) if a is None or b is None or a < 0 or b < 0: missing += 1 continue r["gap"] = abs(a - b) by_gap[gap_bin(r["gap"])][r["cls"]] += 1 out["gap_missing"] = missing out["by_gap"] = {b: {"pairs": sum(c.values()), "consistent": c["consistent"], "primacy": c["primacy"], "recency": c["recency"], "pc": c["consistent"] / sum(c.values())} for b, c in by_gap.items()} # parser check: FastChat's first-match rule against a last-token rule, on every output text texts = [(r, g) for r in pairs for g in ("g1", "g2")] differ = [(r, g) for r, g in texts if fastchat_parse(r[g + "_judgment"]) != last_token_parse(r[g + "_judgment"])] multi = sum(1 for r, g in texts if len(set(VERDICT_RE.findall(r[g + "_judgment"]))) > 1) out["parse"] = {"outputs": len(texts), "multi_token": multi, "differ": len(differ), "differ_ids": [(r["question_id"], r["model_1"], r["turn"], g) for r, g in differ]} stamps = [r["tstamp"] for r in pairs] out["tstamp"] = [time.strftime("%Y-%m-%d", time.gmtime(min(stamps))), time.strftime("%Y-%m-%d", time.gmtime(max(stamps)))] out["_cells_rows"] = pairs return out # ---------------------------------------------------------------- agreement with experts (CC BY 4.0 dataset) def item_key(r: dict) -> tuple[tuple, str]: w = {"model_a": r["model_a"], "model_b": r["model_b"]}.get(r["winner"], r["winner"]) return (r["question_id"], tuple(sorted((r["model_a"], r["model_b"]))), r["turn"]), w def as_tie(w: str) -> str: return "tie" if w in ("tie", "tie (inconsistent)") else w def agreement(g4: list[dict], human: list[dict]) -> dict: G = {} for r in g4: k, w = item_key(r) assert k not in G, k G[k] = w H = defaultdict(list) for r in human: k, w = item_key(r) H[k].append((r["judge"], w)) assert all(k in G for k in H) def g4_vs(group: str, keep) -> dict: s1 = [0, 0] s2 = [0, 0] for k, votes in H.items(): if not keep(k): continue g = as_tie(G[k]) for j, w in votes: if not j.startswith(group): continue s1[0] += g == w s1[1] += 1 if g != "tie" and w != "tie": s2[0] += g == w s2[1] += 1 return {"s1": s1, "s2": s2} def hh(keep) -> dict: s1 = [0, 0] s2 = [0, 0] for k, votes in H.items(): if not keep(k): continue ex = [(j, w) for j, w in votes if j.startswith("expert")] for (j1, w1), (j2, w2) in itertools.combinations(ex, 2): if j1 == j2: continue s1[0] += w1 == w2 s1[1] += 1 if w1 != "tie" and w2 != "tie": s2[0] += w1 == w2 s2[1] += 1 return {"s1": s1, "s2": s2} def kind(k): return "inconsistent" if G[k] == "tie (inconsistent)" else "tie" if G[k] == "tie" else "winner" out = {"items": len(G), "g4_kinds": Counter(kind(k) for k in G), "human_items": len(H), "votes": Counter(j.split("_")[0] for v in H.values() for j, _ in v)} for t in (1, 2): out[f"table5_turn{t}"] = {"g4_expert": g4_vs("expert", lambda k, t=t: k[2] == t), "g4_author": g4_vs("author", lambda k, t=t: k[2] == t), "expert_expert": hh(lambda k, t=t: k[2] == t), "g4_expert_consistent_only": g4_vs("expert", lambda k, t=t: k[2] == t and kind(k) != "inconsistent")} for kd in ("winner", "tie", "inconsistent"): votes = [w for k in H if kind(k) == kd for j, w in H[k] if j.startswith("expert")] out[f"by_kind_{kd}"] = {"items_with_expert_votes": sum(1 for k in H if kind(k) == kd and any(j.startswith("expert") for j, _ in H[k])), "expert_votes": len(votes), "expert_tie_votes": sum(w == "tie" for w in votes), "g4_expert": g4_vs("expert", lambda k, kd=kd: kind(k) == kd), "expert_expert": hh(lambda k, kd=kd: kind(k) == kd)} out["_G"], out["_H"] = G, H return out # ---------------------------------------------------------------- outputs def write_swaps_csv(pairs: list[dict]) -> int: cells = defaultdict(Counter) for r in pairs: key = (r["model_1"], r["cat"], r["turn"], r["prompt"]) lt = letters(r) if lt is None: cells[key]["error"] += 1 continue cells[key]["".join(lt)] += 1 cols = ["AB", "BA", "CC", "AA", "CA", "AC", "BB", "CB", "BC", "error"] with open(SWAPS_CSV, "w", newline="", encoding="utf-8") as f: w = csv.writer(f, lineterminator="\n") w.writerow(["model", "baseline", "category", "turn", "prompt", "pairs", "consistent", "primacy", "recency"] + [f"n_{c}" for c in cols]) for (m, cat, t, p) in sorted(cells, key=lambda k: (k[0], CATEGORIES.index(k[1]), k[2])): c = cells[(m, cat, t, p)] w.writerow([m, "gpt-3.5-turbo", cat, t, p, sum(c.values()), c["AB"] + c["BA"] + c["CC"], c["AA"] + c["CA"] + c["AC"], c["BB"] + c["CB"] + c["BC"]] + [c[x] for x in cols]) return len(cells) def write_experts_csv(ag: dict, cat_of: dict) -> int: G, H = ag["_G"], ag["_H"] with open(EXPERTS_CSV, "w", newline="", encoding="utf-8") as f: w = csv.writer(f, lineterminator="\n") w.writerow(["question_id", "category", "turn", "model_x", "model_y", "gpt4_pair", "expert_x", "expert_y", "expert_tie", "author_x", "author_y", "author_tie"]) for k in sorted(G): q, (mx, my), t = k g = G[k] g = "x" if g == mx else "y" if g == my else g cnt = Counter((j.split("_")[0], "x" if v == mx else "y" if v == my else "tie") for j, v in H.get(k, [])) w.writerow([q, cat_of[q], t, mx, my, g, cnt[("expert", "x")], cnt[("expert", "y")], cnt[("expert", "tie")], cnt[("author", "x")], cnt[("author", "y")], cnt[("author", "tie")]]) return len(G) def chart(sw: dict) -> None: try: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt except ImportError: print("matplotlib not installed; chart skipped") return surface, ink, ink2, muted, grid, axis = "#fcfcfb", "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7" pri_c, rec_c = "#2a78d6", "#eb6834" fig = plt.figure(figsize=(11.6, 5.4), dpi=130, facecolor=surface) gs = fig.add_gridspec(1, 2, width_ratios=[1.35, 1], wspace=0.32) ax1, ax2 = fig.add_subplot(gs[0]), fig.add_subplot(gs[1]) rows = [("all, turn 1", sw["by_turn"][1]), ("all, turn 2", sw["by_turn"][2])] + \ [(c, sw["by_category"][c]) for c in CATEGORIES] rows = rows[::-1] h = 0.36 for i, (label, s) in enumerate(rows): ax1.barh(i + h / 2 + 0.01, s["primacy"], height=h, color=pri_c, zorder=3, label="primacy-preferred: the verdicts favour the answer shown first" if i == 0 else None) ax1.barh(i - h / 2 - 0.01, s["recency"], height=h, color=rec_c, zorder=3, label="recency-preferred: they favour the answer shown second" if i == 0 else None) ax1.text(s["primacy"] + 3, i + h / 2, str(s["primacy"]), va="center", fontsize=7.5, color=ink2) ax1.text(s["recency"] + 3, i - h / 2, str(s["recency"]), va="center", fontsize=7.5, color=ink2) ax1.set_yticks(range(len(rows))) ax1.set_yticklabels([r[0] for r in rows], fontsize=8.5, color=ink) ax1.axhline(7.5, color=axis, linewidth=0.8) ax1.set_xlim(0, 300) ax1.set_facecolor(surface) ax1.tick_params(colors=muted, labelsize=8.5, length=0) ax1.tick_params(axis="y", labelcolor=ink) ax1.xaxis.grid(True, color=grid, linewidth=0.8, zorder=0) for side in ("top", "right", "left"): ax1.spines[side].set_visible(False) ax1.spines["bottom"].set_color(axis) ax1.set_xlabel("inconsistent pairs (count)", fontsize=8.5, color=muted) ax1.set_title("Which way GPT-4 leaned when it flipped", loc="left", fontsize=11, color=ink, fontweight="bold") ax1.legend(loc="upper right", bbox_to_anchor=(1.0, 0.7), fontsize=7.5, frameon=False, labelcolor=ink2, handlelength=1.2) bins = GAP_BINS vals = [100 * sw["by_gap"][b]["pc"] for b in bins] ax2.bar(range(len(bins)), vals, width=0.72, color=pri_c, zorder=3) for i, b in enumerate(bins): ax2.text(i, vals[i] + 1.5, f"{vals[i]:.1f}%", ha="center", fontsize=8, color=ink2) ax2.text(i, 4, f"{sw['by_gap'][b]['pairs']:,} pairs", ha="center", fontsize=6.5, color="#ffffff") ax2.set_xticks(range(len(bins))) ax2.set_xticklabels(bins, fontsize=8.5, color=ink) ax2.set_ylim(0, 105) ax2.set_yticks([0, 25, 50, 75, 100]) ax2.set_facecolor(surface) ax2.tick_params(colors=muted, labelsize=8.5, length=0) ax2.yaxis.grid(True, color=grid, linewidth=0.8, zorder=0) for side in ("top", "right", "left"): ax2.spines[side].set_visible(False) ax2.spines["bottom"].set_color(axis) ax2.set_xlabel("gap between GPT-4's 1-10 grades of the two answers", fontsize=8.5, color=muted) ax2.set_ylabel("swap-consistent pairs (%)", fontsize=8.5, color=muted) ax2.set_title("Consistency rises with the quality gap", loc="left", fontsize=11, color=ink, fontweight="bold") fig.text(0.01, 0.01, "Data: GPT-4 pairwise and single-answer judgments published with MT-Bench (Zheng et al., " "arXiv 2306.05685), 4,796 valid pairs (4,792 with both single grades).", fontsize=7, color=muted) fig.subplots_adjust(left=0.1, right=0.98, top=0.9, bottom=0.16) os.makedirs(os.path.dirname(PNG_OUT), exist_ok=True) fig.savefig(PNG_OUT, facecolor=surface) plt.close(fig) print("wrote", PNG_OUT) def pct(a: int, b: int) -> str: return f"{100 * a / b:.1f}% ({a}/{b})" def main() -> None: sys.stdout.reconfigure(encoding="utf-8") ap = argparse.ArgumentParser() ap.add_argument("--cache", default=os.path.join(ROOT, ".mtbench-cache")) ap.add_argument("--json", help="also write every reported number to this JSON file") ap.add_argument("--no-chart", action="store_true") a = ap.parse_args() fetch(a.cache) pairs = read_jsonl(os.path.join(a.cache, "space_gpt-4_pair.jsonl")) singles = read_jsonl(os.path.join(a.cache, "space_gpt-4_single.jsonl")) cat_of = {q["question_id"]: q["category"] for q in read_jsonl(os.path.join(a.cache, "space_question.jsonl"))} sw = swap_analysis(pairs, singles, cat_of) ag = agreement(read_parquet(os.path.join(a.cache, "gpt4_pair.parquet")), read_parquet(os.path.join(a.cache, "human.parquet"))) s = sw["all"] print(f"GPT-4 swap test: {sw['records']} records, {s['valid']} valid, {s['error']} with an unparsed output; " f"judged {sw['tstamp'][0]} to {sw['tstamp'][1]}; baseline {sw['baselines']}, {len(sw['models'])} models") print(f" consistent {pct(s['consistent'], s['valid'])}; primacy {s['primacy']}, recency {s['recency']}; PF {s['pf']:+.3f}") print(f" choice pairs {dict(sw['choice_pairs'])}") print(f" per (model, category) cell: {sw['cells']['n']} cells, PC mean {sw['cells']['pc_mean']:.3f} sd {sw['cells']['pc_sd']:.3f}") for t, v in sw["by_turn"].items(): print(f" turn {t}: PC {pct(v['consistent'], v['valid'])}, primacy {v['primacy']}, recency {v['recency']}, PF {v['pf']:+.3f}") for p, v in sw["by_prompt"].items(): print(f" prompt {p}: PC {pct(v['consistent'], v['valid'])}, primacy {v['primacy']}, recency {v['recency']}") print(" category PC primacy recency PF turn1 p/r turn2 p/r") for c in CATEGORIES: v, t1, t2 = sw["by_category"][c], sw["by_category_turn"][f"{c}|1"], sw["by_category_turn"][f"{c}|2"] print(f" {c:11s} {100 * v['pc']:5.1f}% {v['primacy']:5d} {v['recency']:5d} {v['pf']:+.3f} " f"{t1['primacy']:4d}/{t1['recency']:<4d} {t2['primacy']:4d}/{t2['recency']:<4d}") print(f" by single-grade gap ({sw['gap_missing']} pairs without both grades):") for b, v in sw["by_gap"].items(): print(f" {b:6s} pairs {v['pairs']:5d} PC {100 * v['pc']:.1f}% primacy {v['primacy']} recency {v['recency']}") print(" by Shi et al. overall-win-rate gap (built from these same verdicts):") for b, v in sw["dq_bins"].items(): print(f" dq {b}: pairs {v['pairs']} PC {100 * v['pc']:.1f}%") pr = sw["parse"] print(f" parser check: {pr['outputs']} outputs, {pr['multi_token']} with two different verdict tokens, " f"{pr['differ']} read differently by first-match and last-token rules {pr['differ_ids']}") print(f"\nExperts (CC BY 4.0 dataset): {ag['items']} GPT-4 items {dict(ag['g4_kinds'])}; " f"{ag['human_items']} items with human votes; votes {dict(ag['votes'])}") for t in (1, 2): T = ag[f"table5_turn{t}"] print(f" turn {t}: G4-pair vs expert S1 {pct(*T['g4_expert']['s1'])}, S2 {pct(*T['g4_expert']['s2'])}; " f"vs author S1 {pct(*T['g4_author']['s1'])}, S2 {pct(*T['g4_author']['s2'])}; " f"expert vs expert S1 {pct(*T['expert_expert']['s1'])}, S2 {pct(*T['expert_expert']['s2'])}; " f"S1 without GPT-4's inconsistent items {pct(*T['g4_expert_consistent_only']['s1'])}") for kd in ("winner", "tie", "inconsistent"): K = ag[f"by_kind_{kd}"] print(f" GPT-4 {kd:12s}: {K['items_with_expert_votes']} items, {K['expert_votes']} expert votes, " f"tie votes {pct(K['expert_tie_votes'], K['expert_votes'])}; G4 S1 {pct(*K['g4_expert']['s1'])}; " f"expert vs expert S1 {pct(*K['expert_expert']['s1'])}, S2 {pct(*K['expert_expert']['s2'])}") n1 = write_swaps_csv(sw["_cells_rows"]) n2 = write_experts_csv(ag, cat_of) print(f"\nwrote {SWAPS_CSV} ({n1} rows) and {EXPERTS_CSV} ({n2} rows)") if not a.no_chart: chart(sw) if a.json: dump = {"swap": {k: v for k, v in sw.items() if not k.startswith("_")}, "experts": {k: v for k, v in ag.items() if not k.startswith("_")}} with open(a.json, "w", encoding="utf-8") as f: json.dump(dump, f, indent=1, default=lambda o: dict(o) if isinstance(o, Counter) else str(o)) if __name__ == "__main__": main()