"""llm-as-judge-position-bias.py -- a pairwise judge harness, run with four local judges over
the 48-item golden set from golden-test-set-chatbot.py, measuring position consistency,
length (padding) bias, agreement with the golden verdict, and agreement between judges.

What it does
  1. Loads the golden set, both bots and the scorers from code/golden-test-set-chatbot.py,
     and checks that the responses and verdicts match datasets/golden-test-set-chatbot.csv.
  2. Builds pairwise prompts in the MT-Bench format (Zheng et al., arXiv 2306.05685v4,
     Figure 5, plus an optional reference block that is this file's addition) and parses the
     "[[A]]" / "[[B]]" / "[[C]]" verdict.
  3. Runs four deterministic local judges through that same prompt-in, text-out interface.
     None of them is a language model; each is a named rule, so every verdict reproduces.
     To test a real model, add a function that sends the prompt and returns its text to JUDGES.
  4. Asks every judge about every pair twice, once in each order, and reports: position
     consistency with primacy / recency counts (Shi et al., arXiv 2406.07791v9), the
     conservative both-orders verdict (Zheng et al., section 3.4), agreement with the golden
     verdict with and without ties, Cohen's kappa, the signed verbosity-bias score (Saito et
     al., arXiv 2310.10076v1, equation 6, adapted as described in verbosity_bias), an
     identical-answer probe, a padding probe, and the judge-by-judge kappa matrix.
  5. Writes datasets/llm-as-judge-position-bias.csv (one row per judge call) and, if
     matplotlib is installed, content/images/llm-as-judge-position-bias/judges.png.

Inputs: code/golden-test-set-chatbot.py and datasets/golden-test-set-chatbot.csv.
Run:    python code/llm-as-judge-position-bias.py        (from the site folder)
Deps:   standard library; matplotlib 3.10 optional for the chart; scikit-learn optional,
        used only to cross-check the kappa implementation when it is installed.
"""
from __future__ import annotations

import csv
import importlib.util
import math
import os
import re
import sys
from collections import Counter

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
GOLDEN_PY = os.path.join(ROOT, "code", "golden-test-set-chatbot.py")
GOLDEN_CSV = os.path.join(ROOT, "datasets", "golden-test-set-chatbot.csv")
CSV_OUT = os.path.join(ROOT, "datasets", "llm-as-judge-position-bias.csv")
PNG_OUT = os.path.join(ROOT, "content", "images", "llm-as-judge-position-bias", "judges.png")

# One content-free sentence. Appending it adds words and no information.
FILLER = "I hope that helps, and if there is anything else I can do for you today, just let me know."


def load_golden():
    spec = importlib.util.spec_from_file_location("golden_set", GOLDEN_PY)
    mod = importlib.util.module_from_spec(spec)
    sys.modules["golden_set"] = mod          # dataclasses need the module registered
    spec.loader.exec_module(mod)
    return mod


G = load_golden()

# ------------------------------------------------------------- prompt and parse ---
# The system text is Figure 5 of arXiv 2306.05685v4 (straight apostrophes).
SYSTEM = (
    "Please act as an impartial judge and evaluate the quality of the responses provided by two "
    "AI assistants to the user question displayed below. You should choose the assistant that "
    "follows the user's instructions and answers the user's question better. Your evaluation "
    "should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, "
    "and level of detail of their responses. Begin your evaluation by comparing the two "
    "responses and provide a short explanation. Avoid any position biases and ensure that the "
    "order in which the responses were presented does not influence your decision. Do not allow "
    "the length of the responses to influence your evaluation. Do not favor certain names of "
    "the assistants. Be as objective as possible. After providing your explanation, output your "
    'final verdict by strictly following this format: "[[A]]" if assistant A is better, "[[B]]" '
    'if assistant B is better, and "[[C]]" for a tie.'
)


def build_prompt(question: str, answer_a: str, answer_b: str, reference: str | None = None) -> str:
    parts = ["[System]", SYSTEM]
    if reference:                                   # not in Figure 5; added for reference-guided judges
        parts += ["[Reference Answer]", reference]
    parts += ["[User Question]", question,
              "[The Start of Assistant A's Answer]", answer_a, "[The End of Assistant A's Answer]",
              "[The Start of Assistant B's Answer]", answer_b, "[The End of Assistant B's Answer]"]
    return "\n".join(parts)


VERDICT_RE = re.compile(r"\[\[([ABC])\]\]")


def parse_verdict(text: str) -> str:
    """Last [[A]]/[[B]]/[[C]] in the output wins; anything else is an error, never a guess."""
    found = VERDICT_RE.findall(text)
    return found[-1] if found else "error"


def unpack(prompt: str) -> dict:
    """The local judges read the same prompt text a model would receive."""
    def between(start: str, end: str) -> str:
        m = re.search(re.escape(start) + r"\n(.*?)\n" + re.escape(end), prompt, flags=re.S)
        return m.group(1) if m else ""
    return {
        "reference": between("[Reference Answer]", "[User Question]"),
        "question": between("[User Question]", "[The Start of Assistant A's Answer]"),
        "a": between("[The Start of Assistant A's Answer]", "[The End of Assistant A's Answer]"),
        "b": between("[The Start of Assistant B's Answer]", "[The End of Assistant B's Answer]"),
    }


# ----------------------------------------------------------------- four judges ---
def judge_longer(prompt: str) -> str:
    """Prefers whichever answer has more words. A pure length preference, as a control."""
    f = unpack(prompt)
    la, lb = len(f["a"].split()), len(f["b"].split())
    v = "A" if la > lb else "B" if lb > la else "C"
    return f"Assistant A uses {la} words and assistant B uses {lb}. [[{v}]]"


def scale_score(answer: str, reference: str, points: int) -> int:
    """A 1..points score from token F1 against the reference (points=10 is the Vicuna-style scale)."""
    return 1 + round((points - 1) * G.token_f1(answer, reference))


def make_f1_judge(points: int, tie_to_first: bool):
    """Reference-guided scores on a 1..points scale. tie_to_first=True reproduces the `>=` bug:
    equal scores go to whichever answer was shown as assistant A."""
    def judge(prompt: str) -> str:
        f = unpack(prompt)
        sa, sb = scale_score(f["a"], f["reference"], points), scale_score(f["b"], f["reference"], points)
        if tie_to_first:
            v = "A" if sa >= sb else "B"
        else:
            v = "A" if sa > sb else "B" if sb > sa else "C"
        return f"Scores against the reference (1-{points}): A={sa}, B={sb}. [[{v}]]"
    return judge


STATUS_WORDS = ("in transit", "delivered", "out for delivery", "held at depot", "label created")


def rubric_points(question: str, answer: str) -> int:
    """A reference-free rubric that sounds sensible: on the customer's parcel, concrete, brief."""
    pts = 0
    oid = re.search(r"KP-\d{4}", question)
    if oid and oid.group(0) in answer:
        pts += 1                                    # talks about the parcel that was asked about
    if re.search(G.DATE_RE, answer):
        pts += 1                                    # gives a concrete date
    if any(s in answer.lower() for s in STATUS_WORDS):
        pts += 1                                    # names a status
    if len(answer.split()) <= 20:
        pts += 1                                    # concise
    return pts


def judge_rubric(prompt: str) -> str:
    f = unpack(prompt)
    pa, pb = rubric_points(f["question"], f["a"]), rubric_points(f["question"], f["b"])
    v = "A" if pa > pb else "B" if pb > pa else "C"
    return f"Rubric points: A={pa}/4, B={pb}/4. [[{v}]]"


JUDGES = [
    ("longer", "longer answer wins", judge_longer),
    ("f1_10_first", "F1 1-10, tie to first", make_f1_judge(10, True)),
    ("f1_3_first", "F1 1-3, tie to first", make_f1_judge(3, True)),
    ("f1_3_tie", "F1 1-3, tie declared", make_f1_judge(3, False)),
    ("rubric", "reference-free rubric", judge_rubric),
]
SWEEP_POINTS = [10, 7, 5, 4, 3, 2]

# ---------------------------------------------------------------------- pairs ---
def check_against_csv(items) -> None:
    with open(GOLDEN_CSV, encoding="utf-8", newline="") as fh:
        shipped = {r["item_id"]: r for r in csv.DictReader(fh)}
    assert len(shipped) == len(items) == 48, "expected the 48-item golden set"
    for it in items:
        r = shipped[it.item_id]
        ra, rb = G.bot_a(it.user_message), G.bot_b(it.user_message)
        assert (ra, rb) == (r["bot_a_response"], r["bot_b_response"]), it.item_id
        assert G.score(it, ra)["pass"] == int(r["a_pass"]), it.item_id
        assert G.score(it, rb)["pass"] == int(r["b_pass"]), it.item_id


def build_pairs(items) -> list[dict]:
    """x and y are candidate identities; the order they are shown in is decided per call."""
    pairs = []
    for it in items:
        ra, rb = G.bot_a(it.user_message), G.bot_b(it.user_message)
        pa, pb = G.score(it, ra)["pass"], G.score(it, rb)["pass"]
        base = {"item_id": it.item_id, "intent": it.intent, "question": it.user_message,
                "reference": it.reference}
        # main: bot A (x) against bot B (y), gold from the golden verdicts
        pairs.append({**base, "probe": "main", "x": ra, "y": rb,
                      "gold": "x" if pa > pb else "y" if pb > pa else "tie"})
        good = ra if pa else rb                     # every item has at least one passing reply
        # identical: the same passing reply twice; the only right answer is a tie
        pairs.append({**base, "probe": "identical", "x": good, "y": good, "gold": "tie"})
        # padding: the passing reply against itself plus FILLER, gold from the assertions
        padded = f"{good} {FILLER}"
        padded_ok = G.score(it, padded)["pass"]
        pairs.append({**base, "probe": "padding", "x": good, "y": padded,
                      "gold": "tie" if padded_ok else "x"})
    return pairs


# -------------------------------------------------------------------- metrics ---
def position_class(v1: str, v2: str) -> str:
    """v1: verdict with x shown first; v2: with y shown first. Shi et al. Figure 1(b)."""
    if "error" in (v1, v2):
        return "error"
    if (v1, v2) in {("A", "B"), ("B", "A"), ("C", "C")}:
        return "consistent"
    if (v1, v2) in {("A", "A"), ("A", "C"), ("C", "A")}:
        return "primacy"
    return "recency"


def cohen_kappa(l1: list[str], l2: list[str]) -> float:
    """kappa = (p_o - p_e) / (1 - p_e), p_e from each rater's own label frequencies."""
    n = len(l1)
    po = sum(a == b for a, b in zip(l1, l2)) / n
    c1, c2 = Counter(l1), Counter(l2)
    pe = sum(c1[k] * c2[k] for k in set(l1) | set(l2)) / (n * n)
    return float("nan") if pe == 1 else (po - pe) / (1 - pe)


def verbosity_bias(main: list[dict]) -> tuple[float, int, int]:
    """Saito et al. eq. 6: P(wrong | the correct answer is the shorter one) minus
    P(wrong | the correct answer is the longer one), over pairs whose gold is not a tie.
    Adaptation: a tie from the judge counts as wrong. Positive = favours the longer answer."""
    def longer(r):
        return "x" if len(r["x"].split()) > len(r["y"].split()) else "y"
    shorter_right = [r for r in main if r["gold"] != "tie" and r["gold"] != longer(r)]
    longer_right = [r for r in main if r["gold"] != "tie" and r["gold"] == longer(r)]
    if not shorter_right or not longer_right:
        return float("nan"), len(shorter_right), len(longer_right)
    e1 = sum(r["cons"] != r["gold"] for r in shorter_right) / len(shorter_right)
    e2 = sum(r["cons"] != r["gold"] for r in longer_right) / len(longer_right)
    return e1 - e2, len(shorter_right), len(longer_right)


def pct(k: int, n: int) -> str:
    return f"{k}/{n} ({100 * k / n:.1f}%)"


def fmt(x: float) -> str:
    return "n/a" if math.isnan(x) else f"{x:+.2f}"


# ----------------------------------------------------------------------- main ---
def main() -> None:
    items = G.build_golden_set()
    check_against_csv(items)
    pairs = build_pairs(items)
    results: dict[str, list[dict]] = {}
    rows = []
    calls = 0
    for key, label, judge in JUDGES:
        res = []
        for p in pairs:
            raw1 = judge(build_prompt(p["question"], p["x"], p["y"], p["reference"]))
            raw2 = judge(build_prompt(p["question"], p["y"], p["x"], p["reference"]))
            calls += 2
            v1, v2 = parse_verdict(raw1), parse_verdict(raw2)
            m1 = {"A": "x", "B": "y", "C": "tie"}.get(v1, "error")
            m2 = {"A": "y", "B": "x", "C": "tie"}.get(v2, "error")
            cons = m1 if (m1 == m2 and m1 != "error") else "tie"      # Zheng et al. 3.4
            r = {**p, "v1": v1, "v2": v2, "m1": m1, "m2": m2, "cons": cons,
                 "pos": position_class(v1, v2)}
            res.append(r)
            for order, raw, v, m in (("x_first", raw1, v1, m1), ("y_first", raw2, v2, m2)):
                rows.append([p["probe"], p["item_id"], p["intent"], key, order, v, m, cons,
                             r["pos"], p["gold"], len(p["x"].split()), len(p["y"].split()), raw])
        results[key] = res

    gold_counts = Counter(r["gold"] for r in results["longer"] if r["probe"] == "main")
    print(f"golden set: {len(items)} items; {len(pairs)} pairs; {calls} judge calls")
    print(f"main pairs gold: bot A better {gold_counts['x']}, bot B better {gold_counts['y']}, "
          f"tie {gold_counts['tie']}\n")

    summary = {}
    for key, label, _ in JUDGES:
        res = results[key]
        main_ = [r for r in res if r["probe"] == "main"]
        ident = [r for r in res if r["probe"] == "identical"]
        pad = [r for r in res if r["probe"] == "padding"]
        n = len(main_)
        pos = Counter(r["pos"] for r in main_)
        s2 = [r for r in main_ if r["gold"] != "tie" and r["cons"] != "tie"]
        decisive = [r for r in main_ if r["gold"] != "tie"]
        vb, n_short, n_long = verbosity_bias(main_)
        summary[key] = {
            "consistency": pos["consistent"] / n,
            "primacy": pos["primacy"], "recency": pos["recency"],
            "single_x_first": sum(r["m1"] == r["gold"] for r in main_) / n,
            "single_y_first": sum(r["m2"] == r["gold"] for r in main_) / n,
            "cons_acc": sum(r["cons"] == r["gold"] for r in main_) / n,
            "s2_acc": (sum(r["cons"] == r["gold"] for r in s2) / len(s2)) if s2 else float("nan"),
            "s2_n": len(s2),
            "decisive_hits": sum(r["cons"] == r["gold"] for r in decisive), "decisive_n": len(decisive),
            "decisive_acc": sum(r["cons"] == r["gold"] for r in decisive) / len(decisive),
            "kappa_gold": cohen_kappa([r["gold"] for r in main_], [r["cons"] for r in main_]),
            "verbosity": vb, "vb_n": (n_short, n_long),
            "ident_single_winner": (sum(r["v1"] != "C" for r in ident) + sum(r["v2"] != "C" for r in ident)) / (2 * len(ident)),
            "ident_cons_winner": sum(r["cons"] != "tie" for r in ident) / len(ident),
            "pad_pref": sum(r["cons"] == "y" for r in pad) / len(pad),
            "pad_orig": sum(r["cons"] == "x" for r in pad) / len(pad),
            "pad_acc": sum(r["cons"] == r["gold"] for r in pad) / len(pad),
            "pad_consistency": sum(r["pos"] == "consistent" for r in pad) / len(pad),
        }

    print("main pairs (bot A vs bot B, 48 pairs, each judged in both orders)")
    print(f"{'judge':24s} {'consistent':>15s} {'primacy':>8s} {'recency':>8s} {'1 call, A first':>16s} "
          f"{'1 call, B first':>16s} {'both orders':>12s} {'no-tie agree':>14s} {'kappa':>6s} {'verbosity':>9s}")
    for key, label, _ in JUDGES:
        s = summary[key]
        s2 = "n/a" if math.isnan(s["s2_acc"]) else f"{100 * s['s2_acc']:.1f}% of {s['s2_n']}"
        print(f"{label:24s} {pct(round(s['consistency'] * 48), 48):>15s} {s['primacy']:8d} {s['recency']:8d} "
              f"{100 * s['single_x_first']:15.1f}% {100 * s['single_y_first']:15.1f}% {100 * s['cons_acc']:11.1f}% "
              f"{s2:>14s} {fmt(s['kappa_gold']):>6s} {fmt(s['verbosity']):>9s}")
    ns, nl = summary["longer"]["vb_n"]
    print(f"(verbosity: Saito et al. eq. 6 over the {ns} pairs where the shorter answer is right and "
          f"the {nl} where the longer one is)")

    print("\ndecisive pairs (gold is not a tie): judge's both-orders verdict matches")
    for key, label, _ in JUDGES:
        s = summary[key]
        print(f"  {label:24s} {pct(s['decisive_hits'], s['decisive_n'])}")

    print("\nidentical-answer probe (48 pairs) and padding probe (48 pairs)")
    print(f"{'judge':24s} {'ident: 1 call names a winner':>29s} {'ident: both orders':>19s} "
          f"{'pad: prefers padded':>20s} {'pad: prefers original':>22s} {'pad: agrees w/ gold':>20s}")
    for key, label, _ in JUDGES:
        s = summary[key]
        print(f"{label:24s} {100 * s['ident_single_winner']:28.1f}% {100 * s['ident_cons_winner']:18.1f}% "
              f"{100 * s['pad_pref']:19.1f}% {100 * s['pad_orig']:21.1f}% {100 * s['pad_acc']:19.1f}%")
    pad_gold = Counter(r["gold"] for r in results["longer"] if r["probe"] == "padding")
    print(f"(padding gold: tie {pad_gold['tie']}, original better {pad_gold['x']} -- the padded "
          f"exact readbacks fail their items)")

    print("\nscore gap vs position conflict, F1 1-3 with ties to the first answer (main pairs)")
    gaps = Counter()
    conflicts = Counter()
    for r in results["f1_3_first"]:
        if r["probe"] != "main":
            continue
        gap = abs(scale_score(r["x"], r["reference"], 3) - scale_score(r["y"], r["reference"], 3))
        gaps[gap] += 1
        conflicts[gap] += r["pos"] != "consistent"
    for gap in sorted(gaps):
        print(f"  gap {gap}: {gaps[gap]:2d} pairs, {conflicts[gap]:2d} change winner when swapped")

    print("\nscale sweep, tie to first answer (main pairs): ties appear wherever two scores round into the "
          "same bin, so the count is not monotone in scale size")
    print(f"  {'scale':>6s} {'equal scores':>13s} {'consistent':>15s} {'1 call, A first':>16s} "
          f"{'1 call, B first':>16s} {'both orders':>12s}   (decisive pairs right, of 18)")
    sweep = []
    main_pairs = [p for p in pairs if p["probe"] == "main"]
    for pts in SWEEP_POINTS:
        judge = make_f1_judge(pts, True)
        eq = cons_n = single_hit = single_hit_y = both_hit = 0
        for p in main_pairs:
            eq += scale_score(p["x"], p["reference"], pts) == scale_score(p["y"], p["reference"], pts)
            v1 = parse_verdict(judge(build_prompt(p["question"], p["x"], p["y"], p["reference"])))
            v2 = parse_verdict(judge(build_prompt(p["question"], p["y"], p["x"], p["reference"])))
            m1 = {"A": "x", "B": "y", "C": "tie"}[v1]
            m2 = {"A": "y", "B": "x", "C": "tie"}[v2]
            cons = m1 if m1 == m2 else "tie"
            cons_n += position_class(v1, v2) == "consistent"
            if p["gold"] != "tie":
                single_hit += m1 == p["gold"]
                single_hit_y += m2 == p["gold"]
                both_hit += cons == p["gold"]
        sweep.append((pts, eq, cons_n, single_hit, single_hit_y, both_hit))
        print(f"  {'1-' + str(pts):>6s} {eq:13d} {pct(cons_n, 48):>15s} {single_hit:>16d} "
              f"{single_hit_y:>16d} {both_hit:>12d}")

    print("\nwhere the golden verdict and each judge disagree (main pairs, both-orders verdict)")
    for key, label, _ in JUDGES:
        wrong = Counter((r["intent"], r["gold"], r["cons"]) for r in results[key]
                        if r["probe"] == "main" and r["cons"] != r["gold"])
        print(f"  {label}: " + "; ".join(f"{i} gold={g} judge={c} x{k}" for (i, g, c), k in sorted(wrong.items())))

    names = ["gold"] + [k for k, _, _ in JUDGES]
    labels = {"gold": "golden verdict", **{k: lab for k, lab, _ in JUDGES}}
    verdicts = {"gold": [r["gold"] for r in results["longer"] if r["probe"] == "main"]}
    for key, _, _ in JUDGES:
        verdicts[key] = [r["cons"] for r in results[key] if r["probe"] == "main"]
    print("\nCohen's kappa between raters (main pairs, both-orders verdicts; raw agreement in brackets)")
    print(" " * 24 + "".join(f"{labels[n][:14]:>16s}" for n in names))
    kmat = []
    for a in names:
        row = []
        cells = []
        for b in names:
            k = cohen_kappa(verdicts[a], verdicts[b])
            agree = sum(p == q for p, q in zip(verdicts[a], verdicts[b])) / len(verdicts[a])
            row.append(k)
            cells.append(f"{fmt(k)} [{100 * agree:.0f}%]")
        kmat.append(row)
        print(f"{labels[a]:24s}" + "".join(f"{c:>16s}" for c in cells))
    try:
        from sklearn.metrics import cohen_kappa_score
        worst = max(abs(cohen_kappa(verdicts[a], verdicts[b]) - cohen_kappa_score(verdicts[a], verdicts[b]))
                    for a in names for b in names if a != b and not math.isnan(cohen_kappa(verdicts[a], verdicts[b])))
        print(f"(scikit-learn cross-check: largest difference {worst:.2e})")
    except ImportError:
        print("(scikit-learn not installed; kappa cross-check skipped)")

    os.makedirs(os.path.dirname(CSV_OUT), exist_ok=True)
    with open(CSV_OUT, "w", newline="", encoding="utf-8") as fh:
        w = csv.writer(fh)
        w.writerow(["probe", "item_id", "intent", "judge", "order", "verdict", "maps_to",
                    "both_orders", "position", "gold", "x_words", "y_words", "judge_output"])
        w.writerows(rows)
    print(f"\nwrote {CSV_OUT} ({len(rows)} rows)")
    chart(summary, names, labels, kmat)


def chart(summary: dict, names: list[str], labels: dict, kmat: list[list[float]]) -> None:
    try:
        import matplotlib
        matplotlib.use("Agg")
        import matplotlib.pyplot as plt
        from matplotlib.colors import LinearSegmentedColormap
    except ImportError:
        print("matplotlib not installed; chart skipped")
        return
    surface, ink, ink2, muted, grid, axis = "#fcfcfb", "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
    series = [("position consistency (swap test)", "consistency", "#2a78d6"),
              ("right on the 18 decisive pairs", "decisive_acc", "#eb6834"),
              ("prefers the padded copy", "pad_pref", "#1baf7a")]
    fig = plt.figure(figsize=(11.6, 5.6), dpi=130, facecolor=surface)
    gs = fig.add_gridspec(1, 2, width_ratios=[1.3, 1], wspace=0.42)
    ax1, ax2 = fig.add_subplot(gs[0]), fig.add_subplot(gs[1])

    judges = names[1:]
    h, gap = 0.2, 0.04
    for s_i, (label, key, color) in enumerate(series):
        ys = [j + (s_i - 1) * (h + gap) for j in range(len(judges))]
        vals = [100 * summary[k][key] for k in judges]
        ax1.barh(ys, vals, height=h, color=color, label=label, zorder=3)
        for y, v in zip(ys, vals):
            ax1.text(v + 1.5, y, f"{v:.0f}%", va="center", fontsize=7.5, color=ink2, zorder=4)
    ax1.set_facecolor(surface)
    ax1.tick_params(colors=muted, labelsize=8.5, length=0)
    ax1.set_yticks(range(len(judges)))
    ax1.set_yticklabels([labels[k] for k in judges], fontsize=9)
    ax1.tick_params(axis="y", labelcolor=ink)
    ax1.invert_yaxis()
    ax1.set_xlim(0, 115)
    ax1.set_xticks([0, 25, 50, 75, 100])
    ax1.set_xticklabels(["0%", "25%", "50%", "75%", "100%"])
    ax1.xaxis.grid(True, color=grid, linewidth=0.8, zorder=0)
    for side in ("top", "right", "left"):
        ax1.spines[side].set_visible(False)
    ax1.spines["bottom"].set_color(axis)
    ax1.set_title(f"{len(judges)} local judges, 48 pairs, both orders", loc="left", fontsize=11,
                  color=ink, fontweight="bold")
    ax1.legend(loc="upper left", bbox_to_anchor=(0.0, -0.08), ncol=1, fontsize=8,
               frameon=False, labelcolor=ink2, handlelength=1.2)

    cmap = LinearSegmentedColormap.from_list("kappa", ["#e34948", "#f0efec", "#2a78d6"])
    cmap.set_bad(surface)
    import numpy as np
    arr = np.ma.masked_invalid(np.array(kmat, dtype=float))
    im = ax2.imshow(arr, cmap=cmap, vmin=-1, vmax=1)
    short = {"gold": "golden", "longer": "longer", "f1_10_first": "F1 1-10, 1st",
             "f1_3_first": "F1 1-3, 1st", "f1_3_tie": "F1 1-3, tie", "rubric": "rubric"}
    ax2.set_xticks(range(len(names)))
    ax2.set_xticklabels([short[n] for n in names], rotation=35, ha="right", fontsize=8.5, color=ink)
    ax2.set_yticks(range(len(names)))
    ax2.set_yticklabels([short[n] for n in names], fontsize=8.5, color=ink)
    ax2.tick_params(length=0)
    for side in ax2.spines.values():
        side.set_visible(False)
    ax2.set_xticks(np.arange(-0.5, len(names), 1), minor=True)
    ax2.set_yticks(np.arange(-0.5, len(names), 1), minor=True)
    ax2.grid(which="minor", color=surface, linewidth=2)
    ax2.tick_params(which="minor", length=0)
    for i in range(len(names)):
        for j in range(len(names)):
            k = kmat[i][j]
            if math.isnan(k):
                ax2.text(j, i, "n/a", ha="center", va="center", fontsize=8, color=muted)
                continue
            r, g, b, _ = cmap((k + 1) / 2)
            lum = 0.2126 * r + 0.7152 * g + 0.0722 * b
            ax2.text(j, i, f"{k:.2f}", ha="center", va="center", fontsize=8,
                     color=ink if lum > 0.5 else "#ffffff")
    ax2.set_title("Cohen's kappa between raters", loc="left", fontsize=11, color=ink, fontweight="bold")
    cb = fig.colorbar(im, ax=ax2, fraction=0.046, pad=0.04, ticks=[-1, -0.5, 0, 0.5, 1])
    cb.outline.set_visible(False)
    cb.ax.tick_params(colors=muted, labelsize=8, length=0)
    fig.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.2)
    os.makedirs(os.path.dirname(PNG_OUT), exist_ok=True)
    fig.savefig(PNG_OUT, facecolor=surface)
    print(f"wrote {PNG_OUT} ({os.path.getsize(PNG_OUT) // 1024} KB)")


if __name__ == "__main__":
    main()