esh-pfi-infrastructure/tools/judge-bench/items.py

"""Judge-benchmark items with DESIGNED ground truth.

Two modes, because that is how a judge actually gets used:
  PAIRWISE  — given two responses, which is better (A / B / tie)
  ABSOLUTE  — score one response 1-5 against a rubric

Ground truth is designed to be defensible without appeal to taste: the
"worse" response contains a checkable defect (false fact, violated explicit
constraint, answers a different question, invented citation, arithmetic
error). Ties are pairs that are both correct and equivalent.

POSITION BALANCE IS DELIBERATE. 5 items where A is better, 5 where B is,
2 genuine ties. A judge that just prefers whichever came first will score
~50% here and its bias will show in the per-label breakdown — position bias
is a known LLM-judge failure mode and a wash on accuracy can still hide it.
"""

PAIRWISE = [
    # --- A is better -------------------------------------------------
    dict(id="p01-fact", truth="A",
         q="In what year did the Berlin Wall fall?",
         a="The Berlin Wall fell in 1989.",
         b="The Berlin Wall fell in 1991.",
         defect="B states a false year (1989 is correct)."),
    dict(id="p02-constraint", truth="A",
         q="Answer in exactly one sentence: why is the sky blue?",
         a="Sunlight scatters off air molecules, and shorter blue wavelengths scatter most, so the sky looks blue.",
         b="Great question! The sky is blue because of scattering. Specifically, Rayleigh scattering. "
           "Shorter wavelengths scatter more than longer ones. That is why we see blue overhead and red at sunset.",
         defect="B violates the explicit one-sentence constraint (five sentences)."),
    dict(id="p03-relevance", truth="A",
         q="How do I rotate a Postgres log file without restarting the server?",
         a="Use logrotate with `copytruncate`, or call pg_rotate_logfile() which signals the logging collector to start a new file.",
         b="To back up Postgres, use pg_dump for logical backups or pg_basebackup for physical ones, then verify with pg_restore --list.",
         defect="B answers a different question (backups, not log rotation)."),
    dict(id="p04-invented-cite", truth="A",
         q="Is there evidence that spaced repetition improves long-term retention?",
         a="Yes — the spacing effect is well replicated in memory research going back to Ebbinghaus, and modern reviews find spaced practice beats massed practice for durable retention.",
         b="Yes — Hollingsworth & Pryor (2019), 'Spaced Retrieval in Adult Learners', Journal of Cognitive Methods 44(3), pp. 211-238, found a 340% retention gain.",
         defect="B fabricates a specific citation, journal, volume and effect size."),
    dict(id="p05-incomplete", truth="A",
         q="Name three causes of high memory usage in a Python process.",
         a="Unbounded caches or dicts that are never evicted; reference cycles holding large objects; and loading a whole file or query result into memory instead of streaming it.",
         b="One common cause is a memory leak.",
         defect="B gives one vague cause where three were requested."),

    # --- B is better (position flipped) ------------------------------
    dict(id="p06-arith", truth="B",
         q="A server has 96 GB of VRAM. Three models use 46 GB, 17 GB and 9 GB. How much is free?",
         a="46 + 17 + 9 = 62 GB used, so 34 GB is free.",
         b="46 + 17 + 9 = 72 GB used, so 24 GB is free.",
         defect="A's arithmetic is wrong (46+17+9 = 72, not 62)."),
    dict(id="p07-vague", truth="B",
         q="What does the `--delete` flag do in rsync?",
         a="It deletes things you do not need any more, which helps keep the destination tidy.",
         b="It removes files from the destination that are not present in the source, making the destination mirror the source rather than accumulate.",
         defect="A is vague and omits the defining behaviour; B is precise."),
    dict(id="p08-constraint2", truth="B",
         q="Reply with only a JSON object containing the key \"status\".",
         a="Sure! Here is the JSON you asked for:\n\n{\"status\": \"ok\"}\n\nLet me know if you need anything else.",
         b="{\"status\": \"ok\"}",
         defect="A violates 'only a JSON object' by wrapping it in prose."),
    dict(id="p09-fact2", truth="B",
         q="Which HTTP status code means 'Too Many Requests'?",
         a="HTTP 503 means Too Many Requests.",
         b="HTTP 429 means Too Many Requests.",
         defect="A is wrong; 503 is Service Unavailable, 429 is Too Many Requests."),
    dict(id="p10-hallu2", truth="B",
         q="Does Python's dict preserve insertion order?",
         a="No — dicts are unordered by definition; you must use collections.OrderedDict if order matters.",
         b="Yes — insertion order has been an implementation detail since CPython 3.6 and a language guarantee since 3.7.",
         defect="A states the pre-3.7 behaviour as current fact."),

    # --- genuine ties ------------------------------------------------
    dict(id="p11-tie", truth="tie",
         q="What is the capital of Japan?",
         a="Tokyo.",
         b="The capital of Japan is Tokyo.",
         defect="Both correct and equivalent; phrasing differs only."),
    dict(id="p12-tie", truth="tie",
         q="Give one reason to use a context manager for file handles.",
         a="It closes the file automatically even if an exception is raised.",
         b="The file is guaranteed to be closed on exit from the block, including on error paths.",
         defect="Both correct, equivalent content, different wording."),
]

ABSOLUTE = [
    # --- should score LOW (1-2) --------------------------------------
    dict(id="a01-low", band=(1, 2),
         q="How do I check which process is listening on port 8080 on Linux?",
         r="You should check the port. There are tools for this. Look at your system documentation.",
         why="Evasive, gives no actionable method."),
    dict(id="a02-low", band=(1, 2),
         q="What does `chmod 600 file` do?",
         r="It makes the file executable by everyone and readable by the group.",
         why="Factually wrong on both counts."),
    dict(id="a03-low", band=(1, 2),
         q="Summarise the purpose of a load balancer in one sentence.",
         r="Load balancers are very important in modern infrastructure and have been used for many years by "
           "many companies. There are hardware ones and software ones. Nginx is popular. So is HAProxy. "
           "Cloud providers offer managed ones too.",
         why="Ignores the one-sentence constraint and never states the purpose."),
    dict(id="a04-low", band=(1, 2),
         q="Is 17 a prime number?",
         r="No, 17 is not prime because it is divisible by 3.",
         why="Wrong answer with a wrong justification."),

    # --- should score HIGH (4-5) -------------------------------------
    dict(id="a05-high", band=(4, 5),
         q="How do I check which process is listening on port 8080 on Linux?",
         r="Use `ss -ltnp | grep :8080` (or `sudo lsof -i :8080`). Both show the listening socket and the owning "
           "PID/process; `ss` needs root to reveal the process name for sockets you do not own.",
         why="Accurate, actionable, includes the privilege caveat."),
    dict(id="a06-high", band=(4, 5),
         q="What does `chmod 600 file` do?",
         r="It sets read and write for the owner and removes all permissions for group and others, so only the "
           "file's owner (and root) can read or modify it.",
         why="Precisely correct."),
    dict(id="a07-high", band=(4, 5),
         q="Summarise the purpose of a load balancer in one sentence.",
         r="A load balancer distributes incoming requests across multiple backend servers so that no single server "
           "is overwhelmed and the service survives individual backend failures.",
         why="Correct, complete, obeys the one-sentence constraint."),
    dict(id="a08-high", band=(4, 5),
         q="Is 17 a prime number?",
         r="Yes. 17 has no divisors other than 1 and itself — it is not divisible by 2, 3, or any integer up to its square root (~4.1).",
         why="Correct with valid reasoning."),

    # --- should score MIDDLE (3) -------------------------------------
    dict(id="a09-mid", band=(2, 4),
         q="How do I check which process is listening on port 8080 on Linux?",
         r="Use netstat.",
         why="Correct direction, no flags, no explanation, deprecated tool."),
    dict(id="a10-mid", band=(2, 4),
         q="What does `chmod 600 file` do?",
         r="It restricts the file so other users cannot read it.",
         why="True but incomplete — omits owner rw and the group dimension."),
    dict(id="a11-mid", band=(2, 4),
         q="Name two advantages of connection pooling.",
         r="It is faster.",
         why="Partially responsive: one vague advantage where two were asked."),
    dict(id="a12-mid", band=(2, 4),
         q="Is 17 a prime number?",
         r="Yes.",
         why="Correct but bare — no justification for a question inviting one."),
]