vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
475d6d6bcb3c22280f3c4d4dc03c6cb3d369e969
esh-pfi-infrastructure/services/semif-serve/bench-jev-2026-09-30/code/data/authored144.jsonl
T
vh 475d6d6bcb docs: Jev candidate bench vs SemIf (fv-ml1 GPU 3) — Intern-Decision-4B is the replacement if SemIf is displaced 
Operator ask relayed by brokkr-smithy-dev. Positive control (SemIf 187/231,
hard 0.613) reproduced exactly; negative control and a 4-restart noise floor
(0 flips) measured. On our replaced-baseline sets no candidate beats SemIf-with-
rotations beyond the ~4-pt floor; Intern-Decision-4B native matches it at one
ordering, is better on Wyrd, fits 9.7/10.3 GB and is 1.5-2.3x faster. JevBench
rank does not transfer. Raw per-item data kept out of git.
2026-09-30 05:00:45 -07:00

145 lines
189 KiB
JSON
Raw Blame History

{"family": "evidence_interpretation", "group_id": "2fdaa8a61e6e989dd866", "id": "a3f18f3a63d45345942b", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The optician ordered replacement lenses. The workshop confirms they have not yet been fitted to the customer's glasses.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "Fitting is explicitly not complete.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e01", "variant": "original"}, "question": "Assess the claim: the replacement lenses have been fitted.", "split": "test", "state": "The optician ordered replacement lenses. The workshop confirms they have not yet been fitted to the customer's glasses.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "2fdaa8a61e6e989dd866", "id": "f40beba9088c8db8bbd6", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The optician ordered replacement lenses. The workshop confirms they have not yet been fitted to the customer's glasses.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "Ordering is explicitly complete.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e01", "variant": "criterion_reversal"}, "question": "Assess the claim: replacement lenses were ordered.", "split": "test", "state": "The optician ordered replacement lenses. The workshop confirms they have not yet been fitted to the customer's glasses.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "2fdaa8a61e6e989dd866", "id": "0b43ea8e24e74d621f1b", "label": 1, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The workshop confirms it fitted the replacement lenses to the customer's glasses.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "Fitting is now explicitly complete.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e01", "variant": "evidence_change"}, "question": "Assess the claim: the replacement lenses have been fitted.", "split": "test", "state": "The workshop confirms it fitted the replacement lenses to the customer's glasses.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "2fdaa8a61e6e989dd866", "id": "942ec65dac0bfb9874f9", "label": 0, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The optician records a glasses prescription but no order or fitting status.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "Prescription details settle no fitting status.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e01", "variant": "missing"}, "question": "Assess the claim: the replacement lenses have been fitted.", "split": "test", "state": "The optician records a glasses prescription but no order or fitting status.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f4bea3d3bcc01056740d", "id": "e0c140e9222d1bce5647", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guest writes, \"The listing says towels are supplied. I agree: fresh towels were provided during my stay.\"", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "The guest explicitly agrees.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e02", "variant": "original"}, "question": "Assess the claim: the guest endorses the listing's towel statement.", "split": "test", "state": "The guest writes, \"The listing says towels are supplied. I agree: fresh towels were provided during my stay.\"", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f4bea3d3bcc01056740d", "id": "8e8c3804a3c15ebb31e3", "label": 2, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guest writes, \"The listing says towels are supplied. I agree: fresh towels were provided during my stay.\"", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "Agreement contradicts disputing the statement.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e02", "variant": "criterion_reversal"}, "question": "Assess the claim: the guest disputes the towel statement.", "split": "test", "state": "The guest writes, \"The listing says towels are supplied. I agree: fresh towels were provided during my stay.\"", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f4bea3d3bcc01056740d", "id": "56c419b12cd37cbe790c", "label": 0, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guest writes, \"The listing says towels are supplied, but none were provided. I dispute that statement.\"", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "The guest now explicitly disputes it.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e02", "variant": "evidence_change"}, "question": "Assess the claim: the guest endorses the listing's towel statement.", "split": "test", "state": "The guest writes, \"The listing says towels are supplied, but none were provided. I dispute that statement.\"", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f4bea3d3bcc01056740d", "id": "75b470586834df8c568d", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guest forwards the listing but provides no comment about towels or the stay.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "Forwarding does not settle endorsement.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e02", "variant": "missing"}, "question": "Assess the claim: the guest endorses the listing's towel statement.", "split": "test", "state": "The guest forwards the listing but provides no comment about towels or the stay.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "b51b300da2936decc6b7", "id": "834f3b3e10cc61b33d25", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The technician says terminal Cedar is online. Terminal Maple is explicitly offline.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_identity_scope", "partition": "familiar_mechanism_new_situation", "rationale": "Maple is explicitly offline.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e03", "variant": "original"}, "question": "Assess the claim: terminal Maple is online.", "split": "test", "state": "The technician says terminal Cedar is online. Terminal Maple is explicitly offline.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "b51b300da2936decc6b7", "id": "76c7a79b1ff2363a60c5", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The technician says terminal Cedar is online. Terminal Maple is explicitly offline.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_identity_scope", "partition": "familiar_mechanism_new_situation", "rationale": "Cedar is explicitly online.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e03", "variant": "criterion_reversal"}, "question": "Assess the claim: terminal Cedar is online.", "split": "test", "state": "The technician says terminal Cedar is online. Terminal Maple is explicitly offline.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "b51b300da2936decc6b7", "id": "36c71bb649b7d5eccb1f", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The technician confirms terminal Maple is online and terminal Cedar is offline.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_identity_scope", "partition": "familiar_mechanism_new_situation", "rationale": "Maple is now explicitly online.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e03", "variant": "evidence_change"}, "question": "Assess the claim: terminal Maple is online.", "split": "test", "state": "The technician confirms terminal Maple is online and terminal Cedar is offline.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "b51b300da2936decc6b7", "id": "eac5121a4a1bf391a08c", "label": 1, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The technician lists both terminals but omits their connection status.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_identity_scope", "partition": "familiar_mechanism_new_situation", "rationale": "Connection status is absent.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e03", "variant": "missing"}, "question": "Assess the claim: terminal Maple is online.", "split": "test", "state": "The technician lists both terminals but omits their connection status.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e4cfd694c80e08d3e6a5", "id": "44574c44e243ee1b2f52", "label": 2, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The catalogue originally dated the letter to June. Its author issued a correction stating July was correct and June was mistaken.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_correction", "partition": "familiar_mechanism_new_situation", "rationale": "The correction establishes July.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e04", "variant": "original"}, "question": "Assess the claim: the corrected date is in July.", "split": "test", "state": "The catalogue originally dated the letter to June. Its author issued a correction stating July was correct and June was mistaken.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e4cfd694c80e08d3e6a5", "id": "2608370d01783735c3ea", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The catalogue originally dated the letter to June. Its author issued a correction stating July was correct and June was mistaken.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_correction", "partition": "familiar_mechanism_new_situation", "rationale": "The correction rejects June.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e04", "variant": "criterion_reversal"}, "question": "Assess the claim: the corrected date is in June.", "split": "test", "state": "The catalogue originally dated the letter to June. Its author issued a correction stating July was correct and June was mistaken.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e4cfd694c80e08d3e6a5", "id": "ec94ef909f7918695ab3", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The catalogue author corrects the date to June and expressly rejects the earlier July date.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_correction", "partition": "familiar_mechanism_new_situation", "rationale": "The correction now rejects July.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e04", "variant": "evidence_change"}, "question": "Assess the claim: the corrected date is in July.", "split": "test", "state": "The catalogue author corrects the date to June and expressly rejects the earlier July date.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e4cfd694c80e08d3e6a5", "id": "03d919fd0e9b3a7cd1e6", "label": 0, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The catalogue mentions a date correction but gives no corrected date.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_correction", "partition": "familiar_mechanism_new_situation", "rationale": "The corrected date is missing.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e04", "variant": "missing"}, "question": "Assess the claim: the corrected date is in July.", "split": "test", "state": "The catalogue mentions a date correction but gives no corrected date.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "20460d4ae0db46091e84", "id": "86a15618ba4cb90a307e", "label": 0, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The intern attached instructions for weighing the parcel and explicitly says no weighing has been performed.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "The intern explicitly denies performed weighing.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e05", "variant": "original"}, "question": "Assess the claim: the parcel has been weighed.", "split": "test", "state": "The intern attached instructions for weighing the parcel and explicitly says no weighing has been performed.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "20460d4ae0db46091e84", "id": "3dd9c3ac98f8ddf7bdd3", "label": 1, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The intern attached instructions for weighing the parcel and explicitly says no weighing has been performed.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "Instructions are explicitly attached.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e05", "variant": "criterion_reversal"}, "question": "Assess the claim: the document contains weighing instructions.", "split": "test", "state": "The intern attached instructions for weighing the parcel and explicitly says no weighing has been performed.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "20460d4ae0db46091e84", "id": "36273dd4777876a26338", "label": 1, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The intern performed the weighing and recorded the parcel's measured weight.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "Performed weighing is now explicit.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e05", "variant": "evidence_change"}, "question": "Assess the claim: the parcel has been weighed.", "split": "test", "state": "The intern performed the weighing and recorded the parcel's measured weight.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "20460d4ae0db46091e84", "id": "07ee70cfe4b01330ce89", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The intern supplies a parcel identifier but no procedure or measurement information.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "No measurement information is supplied.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e05", "variant": "missing"}, "question": "Assess the claim: the parcel has been weighed.", "split": "test", "state": "The intern supplies a parcel identifier but no procedure or measurement information.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "51b91ba31984c5939c1c", "id": "51ceab3ce622c7c759f1", "label": 2, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "During a puppet show, a character announces the discovery of a dragon egg. The production note states this is fictional and no real egg was discovered.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_fiction", "partition": "familiar_mechanism_new_situation", "rationale": "The production note explicitly identifies fiction.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e06", "variant": "original"}, "question": "Assess the claim: the egg discovery was fictional.", "split": "test", "state": "During a puppet show, a character announces the discovery of a dragon egg. The production note states this is fictional and no real egg was discovered.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "51b91ba31984c5939c1c", "id": "b8091be929b1e489fcc4", "label": 1, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "During a puppet show, a character announces the discovery of a dragon egg. The production note states this is fictional and no real egg was discovered.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_fiction", "partition": "familiar_mechanism_new_situation", "rationale": "It explicitly denies a real discovery.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e06", "variant": "criterion_reversal"}, "question": "Assess the claim: a real egg was discovered.", "split": "test", "state": "During a puppet show, a character announces the discovery of a dragon egg. The production note states this is fictional and no real egg was discovered.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "51b91ba31984c5939c1c", "id": "c89b86e3f55123bfa506", "label": 1, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The note describes a real bird egg discovered outside a performance and expressly says the discovery was not fictional.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_fiction", "partition": "familiar_mechanism_new_situation", "rationale": "The changed account expressly identifies a real discovery.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e06", "variant": "evidence_change"}, "question": "Assess the claim: the egg discovery was fictional.", "split": "test", "state": "The note describes a real bird egg discovered outside a performance and expressly says the discovery was not fictional.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "51b91ba31984c5939c1c", "id": "3b0958cabc339005cf66", "label": 0, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "A note mentions an egg without describing a discovery or whether the account is fictional.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_fiction", "partition": "familiar_mechanism_new_situation", "rationale": "The account type is absent.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_e06", "variant": "missing"}, "question": "Assess the claim: the egg discovery was fictional.", "split": "test", "state": "A note mentions an egg without describing a discovery or whether the account is fictional.", "target_distribution": null}
{"family": "rule_application", "group_id": "770c1607f0a11f88f81e", "id": "f2b4ec4930322fe45118", "label": 1, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The architect permits reproducing a plan in an internal training booklet and expressly forbids selling copies.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_permission", "partition": "familiar_mechanism_new_situation", "rationale": "Sales are expressly forbidden.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r01", "variant": "original"}, "question": "Sell copies only with the architect's sales permission. May copies be sold?", "split": "test", "state": "The architect permits reproducing a plan in an internal training booklet and expressly forbids selling copies.", "target_distribution": null}
{"family": "rule_application", "group_id": "770c1607f0a11f88f81e", "id": "61ac20b07a5189231e15", "label": 1, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The architect permits reproducing a plan in an internal training booklet and expressly forbids selling copies.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_permission", "partition": "familiar_mechanism_new_situation", "rationale": "Internal training reproduction is allowed.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r01", "variant": "criterion_reversal"}, "question": "Internal training reproduction is allowed when the architect permits that use. May the plan appear in the training booklet?", "split": "test", "state": "The architect permits reproducing a plan in an internal training booklet and expressly forbids selling copies.", "target_distribution": null}
{"family": "rule_application", "group_id": "770c1607f0a11f88f81e", "id": "4b3d794c004566610181", "label": 2, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The architect expressly permits selling copies of the plan.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_permission", "partition": "familiar_mechanism_new_situation", "rationale": "Sales are now allowed.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r01", "variant": "evidence_change"}, "question": "Sell copies only with the architect's sales permission. May copies be sold?", "split": "test", "state": "The architect expressly permits selling copies of the plan.", "target_distribution": null}
{"family": "rule_application", "group_id": "770c1607f0a11f88f81e", "id": "b7c4e62d07986286fce0", "label": 0, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "An architect's permission letter exists, but the uses it allows are omitted.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_permission", "partition": "familiar_mechanism_new_situation", "rationale": "The scope of permission is missing.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r01", "variant": "missing"}, "question": "Sell copies only with the architect's sales permission. May copies be sold?", "split": "test", "state": "An architect's permission letter exists, but the uses it allows are omitted.", "target_distribution": null}
{"family": "rule_application", "group_id": "ffd081ada3bd9be9207a", "id": "9473603b44af63beab79", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guide completed first-aid training but not mountain-navigation training.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_required_qualification", "partition": "familiar_mechanism_new_situation", "rationale": "The sole original condition is satisfied.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r02", "variant": "original"}, "question": "The first-aid badge requires only first-aid training. May this guide receive it?", "split": "test", "state": "The guide completed first-aid training but not mountain-navigation training.", "target_distribution": null}
{"family": "rule_application", "group_id": "ffd081ada3bd9be9207a", "id": "d41072dc35d8cd14c056", "label": 1, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guide completed first-aid training but not mountain-navigation training.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_required_qualification", "partition": "familiar_mechanism_new_situation", "rationale": "The added navigation condition is not satisfied.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r02", "variant": "criterion_reversal"}, "question": "Under the alternative rule, the badge requires both first-aid and mountain-navigation training. May this guide receive it?", "split": "test", "state": "The guide completed first-aid training but not mountain-navigation training.", "target_distribution": null}
{"family": "rule_application", "group_id": "ffd081ada3bd9be9207a", "id": "33d6e5da58f0fee2490d", "label": 0, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guide completed mountain-navigation training but not first-aid training.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_required_qualification", "partition": "familiar_mechanism_new_situation", "rationale": "The original first-aid condition is now absent.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r02", "variant": "evidence_change"}, "question": "The first-aid badge requires only first-aid training. May this guide receive it?", "split": "test", "state": "The guide completed mountain-navigation training but not first-aid training.", "target_distribution": null}
{"family": "rule_application", "group_id": "ffd081ada3bd9be9207a", "id": "823bf736b92e9d8fa66d", "label": 1, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The guide registered for training, but completion information is not supplied.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_required_qualification", "partition": "familiar_mechanism_new_situation", "rationale": "Registration does not establish completion.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r02", "variant": "missing"}, "question": "The first-aid badge requires only first-aid training. May this guide receive it?", "split": "test", "state": "The guide registered for training, but completion information is not supplied.", "target_distribution": null}
{"family": "rule_application", "group_id": "2c3a0cb61f7bae78739e", "id": "8a4c3de28be95c105ca1", "label": 2, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The designer approves making a sample badge but expressly withholds approval for mass production.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_approval_stage", "partition": "familiar_mechanism_new_situation", "rationale": "Mass-production approval is withheld.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r03", "variant": "original"}, "question": "Mass production is allowed only after mass-production approval. May it begin?", "split": "test", "state": "The designer approves making a sample badge but expressly withholds approval for mass production.", "target_distribution": null}
{"family": "rule_application", "group_id": "2c3a0cb61f7bae78739e", "id": "ea7ec66a5e8eac886548", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The designer approves making a sample badge but expressly withholds approval for mass production.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_approval_stage", "partition": "familiar_mechanism_new_situation", "rationale": "Sample production is approved.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r03", "variant": "criterion_reversal"}, "question": "A sample may be made when sample production is approved. May the sample be made?", "split": "test", "state": "The designer approves making a sample badge but expressly withholds approval for mass production.", "target_distribution": null}
{"family": "rule_application", "group_id": "2c3a0cb61f7bae78739e", "id": "25172cd6ddc49ab0ca70", "label": 2, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The designer explicitly approves mass production of the badge.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_approval_stage", "partition": "familiar_mechanism_new_situation", "rationale": "Mass-production approval is now present.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r03", "variant": "evidence_change"}, "question": "Mass production is allowed only after mass-production approval. May it begin?", "split": "test", "state": "The designer explicitly approves mass production of the badge.", "target_distribution": null}
{"family": "rule_application", "group_id": "2c3a0cb61f7bae78739e", "id": "fb73a85d386cf59b0c51", "label": 0, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The designer issued a decision but its production scope is unavailable.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_approval_stage", "partition": "familiar_mechanism_new_situation", "rationale": "The approved stage is missing.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r03", "variant": "missing"}, "question": "Mass production is allowed only after mass-production approval. May it begin?", "split": "test", "state": "The designer issued a decision but its production scope is unavailable.", "target_distribution": null}
{"family": "rule_application", "group_id": "42ed4cbfde7f2f64274b", "id": "a02fd582d2881cc462b2", "label": 2, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The visitor's wristband covers the sculpture garden and expressly excludes the rooftop terrace.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_named_privilege", "partition": "familiar_mechanism_new_situation", "rationale": "The garden is covered.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r04", "variant": "original"}, "question": "A wristband permits entry exactly to the areas it covers. May the visitor enter the sculpture garden?", "split": "test", "state": "The visitor's wristband covers the sculpture garden and expressly excludes the rooftop terrace.", "target_distribution": null}
{"family": "rule_application", "group_id": "42ed4cbfde7f2f64274b", "id": "19f37d5a97034bd7d04f", "label": 1, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The visitor's wristband covers the sculpture garden and expressly excludes the rooftop terrace.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_named_privilege", "partition": "familiar_mechanism_new_situation", "rationale": "The terrace is excluded.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r04", "variant": "criterion_reversal"}, "question": "A wristband permits entry exactly to the areas it covers. May the visitor enter the rooftop terrace?", "split": "test", "state": "The visitor's wristband covers the sculpture garden and expressly excludes the rooftop terrace.", "target_distribution": null}
{"family": "rule_application", "group_id": "42ed4cbfde7f2f64274b", "id": "fda0ca94652c6d739ecf", "label": 0, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The wristband expressly excludes the sculpture garden and covers only the rooftop terrace.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_named_privilege", "partition": "familiar_mechanism_new_situation", "rationale": "The garden is now excluded.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r04", "variant": "evidence_change"}, "question": "A wristband permits entry exactly to the areas it covers. May the visitor enter the sculpture garden?", "split": "test", "state": "The wristband expressly excludes the sculpture garden and covers only the rooftop terrace.", "target_distribution": null}
{"family": "rule_application", "group_id": "42ed4cbfde7f2f64274b", "id": "d0e36c4cb10a29b3191c", "label": 0, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The visitor has a wristband but its covered areas are not supplied.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_named_privilege", "partition": "familiar_mechanism_new_situation", "rationale": "Covered areas are unknown.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r04", "variant": "missing"}, "question": "A wristband permits entry exactly to the areas it covers. May the visitor enter the sculpture garden?", "split": "test", "state": "The visitor has a wristband but its covered areas are not supplied.", "target_distribution": null}
{"family": "rule_application", "group_id": "6dd193172b9240a04bf9", "id": "a0a39ace7e5337d038a7", "label": 0, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The account owner permits changing the display name but expressly forbids changing the recovery email.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_local_action", "partition": "familiar_mechanism_new_situation", "rationale": "Recovery-email change is expressly forbidden.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r05", "variant": "original"}, "question": "Change an account field only when the owner authorizes changing that field. May the recovery email be changed?", "split": "test", "state": "The account owner permits changing the display name but expressly forbids changing the recovery email.", "target_distribution": null}
{"family": "rule_application", "group_id": "6dd193172b9240a04bf9", "id": "eeae283841ce75dd25bf", "label": 1, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The account owner permits changing the display name but expressly forbids changing the recovery email.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_local_action", "partition": "familiar_mechanism_new_situation", "rationale": "Display-name change is authorized.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r05", "variant": "criterion_reversal"}, "question": "Change an account field only when the owner authorizes changing that field. May the display name be changed?", "split": "test", "state": "The account owner permits changing the display name but expressly forbids changing the recovery email.", "target_distribution": null}
{"family": "rule_application", "group_id": "6dd193172b9240a04bf9", "id": "00bb4cc043f7d2b288d4", "label": 2, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The owner expressly permits changing the recovery email.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_local_action", "partition": "familiar_mechanism_new_situation", "rationale": "Recovery-email change is now authorized.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r05", "variant": "evidence_change"}, "question": "Change an account field only when the owner authorizes changing that field. May the recovery email be changed?", "split": "test", "state": "The owner expressly permits changing the recovery email.", "target_distribution": null}
{"family": "rule_application", "group_id": "6dd193172b9240a04bf9", "id": "1a81ead69e93e5d0c86a", "label": 2, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The owner requested an account update, but the fields to be changed are omitted.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_local_action", "partition": "familiar_mechanism_new_situation", "rationale": "The authorized field is missing.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r05", "variant": "missing"}, "question": "Change an account field only when the owner authorizes changing that field. May the recovery email be changed?", "split": "test", "state": "The owner requested an account update, but the fields to be changed are omitted.", "target_distribution": null}
{"family": "rule_application", "group_id": "3200eb252d6737e03027", "id": "4444dad25e438f128673", "label": 1, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The applicant lives in the valley but works outside it.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_residence_employment", "partition": "familiar_mechanism_new_situation", "rationale": "Valley residence meets the original condition.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r06", "variant": "original"}, "question": "A resident's pass requires living in the valley and has no workplace restriction. May it be issued?", "split": "test", "state": "The applicant lives in the valley but works outside it.", "target_distribution": null}
{"family": "rule_application", "group_id": "3200eb252d6737e03027", "id": "3226202ba1ada2570e9f", "label": 0, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The applicant lives in the valley but works outside it.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_residence_employment", "partition": "familiar_mechanism_new_situation", "rationale": "The alternative workplace condition is not met.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r06", "variant": "criterion_reversal"}, "question": "Under the alternative rule, a pass requires working in the valley; residence alone is insufficient. May it be issued?", "split": "test", "state": "The applicant lives in the valley but works outside it.", "target_distribution": null}
{"family": "rule_application", "group_id": "3200eb252d6737e03027", "id": "3f783eec5c90812479bc", "label": 1, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The applicant lives outside the valley and works inside it.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_residence_employment", "partition": "familiar_mechanism_new_situation", "rationale": "Valley residence is now absent.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r06", "variant": "evidence_change"}, "question": "A resident's pass requires living in the valley and has no workplace restriction. May it be issued?", "split": "test", "state": "The applicant lives outside the valley and works inside it.", "target_distribution": null}
{"family": "rule_application", "group_id": "3200eb252d6737e03027", "id": "ffee861474a9665a093a", "label": 2, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The applicant supplies a name but no home or work location.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_residence_employment", "partition": "familiar_mechanism_new_situation", "rationale": "The relevant locations are unknown.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_r06", "variant": "missing"}, "question": "A resident's pass requires living in the valley and has no workplace restriction. May it be issued?", "split": "test", "state": "The applicant supplies a name but no home or work location.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c0cbeb97315307af8111", "id": "8c992e17e4fb54b044f5", "label": 1, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The florist confirms that the wreath was delivered. Candidate B: The florist offers to deliver a wreath tomorrow.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "A reports completed delivery.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c01", "variant": "original"}, "question": "Choose the candidate reporting completed delivery.", "split": "test", "state": "Candidate A: The florist confirms that the wreath was delivered. Candidate B: The florist offers to deliver a wreath tomorrow.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c0cbeb97315307af8111", "id": "695dd61195ca025c4f4c", "label": 0, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The florist confirms that the wreath was delivered. Candidate B: The florist offers to deliver a wreath tomorrow.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "B offers future action.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c01", "variant": "criterion_reversal"}, "question": "Choose the candidate offering a future delivery.", "split": "test", "state": "Candidate A: The florist confirms that the wreath was delivered. Candidate B: The florist offers to deliver a wreath tomorrow.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c0cbeb97315307af8111", "id": "e6155431ab7b0f8170d4", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: I can deliver a wreath tomorrow. Candidate B: The wreath was delivered.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "Completed delivery is now B.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c01", "variant": "evidence_change"}, "question": "Choose the candidate reporting completed delivery.", "split": "test", "state": "Candidate A: I can deliver a wreath tomorrow. Candidate B: The wreath was delivered.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c0cbeb97315307af8111", "id": "b49ecda75086d8c68d2a", "label": 2, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The florist gives an address. Candidate B: The wreath contains lilies.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_intent_outcome", "partition": "familiar_mechanism_new_situation", "rationale": "Neither provides a delivery report or offer.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c01", "variant": "missing"}, "question": "Choose the candidate reporting completed delivery.", "split": "test", "state": "Candidate A: The florist gives an address. Candidate B: The wreath contains lilies.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "428176cd8ad2c7a74eea", "id": "587ced261c6092cd1325", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The customer asks how an engraving would look but explicitly declines having it engraved yet. Candidate B: The customer instructs the shop to engrave the ring.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_consent", "partition": "familiar_mechanism_new_situation", "rationale": "B instructs engraving.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c02", "variant": "original"}, "question": "Choose the candidate authorizing engraving.", "split": "test", "state": "Candidate A: The customer asks how an engraving would look but explicitly declines having it engraved yet. Candidate B: The customer instructs the shop to engrave the ring.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "428176cd8ad2c7a74eea", "id": "533d4423d311de82b27d", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The customer asks how an engraving would look but explicitly declines having it engraved yet. Candidate B: The customer instructs the shop to engrave the ring.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_consent", "partition": "familiar_mechanism_new_situation", "rationale": "A expressly withholds authorization.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c02", "variant": "criterion_reversal"}, "question": "Choose the candidate asking only for information while withholding authorization.", "split": "test", "state": "Candidate A: The customer asks how an engraving would look but explicitly declines having it engraved yet. Candidate B: The customer instructs the shop to engrave the ring.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "428176cd8ad2c7a74eea", "id": "2bf540d515a51755b25c", "label": 2, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: Please engrave the ring. Candidate B: Show me how it would look, but do not engrave it yet.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_consent", "partition": "familiar_mechanism_new_situation", "rationale": "A now authorizes engraving.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c02", "variant": "evidence_change"}, "question": "Choose the candidate authorizing engraving.", "split": "test", "state": "Candidate A: Please engrave the ring. Candidate B: Show me how it would look, but do not engrave it yet.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "428176cd8ad2c7a74eea", "id": "fdb9fc214aea2710763a", "label": 2, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The customer gives a name. Candidate B: The ring is silver.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_scope_consent", "partition": "familiar_mechanism_new_situation", "rationale": "Neither addresses engraving permission.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c02", "variant": "missing"}, "question": "Choose the candidate authorizing engraving.", "split": "test", "state": "Candidate A: The customer gives a name. Candidate B: The ring is silver.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "389e65bf1ba709b84b12", "id": "f6d68126b43f8c9ed183", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The note records that the completed insulation test passed. Candidate B: The note lists insulation-test steps but says no test was run.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "A reports a completed result.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c03", "variant": "original"}, "question": "Choose the candidate reporting an observed test result.", "split": "test", "state": "Candidate A: The note records that the completed insulation test passed. Candidate B: The note lists insulation-test steps but says no test was run.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "389e65bf1ba709b84b12", "id": "44bae0c8a1615a096c43", "label": 0, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The note records that the completed insulation test passed. Candidate B: The note lists insulation-test steps but says no test was run.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "B supplies unexecuted steps.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c03", "variant": "criterion_reversal"}, "question": "Choose the candidate containing only an unexecuted test procedure.", "split": "test", "state": "Candidate A: The note records that the completed insulation test passed. Candidate B: The note lists insulation-test steps but says no test was run.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "389e65bf1ba709b84b12", "id": "2175379f4c69e6207626", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: Here are insulation-test steps; no test was run. Candidate B: The completed insulation test passed.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "The result is now B.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c03", "variant": "evidence_change"}, "question": "Choose the candidate reporting an observed test result.", "split": "test", "state": "Candidate A: Here are insulation-test steps; no test was run. Candidate B: The completed insulation test passed.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "389e65bf1ba709b84b12", "id": "def9fe59d262357a673a", "label": 0, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The cable is red. Candidate B: The tester is in a drawer.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_result_procedure", "partition": "familiar_mechanism_new_situation", "rationale": "Neither supplies a test result or procedure.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c03", "variant": "missing"}, "question": "Choose the candidate reporting an observed test result.", "split": "test", "state": "Candidate A: The cable is red. Candidate B: The tester is in a drawer.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "45f8ead2522a14384cdc", "id": "712f6ffb763eef7c9a12", "label": 0, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The editor quotes a proposal for a paywall only to argue against it. Candidate B: The editor endorses introducing a paywall.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "B endorses the policy.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c04", "variant": "original"}, "question": "Choose the candidate supporting a paywall.", "split": "test", "state": "Candidate A: The editor quotes a proposal for a paywall only to argue against it. Candidate B: The editor endorses introducing a paywall.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "45f8ead2522a14384cdc", "id": "e1d610bd14e3d16c09b9", "label": 0, "options": [{"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The editor quotes a proposal for a paywall only to argue against it. Candidate B: The editor endorses introducing a paywall.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "A argues against the quoted policy.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c04", "variant": "criterion_reversal"}, "question": "Choose the candidate opposing a paywall.", "split": "test", "state": "Candidate A: The editor quotes a proposal for a paywall only to argue against it. Candidate B: The editor endorses introducing a paywall.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "45f8ead2522a14384cdc", "id": "22ec57822db50ec121cf", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: I endorse introducing a paywall. Candidate B: I reject introducing a paywall.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "Support is now A.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c04", "variant": "evidence_change"}, "question": "Choose the candidate supporting a paywall.", "split": "test", "state": "Candidate A: I endorse introducing a paywall. Candidate B: I reject introducing a paywall.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "45f8ead2522a14384cdc", "id": "3e75f2b84e8e08e3d3f2", "label": 2, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The editor names the magazine. Candidate B: The website has a blue header.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_stance", "partition": "familiar_mechanism_new_situation", "rationale": "Neither supplies a position on paywalls.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c04", "variant": "missing"}, "question": "Choose the candidate supporting a paywall.", "split": "test", "state": "Candidate A: The editor names the magazine. Candidate B: The website has a blue header.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "3ecec8e2c166e397ad07", "id": "ebd74aa05324a1556288", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The owner asks to reseal the existing window while keeping its frame. Candidate B: The owner asks to remove the window and fit a complete replacement.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_retained_replaced", "partition": "familiar_mechanism_new_situation", "rationale": "A retains and repairs the existing window.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c05", "variant": "original"}, "question": "Choose the candidate asking to repair the existing window.", "split": "test", "state": "Candidate A: The owner asks to reseal the existing window while keeping its frame. Candidate B: The owner asks to remove the window and fit a complete replacement.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "3ecec8e2c166e397ad07", "id": "d7e84e4245a29817da1f", "label": 2, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The owner asks to reseal the existing window while keeping its frame. Candidate B: The owner asks to remove the window and fit a complete replacement.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_retained_replaced", "partition": "familiar_mechanism_new_situation", "rationale": "B requests replacement.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c05", "variant": "criterion_reversal"}, "question": "Choose the candidate asking for complete replacement.", "split": "test", "state": "Candidate A: The owner asks to reseal the existing window while keeping its frame. Candidate B: The owner asks to remove the window and fit a complete replacement.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "3ecec8e2c166e397ad07", "id": "f41ae66b3c09630c7a42", "label": 2, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: Remove the window and install a replacement. Candidate B: Reseal the existing window and keep its frame.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_retained_replaced", "partition": "familiar_mechanism_new_situation", "rationale": "Repair is now B.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c05", "variant": "evidence_change"}, "question": "Choose the candidate asking to repair the existing window.", "split": "test", "state": "Candidate A: Remove the window and install a replacement. Candidate B: Reseal the existing window and keep its frame.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "3ecec8e2c166e397ad07", "id": "82c2cd97c160ad27951b", "label": 0, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The window faces a courtyard. Candidate B: The owner gives a postcode.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_retained_replaced", "partition": "familiar_mechanism_new_situation", "rationale": "Neither supplies a window-work request.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c05", "variant": "missing"}, "question": "Choose the candidate asking to repair the existing window.", "split": "test", "state": "Candidate A: The window faces a courtyard. Candidate B: The owner gives a postcode.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a71a17a287f3d30db2aa", "id": "7ada97b6f99f4788e5c7", "label": 0, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The sponsor confirms receipt of a grant application but explicitly says no award decision has been made. Candidate B: The sponsor confirms that the grant was awarded.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_submission_acceptance", "partition": "familiar_mechanism_new_situation", "rationale": "B confirms an award.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c06", "variant": "original"}, "question": "Choose the candidate reporting an awarded grant.", "split": "test", "state": "Candidate A: The sponsor confirms receipt of a grant application but explicitly says no award decision has been made. Candidate B: The sponsor confirms that the grant was awarded.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a71a17a287f3d30db2aa", "id": "4d7a003342db5b0a17e3", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The sponsor confirms receipt of a grant application but explicitly says no award decision has been made. Candidate B: The sponsor confirms that the grant was awarded.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_submission_acceptance", "partition": "familiar_mechanism_new_situation", "rationale": "A expressly leaves the award undecided.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c06", "variant": "criterion_reversal"}, "question": "Choose the candidate reporting receipt with the award still undecided.", "split": "test", "state": "Candidate A: The sponsor confirms receipt of a grant application but explicitly says no award decision has been made. Candidate B: The sponsor confirms that the grant was awarded.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a71a17a287f3d30db2aa", "id": "19d3d8d0bf773e88be7b", "label": 0, "options": [{"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The grant was awarded. Candidate B: The application was received; the award remains undecided.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_submission_acceptance", "partition": "familiar_mechanism_new_situation", "rationale": "An award is now confirmed in A.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c06", "variant": "evidence_change"}, "question": "Choose the candidate reporting an awarded grant.", "split": "test", "state": "Candidate A: The grant was awarded. Candidate B: The application was received; the award remains undecided.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a71a17a287f3d30db2aa", "id": "d6de731cca52359b1c87", "label": 0, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The sponsor gives its name. Candidate B: The application has a reference number.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "familiar_submission_acceptance", "partition": "familiar_mechanism_new_situation", "rationale": "Neither supplies an award status.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "i_c06", "variant": "missing"}, "question": "Choose the candidate reporting an awarded grant.", "split": "test", "state": "Candidate A: The sponsor gives its name. Candidate B: The application has a reference number.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "986d7eaba0c7e9738e79", "id": "b45c63fb92729af27f05", "label": 0, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "On Monday the owner authorized scanning her notebook. On Tuesday she explicitly withdrew that authorization. The record says withdrawals cancel earlier permission, and no later permission was given.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "A later withdrawal canceled the earlier permission.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e01", "variant": "original"}, "question": "Assess the claim: permission to scan the notebook is currently valid.", "split": "test", "state": "On Monday the owner authorized scanning her notebook. On Tuesday she explicitly withdrew that authorization. The record says withdrawals cancel earlier permission, and no later permission was given.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "986d7eaba0c7e9738e79", "id": "46b7029b9a704138b77a", "label": 2, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "On Monday the owner authorized scanning her notebook. On Tuesday she explicitly withdrew that authorization. The record says withdrawals cancel earlier permission, and no later permission was given.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "The earlier grant remains a historical fact.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e01", "variant": "criterion_reversal"}, "question": "Assess the claim: the owner gave permission on Monday.", "split": "test", "state": "On Monday the owner authorized scanning her notebook. On Tuesday she explicitly withdrew that authorization. The record says withdrawals cancel earlier permission, and no later permission was given.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "986d7eaba0c7e9738e79", "id": "caf773b23934fa7362a1", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The owner authorized scanning the notebook on Monday. The complete permission record confirms that she has not withdrawn it and it remains valid.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "The complete record confirms continuing validity.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e01", "variant": "evidence_change"}, "question": "Assess the claim: permission to scan the notebook is currently valid.", "split": "test", "state": "The owner authorized scanning the notebook on Monday. The complete permission record confirms that she has not withdrawn it and it remains valid.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "986d7eaba0c7e9738e79", "id": "f46f392ef9e9e9df564b", "label": 1, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The owner authorized scanning on Monday. The supplied record omits all later permission changes and does not establish current validity.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "The incomplete later record does not settle current permission.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e01", "variant": "missing"}, "question": "Assess the claim: permission to scan the notebook is currently valid.", "split": "test", "state": "The owner authorized scanning on Monday. The supplied record omits all later permission changes and does not establish current validity.", "target_distribution": null}
{"family": "rule_application", "group_id": "ca2e0b043985fa49bcd2", "id": "005dcf6d2c0f21d64796", "label": 1, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The homeowner first allowed a surveyor to enter the attic. Before the visit she explicitly withdrew that permission. No later decision exists.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "The latest decision withdraws entry permission.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r01", "variant": "original"}, "question": "Use the homeowner's latest permission decision; a withdrawal cancels the earlier grant. May the surveyor enter the attic?", "split": "test", "state": "The homeowner first allowed a surveyor to enter the attic. Before the visit she explicitly withdrew that permission. No later decision exists.", "target_distribution": null}
{"family": "rule_application", "group_id": "ca2e0b043985fa49bcd2", "id": "edb708bd0fe5fd67d6ee", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The homeowner first allowed a surveyor to enter the attic. Before the visit she explicitly withdrew that permission. No later decision exists.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "The alternate fictional rule uses the initial grant.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r01", "variant": "criterion_reversal"}, "question": "For this fictional protocol, the first recorded decision is binding and later withdrawals are ignored. May the surveyor enter the attic?", "split": "test", "state": "The homeowner first allowed a surveyor to enter the attic. Before the visit she explicitly withdrew that permission. No later decision exists.", "target_distribution": null}
{"family": "rule_application", "group_id": "ca2e0b043985fa49bcd2", "id": "97a35499bd8717f3e786", "label": 2, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The homeowner first refused attic entry, then explicitly granted it before the visit. No later decision exists.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "The latest decision now grants entry.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r01", "variant": "evidence_change"}, "question": "Use the homeowner's latest permission decision; a withdrawal cancels the earlier grant. May the surveyor enter the attic?", "split": "test", "state": "The homeowner first refused attic entry, then explicitly granted it before the visit. No later decision exists.", "target_distribution": null}
{"family": "rule_application", "group_id": "ca2e0b043985fa49bcd2", "id": "31eb4533d2c89959ec46", "label": 0, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The homeowner gave an early permission decision, but neither its content nor any later decisions are available.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "The governing decision cannot be identified.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r01", "variant": "missing"}, "question": "Use the homeowner's latest permission decision; a withdrawal cancels the earlier grant. May the surveyor enter the attic?", "split": "test", "state": "The homeowner gave an early permission decision, but neither its content nor any later decisions are available.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "85051ba3c30277824ebe", "id": "0aa24a6701ac9c56c4a0", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The speaker authorized streaming, then explicitly withdrew the authorization before the event. Candidate B: The speaker authorized streaming and the complete record confirms it remains in force.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "Only B retains valid permission.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c01", "variant": "original"}, "question": "Select the candidate with currently valid streaming permission; a withdrawal cancels an earlier grant.", "split": "test", "state": "Candidate A: The speaker authorized streaming, then explicitly withdrew the authorization before the event. Candidate B: The speaker authorized streaming and the complete record confirms it remains in force.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "85051ba3c30277824ebe", "id": "796d5c0da6eaadae6a3f", "label": 1, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: The speaker authorized streaming, then explicitly withdrew the authorization before the event. Candidate B: The speaker authorized streaming and the complete record confirms it remains in force.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "A documents the later withdrawal.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c01", "variant": "criterion_reversal"}, "question": "Select the candidate documenting an authorization that was later withdrawn.", "split": "test", "state": "Candidate A: The speaker authorized streaming, then explicitly withdrew the authorization before the event. Candidate B: The speaker authorized streaming and the complete record confirms it remains in force.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "85051ba3c30277824ebe", "id": "599b3092b6fd18312d23", "label": 2, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: Streaming was authorized and the complete record says permission remains in force. Candidate B: Streaming was authorized, then explicitly withdrawn.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "Only A now retains valid permission.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c01", "variant": "evidence_change"}, "question": "Select the candidate with currently valid streaming permission; a withdrawal cancels an earlier grant.", "split": "test", "state": "Candidate A: Streaming was authorized and the complete record says permission remains in force. Candidate B: Streaming was authorized, then explicitly withdrawn.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "85051ba3c30277824ebe", "id": "960a4bcd27363e6db4b9", "label": 2, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A: A permission form is mentioned but its terms and later changes are missing. Candidate B: The event name is supplied without permission history.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_revocation", "partition": "heldout_family", "rationale": "Neither supplies enough permission history.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c01", "variant": "missing"}, "question": "Select the candidate with currently valid streaming permission; a withdrawal cancels an earlier grant.", "split": "test", "state": "Candidate A: A permission form is mentioned but its terms and later changes are missing. Candidate B: The event name is supplied without permission history.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e7798dab5f4f121cab77", "id": "c6a411e7f2bbdf377173", "label": 0, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The director gave Lina authority to approve exhibit labels and explicitly forbade further delegation. Lina then told Mo to approve them. The rule states an attempted forbidden delegation grants no authority; Mo has no other grant.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "Lina's forbidden redelegation grants Mo no authority.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e02", "variant": "original"}, "question": "Assess the claim: Mo has authority to approve exhibit labels.", "split": "test", "state": "The director gave Lina authority to approve exhibit labels and explicitly forbade further delegation. Lina then told Mo to approve them. The rule states an attempted forbidden delegation grants no authority; Mo has no other grant.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e7798dab5f4f121cab77", "id": "e332aa927d0c2ed044aa", "label": 2, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The director gave Lina authority to approve exhibit labels and explicitly forbade further delegation. Lina then told Mo to approve them. The rule states an attempted forbidden delegation grants no authority; Mo has no other grant.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "The director directly authorized Lina.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e02", "variant": "criterion_reversal"}, "question": "Assess the claim: Lina has authority to approve exhibit labels.", "split": "test", "state": "The director gave Lina authority to approve exhibit labels and explicitly forbade further delegation. Lina then told Mo to approve them. The rule states an attempted forbidden delegation grants no authority; Mo has no other grant.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e7798dab5f4f121cab77", "id": "a0128e551d095f3449f8", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The director gives Mo direct authority to approve exhibit labels; the record confirms this grant is valid.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "Mo now has a direct valid grant.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e02", "variant": "evidence_change"}, "question": "Assess the claim: Mo has authority to approve exhibit labels.", "split": "test", "state": "The director gives Mo direct authority to approve exhibit labels; the record confirms this grant is valid.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "e7798dab5f4f121cab77", "id": "46d6a0731bd288c370f0", "label": 0, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Mo is named in correspondence about labels, but no authority grants or delegation terms are supplied.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "No authority information is supplied.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e02", "variant": "missing"}, "question": "Assess the claim: Mo has authority to approve exhibit labels.", "split": "test", "state": "Mo is named in correspondence about labels, but no authority grants or delegation terms are supplied.", "target_distribution": null}
{"family": "rule_application", "group_id": "928ebc7817a8f1cac3e4", "id": "21ca7a8b970e36b8e20f", "label": 0, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The producer authorizes the stage manager to approve props. The stage manager passes the task to an assistant, who has no other approval authority.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "The original rule forbids the only attempted authority transfer.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r02", "variant": "original"}, "question": "The stage manager may approve props but may not delegate that authority. Only authorized approvers may approve. May the assistant approve a prop?", "split": "test", "state": "The producer authorizes the stage manager to approve props. The stage manager passes the task to an assistant, who has no other approval authority.", "target_distribution": null}
{"family": "rule_application", "group_id": "928ebc7817a8f1cac3e4", "id": "4aa83af95bb69e90bc2d", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The producer authorizes the stage manager to approve props. The stage manager passes the task to an assistant, who has no other approval authority.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "The alternate rule explicitly permits that transfer.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r02", "variant": "criterion_reversal"}, "question": "The stage manager may delegate prop approval to an assistant, and doing so grants the assistant approval authority. May this assistant approve?", "split": "test", "state": "The producer authorizes the stage manager to approve props. The stage manager passes the task to an assistant, who has no other approval authority.", "target_distribution": null}
{"family": "rule_application", "group_id": "928ebc7817a8f1cac3e4", "id": "cdfa9eed51fafcf09b72", "label": 2, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The producer directly authorizes the assistant to approve props.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "A direct grant gives the assistant authority.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r02", "variant": "evidence_change"}, "question": "The stage manager may approve props but may not delegate that authority. Only authorized approvers may approve. May the assistant approve a prop?", "split": "test", "state": "The producer directly authorizes the assistant to approve props.", "target_distribution": null}
{"family": "rule_application", "group_id": "928ebc7817a8f1cac3e4", "id": "d51b05bfb9d6d7f09331", "label": 1, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "An assistant works on the production, but no approval grants or delegation history are provided.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "The relevant grants are unknown.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r02", "variant": "missing"}, "question": "The stage manager may approve props but may not delegate that authority. Only authorized approvers may approve. May the assistant approve a prop?", "split": "test", "state": "An assistant works on the production, but no approval grants or delegation history are provided.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "53b0770ec36835b81d3a", "id": "fdd902cc6e65fad7cb4f", "label": 1, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The rule permits direct delegation by the director but forbids any further delegation. Candidate A: I received prop approval authority directly from the director. Candidate B: I received it only from A, and have no other grant.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "A has the permitted direct grant.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c02", "variant": "original"}, "question": "Select the candidate holding valid prop approval authority under the stated delegation rule.", "split": "test", "state": "The rule permits direct delegation by the director but forbids any further delegation. Candidate A: I received prop approval authority directly from the director. Candidate B: I received it only from A, and have no other grant.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "53b0770ec36835b81d3a", "id": "42384af9ae56beb37ff5", "label": 2, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The rule permits direct delegation by the director but forbids any further delegation. Candidate A: I received prop approval authority directly from the director. Candidate B: I received it only from A, and have no other grant.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "B relies only on prohibited redelegation.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c02", "variant": "criterion_reversal"}, "question": "Select the candidate whose claimed authority comes only through a prohibited second delegation.", "split": "test", "state": "The rule permits direct delegation by the director but forbids any further delegation. Candidate A: I received prop approval authority directly from the director. Candidate B: I received it only from A, and have no other grant.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "53b0770ec36835b81d3a", "id": "cdf7f428857e1b841cb3", "label": 2, "options": [{"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The rule permits direct delegation by the director but forbids further delegation. Candidate A: My only claimed authority came from B. Candidate B: The director directly granted me prop approval authority.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "B now holds the direct grant.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c02", "variant": "evidence_change"}, "question": "Select the candidate holding valid prop approval authority under the stated delegation rule.", "split": "test", "state": "The rule permits direct delegation by the director but forbids further delegation. Candidate A: My only claimed authority came from B. Candidate B: The director directly granted me prop approval authority.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "53b0770ec36835b81d3a", "id": "774f7f9245f83c796d44", "label": 2, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The director's delegation rule is supplied, but Candidate A and Candidate B provide only their names and no authority histories.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_delegation_limits", "partition": "heldout_family", "rationale": "Neither supplies a grant history.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c02", "variant": "missing"}, "question": "Select the candidate holding valid prop approval authority under the stated delegation rule.", "split": "test", "state": "The director's delegation rule is supplied, but Candidate A and Candidate B provide only their names and no authority histories.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "056cc5b726d6e8300a77", "id": "92ec2ffa3dd46ada1335", "label": 0, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The return policy accepts faulty lamps despite opened packaging, but explicitly excludes water damage even when the lamp is faulty. This opened lamp is faulty because water entered it. No other cause is present.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "The explicit water-damage exclusion overrides the faulty-lamp exception.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e03", "variant": "original"}, "question": "Assess the claim: this lamp qualifies for return under the stated policy.", "split": "test", "state": "The return policy accepts faulty lamps despite opened packaging, but explicitly excludes water damage even when the lamp is faulty. This opened lamp is faulty because water entered it. No other cause is present.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "056cc5b726d6e8300a77", "id": "6149a17bc154f9c5b4a0", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The return policy accepts faulty lamps despite opened packaging, but explicitly excludes water damage even when the lamp is faulty. This opened lamp is faulty because water entered it. No other cause is present.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "The lamp is explicitly faulty despite being excluded.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e03", "variant": "criterion_reversal"}, "question": "Assess the claim: the lamp has a fault.", "split": "test", "state": "The return policy accepts faulty lamps despite opened packaging, but explicitly excludes water damage even when the lamp is faulty. This opened lamp is faulty because water entered it. No other cause is present.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "056cc5b726d6e8300a77", "id": "be866e1dbec5d42d9423", "label": 2, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The return policy accepts faulty lamps despite opened packaging, but excludes water damage even when faulty. This opened lamp has a manufacturing wiring fault, with no water damage or other exclusion.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "The manufacturing fault qualifies and the overriding exclusion is absent.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e03", "variant": "evidence_change"}, "question": "Assess the claim: this lamp qualifies for return under the stated policy.", "split": "test", "state": "The return policy accepts faulty lamps despite opened packaging, but excludes water damage even when faulty. This opened lamp has a manufacturing wiring fault, with no water damage or other exclusion.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "056cc5b726d6e8300a77", "id": "97d2ed2591fc7638806c", "label": 0, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The return policy accepts faulty lamps despite opened packaging but excludes water damage even when faulty. The lamp is faulty, but the cause and presence of water damage are omitted.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "The exclusion's applicability is unknown.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e03", "variant": "missing"}, "question": "Assess the claim: this lamp qualifies for return under the stated policy.", "split": "test", "state": "The return policy accepts faulty lamps despite opened packaging but excludes water damage even when faulty. The lamp is faulty, but the cause and presence of water damage are omitted.", "target_distribution": null}
{"family": "rule_application", "group_id": "e3ce8ec496dbc9974f94", "id": "4533dd91267b4d815690", "label": 2, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The opened paint tin has a manufacturing defect and also contains added solvent. Both facts are confirmed.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "Added solvent triggers the overriding exclusion.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r03", "variant": "original"}, "question": "Normally reject opened paint. Accept it for a manufacturing defect, except that added solvent always makes it ineligible. May this return be accepted?", "split": "test", "state": "The opened paint tin has a manufacturing defect and also contains added solvent. Both facts are confirmed.", "target_distribution": null}
{"family": "rule_application", "group_id": "e3ce8ec496dbc9974f94", "id": "d880e65fef5e2993fdd0", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The opened paint tin has a manufacturing defect and also contains added solvent. Both facts are confirmed.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "The alternate rule explicitly makes solvent irrelevant.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r03", "variant": "criterion_reversal"}, "question": "Accept defective paint even if opened or mixed with solvent; neither condition overrides the defect exception. May this return be accepted?", "split": "test", "state": "The opened paint tin has a manufacturing defect and also contains added solvent. Both facts are confirmed.", "target_distribution": null}
{"family": "rule_application", "group_id": "e3ce8ec496dbc9974f94", "id": "90494c76078ffc3c717a", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The opened tin has a manufacturing defect and the inspection confirms no solvent was added.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "The defect exception now applies without the exclusion.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r03", "variant": "evidence_change"}, "question": "Normally reject opened paint. Accept it for a manufacturing defect, except that added solvent always makes it ineligible. May this return be accepted?", "split": "test", "state": "The opened tin has a manufacturing defect and the inspection confirms no solvent was added.", "target_distribution": null}
{"family": "rule_application", "group_id": "e3ce8ec496dbc9974f94", "id": "9ac5cc82c4126c91174a", "label": 1, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The opened tin is defective, but whether solvent was added is not known.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "The overriding exclusion cannot be evaluated.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r03", "variant": "missing"}, "question": "Normally reject opened paint. Accept it for a manufacturing defect, except that added solvent always makes it ineligible. May this return be accepted?", "split": "test", "state": "The opened tin is defective, but whether solvent was added is not known.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a5b234c2e4fb304f1071", "id": "386e10b1c1ce906473a4", "label": 1, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A: My submission was late because of a documented outage and includes the required signature. Candidate B: Mine was late because of a documented outage but lacks the signature.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "A satisfies the exception and avoids the exclusion.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c03", "variant": "original"}, "question": "Select the submission qualifying under the stated exception and exclusion.", "split": "test", "state": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A: My submission was late because of a documented outage and includes the required signature. Candidate B: Mine was late because of a documented outage but lacks the signature.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a5b234c2e4fb304f1071", "id": "4da6aedc1139224eeff2", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A: My submission was late because of a documented outage and includes the required signature. Candidate B: Mine was late because of a documented outage but lacks the signature.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "B triggers the overriding missing-signature exclusion.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c03", "variant": "criterion_reversal"}, "question": "Select the submission disqualified specifically by the signature exclusion despite the outage exception.", "split": "test", "state": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A: My submission was late because of a documented outage and includes the required signature. Candidate B: Mine was late because of a documented outage but lacks the signature.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a5b234c2e4fb304f1071", "id": "a22dab8ba073ff6cdb8d", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A: My late outage-related submission lacks the signature. Candidate B: My late outage-related submission has the signature. Both outages are documented.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "B now qualifies.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c03", "variant": "evidence_change"}, "question": "Select the submission qualifying under the stated exception and exclusion.", "split": "test", "state": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A: My late outage-related submission lacks the signature. Candidate B: My late outage-related submission has the signature. Both outages are documented.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "a5b234c2e4fb304f1071", "id": "42c54fa2ad45743e24d8", "label": 1, "options": [{"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A and Candidate B report late submissions but provide neither outage documentation nor signature information.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_exception_exclusion", "partition": "heldout_family", "rationale": "Neither supplies the facts needed for eligibility.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c03", "variant": "missing"}, "question": "Select the submission qualifying under the stated exception and exclusion.", "split": "test", "state": "Accept late submissions for documented outages, except that missing signatures always disqualify. Candidate A and Candidate B report late submissions but provide neither outage documentation nor signature information.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "d142f7fd0efa0eddc788", "id": "f3d13cf184f916677def", "label": 0, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The desk log says the crate is in storage. The warehouse inventory says it has left storage. Both reports are current. The supplied protocol says the warehouse inventory controls location decisions whenever these two sources conflict.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "The controlling inventory says it has left.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e04", "variant": "original"}, "question": "Assess the claim: under the supplied protocol, the crate is currently in storage.", "split": "test", "state": "The desk log says the crate is in storage. The warehouse inventory says it has left storage. Both reports are current. The supplied protocol says the warehouse inventory controls location decisions whenever these two sources conflict.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "d142f7fd0efa0eddc788", "id": "1105577da4c8dad4609d", "label": 2, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The desk log says the crate is in storage. The warehouse inventory says it has left storage. Both reports are current. The supplied protocol says the warehouse inventory controls location decisions whenever these two sources conflict.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "The noncontrolling desk log still explicitly reports storage.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e04", "variant": "criterion_reversal"}, "question": "Assess the claim: the desk log reports the crate in storage.", "split": "test", "state": "The desk log says the crate is in storage. The warehouse inventory says it has left storage. Both reports are current. The supplied protocol says the warehouse inventory controls location decisions whenever these two sources conflict.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "d142f7fd0efa0eddc788", "id": "0ebd299739302d8b057b", "label": 2, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The desk log says the crate has left storage; the warehouse inventory says it is in storage. Both reports are current, and the inventory controls conflicts under the protocol.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "The controlling inventory now places it in storage.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e04", "variant": "evidence_change"}, "question": "Assess the claim: under the supplied protocol, the crate is currently in storage.", "split": "test", "state": "The desk log says the crate has left storage; the warehouse inventory says it is in storage. Both reports are current, and the inventory controls conflicts under the protocol.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "d142f7fd0efa0eddc788", "id": "eafc22c8c40df3932a8e", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The protocol gives the inventory priority, but the current inventory and desk-log entries for the crate are omitted.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "The controlling report is missing.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e04", "variant": "missing"}, "question": "Assess the claim: under the supplied protocol, the crate is currently in storage.", "split": "test", "state": "The protocol gives the inventory priority, but the current inventory and desk-log entries for the crate are omitted.", "target_distribution": null}
{"family": "rule_application", "group_id": "3476d3f4f32290fe57c3", "id": "0aa1447680e4f25862bb", "label": 1, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The desk clerk reports the equipment deposit as paid. The signed ledger reports it as unpaid. Both reports are current.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "The original controlling source reports unpaid.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r04", "variant": "original"}, "question": "Release equipment exactly when the deposit is recorded paid; in conflicts, the signed ledger overrides the desk clerk. May it be released?", "split": "test", "state": "The desk clerk reports the equipment deposit as paid. The signed ledger reports it as unpaid. Both reports are current.", "target_distribution": null}
{"family": "rule_application", "group_id": "3476d3f4f32290fe57c3", "id": "980e76145c925747220c", "label": 2, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The desk clerk reports the equipment deposit as paid. The signed ledger reports it as unpaid. Both reports are current.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "The alternate controlling source reports paid.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r04", "variant": "criterion_reversal"}, "question": "Release equipment exactly when the deposit is recorded paid; under this alternative protocol, the desk clerk overrides the ledger. May it be released?", "split": "test", "state": "The desk clerk reports the equipment deposit as paid. The signed ledger reports it as unpaid. Both reports are current.", "target_distribution": null}
{"family": "rule_application", "group_id": "3476d3f4f32290fe57c3", "id": "4aee37a6a155747766aa", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The desk clerk reports the deposit unpaid, while the signed ledger reports it paid. Both reports are current.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "The original controlling ledger now reports paid.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r04", "variant": "evidence_change"}, "question": "Release equipment exactly when the deposit is recorded paid; in conflicts, the signed ledger overrides the desk clerk. May it be released?", "split": "test", "state": "The desk clerk reports the deposit unpaid, while the signed ledger reports it paid. Both reports are current.", "target_distribution": null}
{"family": "rule_application", "group_id": "3476d3f4f32290fe57c3", "id": "6db17cc22bd656d78558", "label": 2, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The record mentions the desk clerk and ledger but supplies neither deposit status.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "Neither source's status is available.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r04", "variant": "missing"}, "question": "Release equipment exactly when the deposit is recorded paid; in conflicts, the signed ledger overrides the desk clerk. May it be released?", "split": "test", "state": "The record mentions the desk clerk and ledger but supplies neither deposit status.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c979db7416b206c1eb09", "id": "ef3e0f8e657bf59f8a29", "label": 2, "options": [{"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "For this task, the current signed checklist controls readiness when it conflicts with chat. Candidate A: Chat says ready, but the signed checklist says not ready. Candidate B: Chat says not ready, but the signed checklist says ready.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "B alone is ready in the controlling checklist.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c04", "variant": "original"}, "question": "Select the candidate ready according to the controlling signed checklist.", "split": "test", "state": "For this task, the current signed checklist controls readiness when it conflicts with chat. Candidate A: Chat says ready, but the signed checklist says not ready. Candidate B: Chat says not ready, but the signed checklist says ready.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c979db7416b206c1eb09", "id": "f756e1cb04e11766341e", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "For this task, the current signed checklist controls readiness when it conflicts with chat. Candidate A: Chat says ready, but the signed checklist says not ready. Candidate B: Chat says not ready, but the signed checklist says ready.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "A alone is ready in chat.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c04", "variant": "criterion_reversal"}, "question": "Under the alternative protocol, chat controls conflicts. Select the candidate ready according to chat.", "split": "test", "state": "For this task, the current signed checklist controls readiness when it conflicts with chat. Candidate A: Chat says ready, but the signed checklist says not ready. Candidate B: Chat says not ready, but the signed checklist says ready.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c979db7416b206c1eb09", "id": "d167e51bb1f499099645", "label": 0, "options": [{"description": "Candidate A", "id": "A"}, {"description": "Candidate B", "id": "B"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The signed checklist controls conflicts. Candidate A: The checklist says ready and chat says not ready. Candidate B: The checklist says not ready and chat says ready.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "A is now ready in the controlling checklist.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c04", "variant": "evidence_change"}, "question": "Select the candidate ready according to the controlling signed checklist.", "split": "test", "state": "The signed checklist controls conflicts. Candidate A: The checklist says ready and chat says not ready. Candidate B: The checklist says not ready and chat says ready.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "c979db7416b206c1eb09", "id": "674ca4e9346216255466", "label": 0, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The signed checklist controls conflicts, but Candidate A and Candidate B provide only project names without checklist or chat status.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_authority_hierarchy", "partition": "heldout_family", "rationale": "Neither readiness report is supplied.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c04", "variant": "missing"}, "question": "Select the candidate ready according to the controlling signed checklist.", "split": "test", "state": "The signed checklist controls conflicts, but Candidate A and Candidate B provide only project names without checklist or chat status.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f7403e39d6796072491c", "id": "1df851a202994a07f123", "label": 0, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The agreed handover requires both the source files and the user guide. The delivery record confirms the source files arrived and explicitly says the guide was not delivered.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "A required deliverable is explicitly missing.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e05", "variant": "original"}, "question": "Assess the claim: the agreed handover is complete.", "split": "test", "state": "The agreed handover requires both the source files and the user guide. The delivery record confirms the source files arrived and explicitly says the guide was not delivered.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f7403e39d6796072491c", "id": "b5e6788d95b8395d2888", "label": 2, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The agreed handover requires both the source files and the user guide. The delivery record confirms the source files arrived and explicitly says the guide was not delivered.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "One required component was delivered.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e05", "variant": "criterion_reversal"}, "question": "Assess the claim: the source files were delivered.", "split": "test", "state": "The agreed handover requires both the source files and the user guide. The delivery record confirms the source files arrived and explicitly says the guide was not delivered.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f7403e39d6796072491c", "id": "32f36cd0fbbfb3604586", "label": 0, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The agreed handover requires the source files and user guide. The delivery record confirms that both arrived.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "All required deliverables are now confirmed.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e05", "variant": "evidence_change"}, "question": "Assess the claim: the agreed handover is complete.", "split": "test", "state": "The agreed handover requires the source files and user guide. The delivery record confirms that both arrived.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "f7403e39d6796072491c", "id": "843185706adff0c60f8b", "label": 2, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The agreement requires source files and a user guide, but the delivery record gives no information about either item.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "Neither required delivery is established.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e05", "variant": "missing"}, "question": "Assess the claim: the agreed handover is complete.", "split": "test", "state": "The agreement requires source files and a user guide, but the delivery record gives no information about either item.", "target_distribution": null}
{"family": "rule_application", "group_id": "a060fd4621be415e9f1a", "id": "b531dd603bf080dab820", "label": 0, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The contractor supplied the translated manual but explicitly did not supply the glossary.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "The complete handover lacks the glossary.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r05", "variant": "original"}, "question": "Approve final handover exactly when both the translated manual and glossary have been supplied. May final handover be approved?", "split": "test", "state": "The contractor supplied the translated manual but explicitly did not supply the glossary.", "target_distribution": null}
{"family": "rule_application", "group_id": "a060fd4621be415e9f1a", "id": "be2dd67bc733c1213bef", "label": 1, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The contractor supplied the translated manual but explicitly did not supply the glossary.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "The narrower milestone requires only the delivered manual.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r05", "variant": "criterion_reversal"}, "question": "Approve the translation milestone when the manual has been supplied; the glossary is not required for this milestone. May this milestone be approved?", "split": "test", "state": "The contractor supplied the translated manual but explicitly did not supply the glossary.", "target_distribution": null}
{"family": "rule_application", "group_id": "a060fd4621be415e9f1a", "id": "b8e2750307df9761e716", "label": 0, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The contractor supplied both the translated manual and the glossary.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "Both final-handover deliverables are now present.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r05", "variant": "evidence_change"}, "question": "Approve final handover exactly when both the translated manual and glossary have been supplied. May final handover be approved?", "split": "test", "state": "The contractor supplied both the translated manual and the glossary.", "target_distribution": null}
{"family": "rule_application", "group_id": "a060fd4621be415e9f1a", "id": "5d31b4bf2d0fc193f53b", "label": 1, "options": [{"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The contractor reports working on the handover but supplies no delivery record for either item.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "Work in progress does not establish delivery.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r05", "variant": "missing"}, "question": "Approve final handover exactly when both the translated manual and glossary have been supplied. May final handover be approved?", "split": "test", "state": "The contractor reports working on the handover but supplies no delivery record for either item.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2b0f8eda8903e38a47de", "id": "a9230edfc6ec9e524af0", "label": 2, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "A complete incident package requires both a timeline and a remedy report. Candidate A: Both documents are attached. Candidate B: Only the timeline is attached; the remedy report is absent.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "A includes both required deliverables.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c05", "variant": "original"}, "question": "Select the complete incident package.", "split": "test", "state": "A complete incident package requires both a timeline and a remedy report. Candidate A: Both documents are attached. Candidate B: Only the timeline is attached; the remedy report is absent.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2b0f8eda8903e38a47de", "id": "c100e84585149a04be49", "label": 0, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "A complete incident package requires both a timeline and a remedy report. Candidate A: Both documents are attached. Candidate B: Only the timeline is attached; the remedy report is absent.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "B is explicitly partial in the specified way.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c05", "variant": "criterion_reversal"}, "question": "Select the package containing the timeline but missing the remedy report.", "split": "test", "state": "A complete incident package requires both a timeline and a remedy report. Candidate A: Both documents are attached. Candidate B: Only the timeline is attached; the remedy report is absent.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2b0f8eda8903e38a47de", "id": "579beed184ffc1104aa9", "label": 0, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "A complete package requires both documents. Candidate A: The timeline is attached but the remedy report is absent. Candidate B: Both the timeline and remedy report are attached.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "B now includes both deliverables.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c05", "variant": "evidence_change"}, "question": "Select the complete incident package.", "split": "test", "state": "A complete package requires both documents. Candidate A: The timeline is attached but the remedy report is absent. Candidate B: Both the timeline and remedy report are attached.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2b0f8eda8903e38a47de", "id": "c4e9bd463b6fde5be743", "label": 1, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "A complete package requires both documents. Candidate A and Candidate B give only package titles and no attachment information.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_partial_deliverables", "partition": "heldout_family", "rationale": "Neither supplies attachment status.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c05", "variant": "missing"}, "question": "Select the complete incident package.", "split": "test", "state": "A complete package requires both documents. Candidate A and Candidate B give only package titles and no attachment information.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "01020a1eaf46ec2fba5d", "id": "b863998d4be2721a5dfd", "label": 2, "options": [{"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. Both workshops confirm they can perform this repair.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "A capable primary workshop prevents use of the fallback.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e06", "variant": "original"}, "question": "Assess the claim: the routing rule assigns this repair to the outside workshop.", "split": "test", "state": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. Both workshops confirm they can perform this repair.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "01020a1eaf46ec2fba5d", "id": "37cebc3792e98049439f", "label": 1, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. Both workshops confirm they can perform this repair.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The outside workshop is capable even though not selected.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e06", "variant": "criterion_reversal"}, "question": "Assess the claim: the outside workshop can perform this repair.", "split": "test", "state": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. Both workshops confirm they can perform this repair.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "01020a1eaf46ec2fba5d", "id": "64d63f86e328aff1481c", "label": 2, "options": [{"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. The in-house workshop cannot perform this repair; the outside workshop can.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The primary is inapplicable and the fallback is capable.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e06", "variant": "evidence_change"}, "question": "Assess the claim: the routing rule assigns this repair to the outside workshop.", "split": "test", "state": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. The in-house workshop cannot perform this repair; the outside workshop can.", "target_distribution": null}
{"family": "evidence_interpretation", "group_id": "01020a1eaf46ec2fba5d", "id": "6d64a8459f64eac7f4cc", "label": 0, "options": [{"description": "The evidence does not establish either", "id": "insufficient"}, {"description": "The evidence establishes the opposite", "id": "contradicted"}, {"description": "The evidence establishes the claim", "id": "supported"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. Neither workshop's ability to perform this repair is supplied.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The conditions governing the fallback are unknown.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_e06", "variant": "missing"}, "question": "Assess the claim: the routing rule assigns this repair to the outside workshop.", "split": "test", "state": "The routing rule assigns repairs to the in-house workshop when it can perform the work; use the outside workshop only when the in-house workshop cannot. Neither workshop's ability to perform this repair is supplied.", "target_distribution": null}
{"family": "rule_application", "group_id": "6a06a2f278301c1412ee", "id": "2708193212d8a4a523c7", "label": 0, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The town hall and school both confirm they are available and suitable for the meeting.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The suitable available primary blocks the fallback.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r06", "variant": "original"}, "question": "Book the school only as a fallback when the town hall is unavailable or unsuitable; otherwise book the town hall. May the school be booked under this policy?", "split": "test", "state": "The town hall and school both confirm they are available and suitable for the meeting.", "target_distribution": null}
{"family": "rule_application", "group_id": "6a06a2f278301c1412ee", "id": "5a37d1dfcad20810a76a", "label": 1, "options": [{"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The town hall and school both confirm they are available and suitable for the meeting.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The alternate policy removes that priority restriction.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r06", "variant": "criterion_reversal"}, "question": "Under the alternative policy, either suitable available venue may be booked without priority. May the school be booked?", "split": "test", "state": "The town hall and school both confirm they are available and suitable for the meeting.", "target_distribution": null}
{"family": "rule_application", "group_id": "6a06a2f278301c1412ee", "id": "acacd8b46acd25cb954b", "label": 1, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule permits the action", "id": "permitted"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The town hall is confirmed unavailable, while the school is available and suitable.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The primary is unavailable and the fallback qualifies.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r06", "variant": "evidence_change"}, "question": "Book the school only as a fallback when the town hall is unavailable or unsuitable; otherwise book the town hall. May the school be booked under this policy?", "split": "test", "state": "The town hall is confirmed unavailable, while the school is available and suitable.", "target_distribution": null}
{"family": "rule_application", "group_id": "6a06a2f278301c1412ee", "id": "4089c199b8b796ec3ed8", "label": 0, "options": [{"description": "The supplied information does not settle whether the rule permits the action", "id": "insufficient"}, {"description": "The stated rule prohibits the action", "id": "prohibited"}, {"description": "The stated rule permits the action", "id": "permitted"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "The two venues are named, but neither availability nor suitability is provided.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The priority conditions are unknown.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_r06", "variant": "missing"}, "question": "Book the school only as a fallback when the town hall is unavailable or unsuitable; otherwise book the town hall. May the school be booked under this policy?", "split": "test", "state": "The two venues are named, but neither availability nor suitability is provided.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2797f8c23d05c0ee06b3", "id": "02f2b95a5c5a4777a45c", "label": 1, "options": [{"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A is the preferred translator and is confirmed able to handle this language. Candidate B is the fallback translator and is also able to handle it.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The capable preferred candidate takes priority.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c06", "variant": "original"}, "question": "Choose A whenever A is able; choose B only when A is unable and B is able.", "split": "test", "state": "Candidate A is the preferred translator and is confirmed able to handle this language. Candidate B is the fallback translator and is also able to handle it.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2797f8c23d05c0ee06b3", "id": "ebea4681e3dcc7d92ca2", "label": 1, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A is the preferred translator and is confirmed able to handle this language. Candidate B is the fallback translator and is also able to handle it.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The alternative priority selects capable B.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c06", "variant": "criterion_reversal"}, "question": "Under the alternative priority, choose B whenever B is able; choose A only when B is unable and A is able.", "split": "test", "state": "Candidate A is the preferred translator and is confirmed able to handle this language. Candidate B is the fallback translator and is also able to handle it.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2797f8c23d05c0ee06b3", "id": "b8c5b9b285cbdddc9f8d", "label": 2, "options": [{"description": "Candidate A", "id": "A"}, {"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A is the preferred translator but is confirmed unable to handle this language. Candidate B is the fallback and is confirmed able.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "A is inapplicable and the capable fallback B qualifies.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c06", "variant": "evidence_change"}, "question": "Choose A whenever A is able; choose B only when A is unable and B is able.", "split": "test", "state": "Candidate A is the preferred translator but is confirmed unable to handle this language. Candidate B is the fallback and is confirmed able.", "target_distribution": null}
{"family": "candidate_selection", "group_id": "2797f8c23d05c0ee06b3", "id": "7ed75d854d99b3a81916", "label": 0, "options": [{"description": "Neither candidate supplies the requested evidence", "id": "insufficient"}, {"description": "Candidate B", "id": "B"}, {"description": "Candidate A", "id": "A"}], "provenance": {"annotation_status": "all_source_groups_independently_model_reviewed_not_human_adjudicated", "evidence_span": "Candidate A is preferred and Candidate B is the fallback, but neither candidate's ability to handle the language is supplied.", "evidence_span_note": "Complete short situation retained; rationale identifies decisive evidence.", "generator_revision": "semantic-eval-v1", "kind": "authored_synthetic", "mechanism": "heldout_priority_fallback", "partition": "heldout_family", "rationale": "The applicability conditions are unknown.", "rights": "Project authored; no copied external text", "source": "project-authored", "source_key": "h_c06", "variant": "missing"}, "question": "Choose A whenever A is able; choose B only when A is unable and B is able.", "split": "test", "state": "Candidate A is preferred and Candidate B is the fallback, but neither candidate's ability to handle the language is supplied.", "target_distribution": null}
Reference in New Issue View Git Blame Copy Permalink