scripts/test_claim_audit_calibration.py

• _load_gold_set · function · L51-L53 — def _load_gold_set() -> list[dict[str, Any]]
• _tuple_lookup_key · function · L56-L71 — def _tuple_lookup_key(tup: dict[str, Any]) -> tuple[str, str, str | None]
• _perfect_judge · function · L74-L130 — def _perfect_judge() -> Callable[..., dict[str, Any]]
• fn · function · L90-L128 — def fn(**kwargs: Any) -> dict[str, Any]
• _bad_judge · function · L141-L193 — def _bad_judge() -> Callable[..., dict[str, Any]]
• fn · function · L168-L191 — def fn(**kwargs: Any) -> dict[str, Any]
• TC3GoldSetShape · class · L201-L488 — class TC3GoldSetShape(unittest.TestCase)
• setUp · method · L217-L218 — def setUp(self) -> None
• test_a_every_tuple_has_valid_tuple_kind · method · L220-L228 — def test_a_every_tuple_has_valid_tuple_kind(self) -> None: # All tuples must declare tuple_kind ∈ {alignment, constraint}. # Spec §7.7 rule (a).
• test_b_alignment_tuples_shape_pinned · method · L230-L253 — def test_b_alignment_tuples_shape_pinned(self) -> None: # Alignment tuples carry expected_judgment in 4-judgment set AND # MUST NOT carry constraint fields. Spec §7.7 rule (b).
• test_c_constraint_tuples_have_required_fields · method · L255-L279 — def test_c_constraint_tuples_have_required_fields(self) -> None: # Constraint tuples carry expected_judgment ∈ {VIOLATED, NOT_VIOLATED} # AND constraint_under_test_id AND # (constraint_under_test_rule_text OR manifest_fixture_path). Spec §7.7 rule (c).
• test_d_at_least_three_not_violated_constraint_tuples · method · L281-L295 — def test_d_at_least_three_not_violated_constraint_tuples(self) -> None: # ≥3 NOT_VIOLATED constraint tuples — without them constraint FPR # is unmeasurable and T-C1 cannot fail-on-threshold on the # constraint line. Spec §7.7 rule (d).
• test_validate_gold_set_rejects_invalid_tuple_kind · method · L297-L311 — def test_validate_gold_set_rejects_invalid_tuple_kind(self) -> None: # Rule (a) negative ingestion test — round-1 review # closure on T-C3 rule coverage. Diagnostic must name rule (a).
• test_validate_gold_set_rejects_alignment_with_constraint_field · method · L313-L330 — def test_validate_gold_set_rejects_alignment_with_constraint_field(self) -> None: # validate_gold_set is the production entrypoint that the # calibration runner calls at ingestion time. It MUST raise the # documented error on rule-(b) violation. round-1 review closure: # also assert rule (b) name in diagnostic.
• test_validate_gold_set_rejects_constraint_without_rule_text · method · L332-L349 — def test_validate_gold_set_rejects_constraint_without_rule_text(self) -> None: # Rule (c) — constraint tuple missing both rule_text AND # manifest_fixture_path. Most common silent-skip authoring bug.
• test_validate_gold_set_rejects_under_three_not_violated · method · L351-L375 — def test_validate_gold_set_rejects_under_three_not_violated(self) -> None: # Rule (d) — fewer than 3 NOT_VIOLATED constraint tuples.
• test_validate_gold_set_accepts_canonical_gold_set · method · L377-L379 — def test_validate_gold_set_accepts_canonical_gold_set(self) -> None: # Positive path — the shipped gold set MUST validate cleanly.
• test_validate_gold_set_rejects_partial_without_expected_sub_claims · method · L381-L405 — def test_validate_gold_set_rejects_partial_without_expected_sub_claims(self) -> None: # #355 P2#4 — rule (e): a PARTIAL fixture is the ONLY thing exercising # the atomic-decomposition subset metric. If it omits expected_sub_claims, # _breakdown_covers_expected early-returns True and the metric is # silently skipped — any generic two-line true-partial breakdown reports # miss_rate=0, defeating the whole point of the #213 calibration. Ingestion # MUST reject a PARTIAL tuple with missing/empty expected_sub_claims.
• test_validate_gold_set_accepts_partial_with_expected_sub_claims · method · L407-L435 — def test_validate_gold_set_accepts_partial_with_expected_sub_claims(self) -> None: # Positive: a PARTIAL fixture that declares expected_sub_claims passes # rule (e). Guards the rule from over-firing on well-formed fixtures. # Padded with 3 NOT_VIOLATED constraint tuples to satisfy rule (d).
• test_run_calibration_rejects_manifest_only_constraint_tuple · method · L437-L488 — def test_run_calibration_rejects_manifest_only_constraint_tuple(self) -> None: # round-2 review closure: validate_gold_set accepts EITHER inline # rule_text OR manifest_fixture_path per spec §7.7 rule (c), but # the v3.8.0 runner only supports the inline form. A manifest-only # tuple validating clean but reaching the judge with rule="" is # exactly the silent-skip authoring bug T-C3 is supposed to # prevent. The runner MUST raise NotImplementedError at run time # rather than pass the empty rule through to the judge. We pin # this against a minimal synthetic set (no stub lookup needed # because run_calibration raises before any judge invocation # for the constraint tuples).
• _never_called_judge · function · L448-L449 — def _never_called_judge(**kwargs: Any) -> dict[str, Any]
• TC2PerClassReport · class · L496-L559 — class TC2PerClassReport(unittest.TestCase)
• setUp · method · L505-L507 — def setUp(self) -> None
• test_report_has_per_class_block · method · L509-L513 — def test_report_has_per_class_block(self) -> None: # Report must surface a per_class section keyed by judgment-class.
• test_per_class_includes_four_judgment_classes · method · L515-L520 — def test_per_class_includes_four_judgment_classes(self) -> None: # Spec §7.7 enumerates SUPPORTED, UNSUPPORTED, AMBIGUOUS, and # violated-constraint as the four classes that must appear.
• test_each_class_exposes_fnr_and_fpr · method · L522-L529 — def test_each_class_exposes_fnr_and_fpr(self) -> None: # Per-class reporting must include both FNR and FPR (not just one). # Reviewer asymmetry — missing FPR on UNSUPPORTED was the # historical hole reviewer-calibration_mode_protocol referenced.
• test_each_class_exposes_denominators · method · L531-L539 — def test_each_class_exposes_denominators(self) -> None: # Protocol doc Phase 4 contract: each class entry carries # n_positive + n_negative so 0.0 FNR on 0 positives is # distinguishable from 0.0 FNR on N positives. round-1 review # closure — earlier test only asserted FNR/FPR keys.
• test_canonical_denominators_match_gold_set · method · L541-L559 — def test_canonical_denominators_match_gold_set(self) -> None: # Pin the expected one-vs-rest denominators against the canonical # 17-alignment + 8-constraint gold set (12 base alignment + 5 #213 # partial fixtures). Contract pinning (round-1 review) — without this a # future gold-set rebalance could silently shift class distributions # away from the documented Phase 4 example.
• TC1ThresholdEnforcement · class · L567-L607 — class TC1ThresholdEnforcement(unittest.TestCase)
• setUp · method · L581-L583 — def setUp(self) -> None
• test_fnr_below_threshold · method · L585-L591 — def test_fnr_below_threshold(self) -> None: # Spec §7.7 gate: FNR < 0.15. Perfect-judge stub yields FNR = 0.
• test_fpr_below_threshold · method · L593-L599 — def test_fpr_below_threshold(self) -> None: # Spec §7.7 gate: FPR < 0.10. Perfect-judge stub yields FPR = 0.
• test_report_records_thresholds_used · method · L601-L607 — def test_report_records_thresholds_used(self) -> None: # Operational concern: when CI fails on T-C1, the report MUST # surface the threshold values it was checked against so the # operator can distinguish a regression (judge degraded) from a # threshold tightening (spec bump). Spec §7.7 + §9 acceptance.
• TC1ThresholdEnforcementBadJudge · class · L610-L664 — class TC1ThresholdEnforcementBadJudge(unittest.TestCase)
• setUp · method · L630-L632 — def setUp(self) -> None
• test_bad_judge_drives_fnr_over_threshold · method · L634-L641 — def test_bad_judge_drives_fnr_over_threshold(self) -> None: # Bad judge flips every alignment expected → opposite class + # every VIOLATED → NOT_VIOLATED. Aggregate FNR must be >= 0.15.
• test_bad_judge_drives_fpr_over_threshold · method · L643-L650 — def test_bad_judge_drives_fpr_over_threshold(self) -> None: # Same logic for FPR. The mirror-flip on alignment + the # NOT_VIOLATED → VIOLATED flip on constraints both contribute.
• test_bad_judge_per_class_fnr_non_zero · method · L652-L664 — def test_bad_judge_per_class_fnr_non_zero(self) -> None: # Per-class reporting must surface non-zero FNR on the flipped # classes — confirms the per_class accumulator is wired to the # judge output (not hardcoded zero) and would expose which # class drove a real-judge regression.
• _bare_unsupported_judge · function · L672-L704 — def _bare_unsupported_judge() -> Callable[..., dict[str, Any]]
• fn · function · L684-L702 — def fn(**kwargs: Any) -> dict[str, Any]
• _dummy_breakdown_judge · function · L707-L740 — def _dummy_breakdown_judge() -> Callable[..., dict[str, Any]]
• fn · function · L719-L738 — def fn(**kwargs: Any) -> dict[str, Any]
• _full_claim_text_judge · function · L743-L774 — def _full_claim_text_judge() -> Callable[..., dict[str, Any]]
• fn · function · L754-L772 — def fn(**kwargs: Any) -> dict[str, Any]
• PartialSupportSubsetMetric · class · L777-L878 — class PartialSupportSubsetMetric(unittest.TestCase)
• setUp · method · L788-L792 — def setUp(self) -> None
• test_gold_set_has_at_least_five_partial_fixtures · method · L794-L799 — def test_gold_set_has_at_least_five_partial_fixtures(self) -> None
• test_report_has_partial_support_block · method · L801-L805 — def test_report_has_partial_support_block(self) -> None
• test_perfect_judge_passes_partial_subset · method · L807-L815 — def test_perfect_judge_passes_partial_subset(self) -> None: # A judge that emits UNSUPPORTED + a well-formed breakdown misses nothing. # Proves the metric is not trivially always-failing.
• test_bare_unsupported_judge_misses_partial_subset · method · L817-L833 — def test_bare_unsupported_judge_misses_partial_subset(self) -> None: # The regression case: aggregate FNR stays under threshold (label matches), # but the subset miss_rate is non-zero because no breakdown was emitted.
• test_partial_fixtures_declare_expected_sub_claims · method · L835-L847 — def test_partial_fixtures_declare_expected_sub_claims(self) -> None: # #213 P1-3: each partial fixture must declare expected_sub_claims so the # subset metric can verify the judge decomposed THIS claim, not just # emitted any two-line breakdown.
• test_full_claim_text_judge_misses_partial_subset · method · L849-L861 — def test_full_claim_text_judge_misses_partial_subset(self) -> None: # Ship-gate round-2: a judge returning two items each = the FULL claim text # (one SUPPORTED, one UNSUPPORTED) passes the verdict-mix gate and contains # every expected token, but is not an atomic decomposition. The atomicity + # distinct-item rules must catch it -> miss_rate = 1.0.
• test_dummy_breakdown_judge_misses_partial_subset · method · L863-L878 — def test_dummy_breakdown_judge_misses_partial_subset(self) -> None: # #213 P1-3: a judge that emits a well-formed-SHAPED but generic two-line # breakdown on every partial fixture passes the aggregate AND the shape # gate, but must FAIL the subset content-match (it never decomposed the # actual claim). This is the gap a later review round flagged: shape-only # verification was gameable.