scripts/test_run_evals.py

• schema · function · L20-L21 — def schema()
• validator · function · L25-L27 — def validator(schema)
• _make_task · function · L30-L48 — def _make_task(tmp_path: Path, *, expected: dict, manifest: dict, tuples: dict | None = None) -> Path
• _citation_manifest · function · L51-L74 — def _citation_manifest(task_name="citation_extraction")
• _ro · function · L77-L94 — def _ro(crossref=None, openalex=None, semantic_scholar=None, arxiv=None)
• cell · function · L82-L88 — def cell(v)
• test_output_validates_against_schema · function · L100-L103 — def test_output_validates_against_schema(validator)
• test_run_id_harness_version_gold_set_version_present · function · L106-L112 — def test_run_id_harness_version_gold_set_version_present()
• test_aggregate_accuracy_on_stub · function · L118-L128 — def test_aggregate_accuracy_on_stub(tmp_path, monkeypatch)
• test_per_class_accuracy_correct · function · L131-L143 — def test_per_class_accuracy_correct(tmp_path): # One wrong "true" tuple (resolvers all skipped -> reducer says unresolvable).
• test_unresolvable_not_collapsed_into_false · function · L146-L162 — def test_unresolvable_not_collapsed_into_false(tmp_path): # A total-outage tuple labeled unresolvable must score as unresolvable, # never as false. If the reducer collapsed it, predicted would be wrong.
• test_expert_concordance_only_over_labeled_subset · function · L168-L193 — def test_expert_concordance_only_over_labeled_subset(tmp_path)
• test_low_concordance_does_not_gate · function · L196-L212 — def test_low_concordance_does_not_gate(tmp_path): # Expert disagrees on the one labeled tuple -> concordance 0.0, but the run # still completes with measured status and aggregate computed normally.
• test_no_task_discovers_all_manifests · function · L218-L221 — def test_no_task_discovers_all_manifests()
• test_discover_runs_all · function · L224-L230 — def test_discover_runs_all(validator)
• test_rq_framing_dispatch_shape · function · L236-L245 — def test_rq_framing_dispatch_shape(validator)
• test_baseline_compare_populates_lift · function · L251-L259 — def test_baseline_compare_populates_lift(validator)
• test_missing_entrypoint_skips_cleanly · function · L265-L285 — def test_missing_entrypoint_skips_cleanly(tmp_path, validator)
• test_run_task_unknown_task_does_not_raise · function · L288-L292 — def test_run_task_unknown_task_does_not_raise(tmp_path)
• test_surface_form_parity_is_pending_not_silently_passing · function · L295-L304 — def test_surface_form_parity_is_pending_not_silently_passing()
• test_implemented_task_corrupt_gold_raises · function · L310-L322 — def test_implemented_task_corrupt_gold_raises(tmp_path): # citation_extraction has a native measurer -> a corrupt expected_outcomes.json # must raise TaskExecutionError, not silently downgrade to a "pending" result.
• test_implemented_task_missing_gold_raises · function · L325-L336 — def test_implemented_task_missing_gold_raises(tmp_path): # Manifest present (so it is discovered + dispatched) but the gold artifact # the measurer reads is absent -> raise, never pending.
• test_native_task_missing_manifest_raises · function · L339-L345 — def test_native_task_missing_manifest_raises(tmp_path): # A NATIVE task whose manifest.yaml has vanished must raise — a vanished # implemented-task asset is a real defect, not a silent skip (codex R2 P1).
• test_citation_missing_tuple_dir_raises · function · L348-L363 — def test_citation_missing_tuple_dir_raises(tmp_path): # expected_outcomes.json present but tuples/ dir absent must RAISE, not # produce a 0-tuple "measured" run that silently scores nothing (codex R2 P2).
• test_discover_always_includes_native_tasks · function · L366-L373 — def test_discover_always_includes_native_tasks(tmp_path): # An empty gold root still yields the native tasks, so a vanished native # manifest is attempted (and raises in run_task) rather than dropped.
• test_absent_phase2_task_still_pending_not_raise · function · L376-L391 — def test_absent_phase2_task_still_pending_not_raise(tmp_path, validator): # A genuinely not-yet-shipped Phase-2 task (no native measurer) is a # legitimate pending — the raise path must NOT catch this case.
• _task · function · L397-L402 — def _task(name, *, status="measured", passed=None, metric="accuracy")
• test_gate_flags_below_threshold_task · function · L405-L407 — def test_gate_flags_below_threshold_task()
• test_gate_passes_when_all_above_threshold · function · L410-L414 — def test_gate_passes_when_all_above_threshold()
• test_gate_ignores_pending_and_thresholdless_tasks · function · L417-L424 — def test_gate_ignores_pending_and_thresholdless_tasks()
• test_gate_real_run_passes_threshold · function · L427-L430 — def test_gate_real_run_passes_threshold(): # End-to-end: the live gold sets meet their thresholds, so the gate is empty.