#!/usr/bin/env python3
"""Tests for verify_submission_package — #394 Slice 1 (CLI skeleton + Family C).
Spec: docs/design/2026-06-10-394-submission-package-verifier-spec.md §3.3 / §5.1
/ §7.3 / §8. Mutation discipline per repo convention: every check has a fixture
that fails it and a test proving the failure fires.
"""
from __future__ import annotations
import json
import shutil
import zipfile
from pathlib import Path
import jsonschema
import pytest
import yaml
from verify_submission_package import run
REPO_ROOT = Path(__file__).resolve().parent.parent
FIXTURES = REPO_ROOT / "scripts" / "fixtures" / "submission_package"
SCHEMA_PATH = (
REPO_ROOT / "shared" / "contracts" / "submission"
/ "submission_verification_report.schema.json"
)
REPORT_BASENAME = "submission_verification_report.json"
def load_schema():
return json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
def run_dir(package_dir, extra_args=()):
"""Run the CLI on a package dir; returns (exit_code, report_dict)."""
rc = run([str(package_dir), *extra_args])
report_path = package_dir / REPORT_BASENAME
report = (
json.loads(report_path.read_text(encoding="utf-8"))
if report_path.is_file() else None
)
return rc, report
def run_on(fixture_name, tmp_path, extra_args=()):
"""Copy a fixture package into tmp and run the CLI on the copy.
Returns (exit_code, report_dict, package_dir). The copy keeps the repo
fixture pristine (the CLI writes its report into the package dir).
"""
package_dir = tmp_path / fixture_name
shutil.copytree(FIXTURES / fixture_name, package_dir)
rc, report = run_dir(package_dir, extra_args)
return rc, report, package_dir
def checks_by_id(report):
return {c["id"]: c for c in report["checks"]}
# --- Round 1: clean package, joined marker path -----------------------------
def test_clean_package_family_c_passes(tmp_path):
# Without a venue profile the Family B checks are NOT-CHECKED (§3.2), so
# the honest exit code is 3 ("passed what was checkable", §8) — Family C
# itself is fully green.
rc, report, _ = run_on("clean", tmp_path)
assert rc == 3
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "pass"
assert by_id["C2"]["status"] == "pass"
def test_clean_package_is_deterministic_joined_marker(tmp_path):
_, report, _ = run_on("clean", tmp_path)
assert report["header"]["extraction_path"] == "joined_marker"
by_id = checks_by_id(report)
for cid in ("C1", "C2"):
assert by_id[cid]["family"] == "reference_integrity"
assert by_id[cid]["signal_class"] == "deterministic"
# strict_eligible is class-level: C1 promotable, C2 (warn-only) never —
# asserted in test_C2_is_never_strict_eligible.
# --- Slice 2: Family B venue limits ------------------------------------------
def test_no_venue_profile_family_b_not_checked(tmp_path):
# §3.2: without a venue profile every Family B check is
# NOT-CHECKED(no venue profile) — never guessed from the journal name
# (R-L3-2-D mirror). The checks stay visible (deterministic,
# strict-eligible) so the slice-4 fail-closed path has something to see.
rc, report, _ = run_on("clean", tmp_path)
assert rc == 3
by_id = checks_by_id(report)
for cid in ("B1", "B2", "B3", "B4", "B5"):
assert by_id[cid]["status"] == "not_checked"
assert "no venue profile" in by_id[cid]["detail"]
assert by_id[cid]["family"] == "venue_limits"
assert by_id[cid]["signal_class"] == "deterministic"
assert by_id[cid]["strict_eligible"] is True
assert report["header"]["not_checked_count"] == 5
jsonschema.validate(report, load_schema())
def test_clean_report_validates_against_schema(tmp_path):
_, report, _ = run_on("clean", tmp_path)
jsonschema.validate(report, load_schema())
def test_policy_slug_is_null_in_standalone_runs(tmp_path):
# §5.2/§5.3: the script never reads terminal_policies; the slug is stamped
# by the slice-4 orchestrator hook. A standalone run always emits null.
_, report, _ = run_on("clean", tmp_path)
assert report["header"]["policy_slug"] is None
def test_report_written_into_package_dir(tmp_path):
_, _, package_dir = run_on("clean", tmp_path)
assert (package_dir / REPORT_BASENAME).is_file()
def test_full_profile_all_family_b_pass(tmp_path):
# venue_clean satisfies every limit in profiles/full.yaml; with both
# families green the exit code is a true 0.
profile = FIXTURES / "profiles" / "full.yaml"
rc, report, _ = run_on("venue_clean", tmp_path,
extra_args=["--venue-profile", str(profile)])
assert rc == 0
by_id = checks_by_id(report)
for cid in ("B1", "B2", "B3", "B4", "B5", "C1", "C2"):
assert by_id[cid]["status"] == "pass", (cid, by_id[cid]["detail"])
# §3.2: the word-count method is declared in the report, never implied
# venue-exact.
assert "whitespace-split" in by_id["B1"]["detail"]
assert "body_only" in by_id["B1"]["detail"]
assert report["header"]["not_checked_count"] == 0
jsonschema.validate(report, load_schema())
def test_violated_profile_every_family_b_check_fails(tmp_path):
# Mutation discipline (§8): every Family B check has a fixture that fails
# it. venue_violations breaks all five limits in profiles/tight.yaml.
profile = FIXTURES / "profiles" / "tight.yaml"
rc, report, _ = run_on("venue_violations", tmp_path,
extra_args=["--venue-profile", str(profile)])
assert rc == 1
by_id = checks_by_id(report)
for cid in ("B1", "B2", "B3", "B4", "B5"):
assert by_id[cid]["status"] == "fail", (cid, by_id[cid]["detail"])
assert by_id[cid]["signal_class"] == "deterministic"
assert by_id[cid]["strict_eligible"] is True
assert "Data Availability" in by_id["B4"]["detail"]
# Family C stays green — the violations are venue limits, not integrity.
assert by_id["C1"]["status"] == "pass"
jsonschema.validate(report, load_schema())
def test_partial_profile_runs_what_it_can(tmp_path):
# §4: a partially-declared profile runs the checks it can and
# NOT-CHECKEDs the rest, each with the undeclared field named.
profile = tmp_path / "partial.yaml"
profile.write_text(
"word_limit: 200\ndeclared_by: scholar\n", encoding="utf-8")
rc, report, _ = run_on("venue_clean", tmp_path,
extra_args=["--venue-profile", str(profile)])
assert rc == 3
by_id = checks_by_id(report)
assert by_id["B1"]["status"] == "pass"
for cid, field in (("B2", "abstract_word_limit"),
("B3", "keyword_range"),
("B4", "required_sections"),
("B5", "reference_limit")):
assert by_id[cid]["status"] == "not_checked"
assert f"{field} not declared" in by_id[cid]["detail"]
def test_word_count_tolerance_two_percent(tmp_path):
# §3.2: ±2% tolerance before fail (format-conversion noise). 101 words
# against a 100 limit passes; 103 fails.
profile = tmp_path / "profile.yaml"
profile.write_text(
"word_limit: 100\ndeclared_by: scholar\n", encoding="utf-8")
for n_words, expected in ((101, "pass"), (103, "fail")):
package = tmp_path / f"pkg{n_words}"
package.mkdir()
(package / "paper.md").write_text(
" ".join(["word"] * n_words) + "\n", encoding="utf-8")
run([str(package), "--venue-profile", str(profile)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
assert checks_by_id(report)["B1"]["status"] == expected, n_words
def test_invalid_venue_profile_is_usage_error(tmp_path):
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("# x\n", encoding="utf-8")
cases = (
"word_limit: 100\n", # no declared_by
"declared_by: tool\n", # wrong provenance
"declared_by: scholar\nword_count_scope: detexed\n", # bad enum
"declared_by: scholar\nkeyword_range: {min: 5, max: 2}\n", # min>max
"declared_by: scholar\nword_limit: -3\n", # bad int
)
for body in cases:
profile = tmp_path / "bad.yaml"
profile.write_text(body, encoding="utf-8")
assert run([str(package), "--venue-profile", str(profile)]) == 2, body
def test_missing_abstract_and_keywords_not_checked(tmp_path):
# Declared limits whose actuals cannot be located are NOT-CHECKED with the
# reason — never folded into pass (§1.4), never guessed.
profile = tmp_path / "profile.yaml"
profile.write_text(
"abstract_word_limit: 50\nkeyword_range: {min: 1, max: 5}\n"
"declared_by: scholar\n", encoding="utf-8")
rc, report, _ = run_on("clean", tmp_path,
extra_args=["--venue-profile", str(profile)])
by_id = checks_by_id(report)
assert by_id["B2"]["status"] == "not_checked"
assert "no abstract section" in by_id["B2"]["detail"]
assert by_id["B3"]["status"] == "not_checked"
assert "no keywords line" in by_id["B3"]["detail"]
def test_latex_manuscript_word_count_declares_detex(tmp_path):
# §10 item 4 (adjudicated at slice 2): LaTeX counting = naive detex +
# whitespace-split, the method is declared in the report, never promised
# venue-exact.
profile = tmp_path / "profile.yaml"
profile.write_text(
"word_limit: 5\ndeclared_by: scholar\n", encoding="utf-8")
rc, report, _ = run_on("fallback_latex", tmp_path,
extra_args=["--venue-profile", str(profile)])
by_id = checks_by_id(report)
assert by_id["B1"]["status"] == "fail"
assert "naive detex" in by_id["B1"]["detail"]
assert by_id["B1"]["location"] == "paper.tex"
def test_word_count_scope_all_counts_everything(tmp_path):
# codex P2: `all` must actually count everything (keywords line included);
# only body_only / body_plus_references exclude it. 10 body words +
# "**Keywords:** a, b" (3 tokens) against limit 10: body_only passes,
# all fails.
base = " ".join(["word"] * 10) + "\n\n**Keywords:** alpha, beta\n"
for scope, expected in (("body_only", "pass"), ("all", "fail")):
package = tmp_path / f"pkg_{scope}"
package.mkdir()
(package / "paper.md").write_text(base, encoding="utf-8")
profile = tmp_path / f"{scope}.yaml"
profile.write_text(
f"word_limit: 10\nword_count_scope: {scope}\n"
"declared_by: scholar\n", encoding="utf-8")
run([str(package), "--venue-profile", str(profile)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
assert checks_by_id(report)["B1"]["status"] == expected, scope
def test_profile_validation_matches_schema_strictness(tmp_path):
# codex P2: the CLI gate must not be looser than
# venue_profile.schema.json — unknown fields (additionalProperties false)
# and booleans-as-integers are rejected.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("# x\n", encoding="utf-8")
cases = (
"declared_by: scholar\nword_limt: 100\n", # unknown field (typo)
"declared_by: scholar\nword_limit: true\n", # bool is not an int
"declared_by: scholar\nvenue_name: 42\n", # venue_name not a string
"declared_by: scholar\nkeyword_range: {min: -1, max: 2}\n", # min < 0
)
for body in cases:
profile = tmp_path / "bad.yaml"
profile.write_text(body, encoding="utf-8")
assert run([str(package), "--venue-profile", str(profile)]) == 2, body
def test_ambiguous_manuscript_not_checked(tmp_path):
# codex P2: with several non-canonical candidates the verifier must not
# silently pick the wordiest (it could be a response letter); it reports
# NOT-CHECKED(ambiguous manuscript). A canonical name (paper.* /
# manuscript.* / main.*) resolves the ambiguity.
profile = tmp_path / "profile.yaml"
profile.write_text(
"word_limit: 100\ndeclared_by: scholar\n", encoding="utf-8")
package = tmp_path / "pkg"
package.mkdir()
(package / "chapter_one.md").write_text("alpha " * 20, encoding="utf-8")
(package / "rejoinder.md").write_text("beta " * 30, encoding="utf-8")
run([str(package), "--venue-profile", str(profile)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
b1 = checks_by_id(report)["B1"]
assert b1["status"] == "not_checked"
assert "ambiguous manuscript" in b1["detail"]
(package / "paper.md").write_text("gamma " * 10, encoding="utf-8")
run([str(package), "--venue-profile", str(profile)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
b1 = checks_by_id(report)["B1"]
assert b1["status"] == "pass"
assert b1["location"] == "paper.md"
def test_abstract_tolerance_and_b5_no_reference_list(tmp_path):
# codex P3: pin the B2 ±2% tolerance and the B5
# declared-limit-but-no-reference-list branch.
profile = tmp_path / "profile.yaml"
profile.write_text(
"abstract_word_limit: 100\nreference_limit: 5\n"
"declared_by: scholar\n", encoding="utf-8")
for n_words, expected in ((101, "pass"), (103, "fail")):
package = tmp_path / f"pkg{n_words}"
package.mkdir()
(package / "paper.md").write_text(
"## Abstract\n\n" + " ".join(["word"] * n_words) + "\n",
encoding="utf-8")
run([str(package), "--venue-profile", str(profile)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
by_id = checks_by_id(report)
assert by_id["B2"]["status"] == expected, n_words
assert by_id["B5"]["status"] == "not_checked"
assert "no machine-readable reference list" in by_id["B5"]["detail"]
def test_profile_with_no_manuscript_not_checked(tmp_path):
profile = tmp_path / "profile.yaml"
profile.write_text(
"word_limit: 100\ndeclared_by: scholar\n", encoding="utf-8")
package = tmp_path / "pkg"
package.mkdir()
(package / "figure.png").write_bytes(b"\x89PNG\r\n")
run([str(package), "--venue-profile", str(profile)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
b1 = checks_by_id(report)["B1"]
assert b1["status"] == "not_checked"
assert "no manuscript found" in b1["detail"]
# --- Slice 3: Family A blind-review residue ----------------------------------
def test_family_a_not_applicable_without_trigger(tmp_path):
# §3.1: the residue scan runs only when the package contains an anonymized
# variant or the profile declares double-blind. Untriggered checks are
# not_applicable — visibly distinct from not_checked (they did not need to
# run), so a single-blind package is not condemned to exit 3 forever.
rc, report, _ = run_on("clean", tmp_path)
by_id = checks_by_id(report)
for cid in ("A1", "A2", "A3", "A4", "A5", "A6", "A7"):
assert by_id[cid]["status"] == "not_applicable", cid
assert "not triggered" in by_id[cid]["detail"]
assert by_id[cid]["family"] == "blind_review_residue"
# not_applicable is not incompleteness: it does not count as not_checked.
assert report["header"]["not_checked_count"] == 5 # the B checks only
jsonschema.validate(report, load_schema())
def test_declared_double_blind_without_variant_fails_A7(tmp_path):
# §3.1: a declared-double-blind package with NO anonymized variant is
# itself a fail — the most basic residue of all, the blind version is
# missing. A1-A6 have nothing to scan: not_checked.
profile = tmp_path / "double.yaml"
profile.write_text(
"blind_review: double\ndeclared_by: scholar\n", encoding="utf-8")
rc, report, _ = run_on("clean", tmp_path,
extra_args=["--venue-profile", str(profile)])
assert rc == 1
by_id = checks_by_id(report)
assert by_id["A7"]["status"] == "fail"
assert "anonymized manuscript variant" in by_id["A7"]["detail"]
assert by_id["A7"]["strict_eligible"] is True
for cid in ("A1", "A2", "A3", "A4", "A5", "A6"):
assert by_id[cid]["status"] == "not_checked", cid
assert "no anonymized variant" in by_id[cid]["detail"]
def test_anonymized_variant_triggers_family_a(tmp_path):
# Presence of an anonymized variant triggers the scan even without a
# profile (presence-or-declaration, §3.1). With an .md-only variant the
# PDF/DOCX metadata checks have no object: not_applicable.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text(
"Body (Smith, 2024) .\n", encoding="utf-8")
shutil.copy(FIXTURES / "clean" / "references.bib",
package / "references.bib")
(package / "paper_anonymized.md").write_text(
"# T\n\nBody text without identity.\n", encoding="utf-8")
rc, report = run_dir(package)
by_id = checks_by_id(report)
assert by_id["A7"]["status"] == "pass"
for cid in ("A1", "A2", "A3"):
assert by_id[cid]["status"] == "not_applicable", cid
assert "no PDF" in by_id[cid]["detail"] or "no DOCX" in by_id[cid]["detail"]
jsonschema.validate(report, load_schema())
_DOCX_CORE_TMPL = (
''
''
"{creator}"
"{last_modified_by}"
"")
_DOCX_DOC_TMPL = (
''
''
"Blind body text.{extra}"
"")
def make_docx(path, creator="", last_modified_by="", ins_author=None,
comment_author=None):
"""Minimal raw-structure .docx (a zip with core.xml + document.xml [+
comments.xml]) — exactly the parts the residue scan reads (§1.3: the
deliverable is the raw file, not the rendered view)."""
extra = ""
if ins_author:
extra = (f''
"inserted")
with zipfile.ZipFile(path, "w") as z:
z.writestr("docProps/core.xml", _DOCX_CORE_TMPL.format(
creator=creator, last_modified_by=last_modified_by))
z.writestr("word/document.xml", _DOCX_DOC_TMPL.format(extra=extra))
if comment_author:
z.writestr(
"word/comments.xml",
''
''
f''
"note"
"")
def _blind_package(tmp_path, **docx_kwargs):
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
make_docx(package / "paper_anonymized.docx", **docx_kwargs)
rc, report = run_dir(package)
return checks_by_id(report)
def test_docx_metadata_author_residue_fails_A2(tmp_path):
# A2 (§3.1): docProps/core.xml creator / lastModifiedBy non-empty in the
# blind variant is a deterministic fact about the file — invisible to any
# rendered-view scan (§1.3).
by_id = _blind_package(tmp_path, creator="Jordan Smith")
assert by_id["A2"]["status"] == "fail"
assert "Jordan Smith" in by_id["A2"]["detail"]
assert by_id["A2"]["strict_eligible"] is True
assert by_id["A3"]["status"] == "pass"
def test_docx_clean_metadata_passes_A2_A3(tmp_path):
by_id = _blind_package(tmp_path)
assert by_id["A2"]["status"] == "pass"
assert by_id["A3"]["status"] == "pass"
def test_docx_tracked_change_author_fails_A3(tmp_path):
by_id = _blind_package(tmp_path, ins_author="J. Smith")
assert by_id["A3"]["status"] == "fail"
assert "J. Smith" in by_id["A3"]["detail"]
def test_docx_comment_author_fails_A3(tmp_path):
by_id = _blind_package(tmp_path, comment_author="Reviewer Zero")
assert by_id["A3"]["status"] == "fail"
assert "Reviewer Zero" in by_id["A3"]["detail"]
def test_corrupt_docx_not_checked(tmp_path):
# An unreadable artifact is incompleteness, never folded into pass (§1.4).
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "paper_anonymized.docx").write_bytes(b"not a zip")
_rc, report = run_dir(package)
by_id = checks_by_id(report)
for cid in ("A2", "A3"):
assert by_id[cid]["status"] == "not_checked"
assert "unreadable" in by_id[cid]["detail"]
def test_pdf_metadata_author_fails_A1(tmp_path):
pypdf = pytest.importorskip("pypdf")
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
writer = pypdf.PdfWriter()
writer.add_blank_page(width=72, height=72)
writer.add_metadata({"/Author": "Jordan Smith"})
with open(package / "paper_blind.pdf", "wb") as f:
writer.write(f)
_rc, report = run_dir(package)
a1 = checks_by_id(report)["A1"]
assert a1["status"] == "fail"
assert "Jordan Smith" in a1["detail"]
assert a1["strict_eligible"] is True
def test_pdf_parser_unavailable_is_not_checked(tmp_path, monkeypatch):
# §1.4/#349: a missing parser must surface as NOT-CHECKED, never read as
# covered. (The slice-4 strict mode turns this into
# VERIFICATION-INCOMPLETE.)
import verify_submission_package as v
monkeypatch.setattr(v, "pypdf", None)
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "paper_blind.pdf").write_bytes(b"%PDF-1.4\n%%EOF\n")
_rc, report = run_dir(package)
a1 = checks_by_id(report)["A1"]
assert a1["status"] == "not_checked"
assert "pypdf" in a1["detail"]
def test_A4_strict_only_when_profile_forbids_acknowledgments(tmp_path):
# §3.1 load-bearing: A4's SIGNAL is deterministic but the judgment is the
# scholar's — strict-eligible ONLY when the venue profile explicitly
# declares acknowledgments must be removed from the blind version.
def build(profile_body):
package = tmp_path / f"pkg{len(profile_body)}"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "paper_anonymized.md").write_text(
"# Title\n\n## Acknowledgments\n\nWe thank our colleagues.\n",
encoding="utf-8")
profile = tmp_path / f"p{len(profile_body)}.yaml"
profile.write_text(profile_body, encoding="utf-8")
run([str(package), "--venue-profile", str(profile)])
return json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
base = "blind_review: double\ndeclared_by: scholar\n"
a4 = checks_by_id(build(base))["A4"]
assert a4["status"] == "fail"
assert a4["strict_eligible"] is False
a4 = checks_by_id(build(
base + "acknowledgments_forbidden_in_blind: true\n"))["A4"]
assert a4["status"] == "fail"
assert a4["strict_eligible"] is True
def test_A6_filename_leakage_from_original_metadata(tmp_path):
# §3.1 A6: author-name tokens harvested from the NON-anonymized artifact's
# metadata, matched against package filenames (heuristic — coincidental
# tokens can false-positive). The metadata-source original itself is not
# scanned (its identified name is expected).
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
make_docx(package / "paper.docx", creator="Jordan Smith")
make_docx(package / "paper_anonymized.docx")
(package / "smith_appendix.csv").write_text("x\n", encoding="utf-8")
_rc, report = run_dir(package)
a6 = checks_by_id(report)["A6"]
assert a6["status"] == "fail"
assert "smith_appendix.csv" in a6["detail"]
assert a6["signal_class"] == "heuristic"
assert a6["strict_eligible"] is False
def test_A6_without_original_metadata_not_checked(tmp_path):
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "paper_anonymized.md").write_text("Body.\n", encoding="utf-8")
_rc, report = run_dir(package)
a6 = checks_by_id(report)["A6"]
assert a6["status"] == "not_checked"
assert "no author" in a6["detail"]
def test_A5_zh_tw_self_citation_phrase(tmp_path):
# §10 item 1: the phrasing list must not be anglophone-only.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "paper_anonymized.md").write_text(
"# 匿名稿\n\n如我們先前的研究所示,品保回饋迴圈存在斷點。\n",
encoding="utf-8")
_rc, report = run_dir(package)
a5 = checks_by_id(report)["A5"]
assert a5["status"] == "fail"
assert "我們先前的研究" in a5["detail"]
def test_A5_zh_tw_third_person_author_self_reference(tmp_path):
# zh-TW curation (#394 follow-up): third-person self-reference via
# 本文作者 — the bare 作者先前 would false-positive on 該作者先前
# (a cited third party's author), so the phrase is anchored on the
# 本文 prefix.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "paper_anonymized.md").write_text(
"# 匿名稿\n\n本文作者先前已就品保回饋迴圈提出分析架構。\n",
encoding="utf-8")
_rc, report = run_dir(package)
a5 = checks_by_id(report)["A5"]
assert a5["status"] == "fail"
assert "本文作者先前" in a5["detail"]
def test_A5_other_authors_prior_work_not_flagged(tmp_path):
# 該作者先前 refers to a cited third party, not the manuscript's own
# authors — must stay clean (the reason the phrase list does not carry
# a bare 作者先前 entry).
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "paper_anonymized.md").write_text(
"# 匿名稿\n\n該作者先前的研究指出品保回饋迴圈存在斷點。\n",
encoding="utf-8")
_rc, report = run_dir(package)
a5 = checks_by_id(report)["A5"]
assert a5["status"] == "pass"
# --- codex slice-3 review round ----------------------------------------------
def test_A4_docx_only_variant_is_not_checked_not_applicable(tmp_path):
# codex P1: a DOCX-only blind variant means the acknowledgments scan
# SHOULD run but cannot (no text-form variant to read headings from) —
# that is not_checked honesty, never not_applicable masquerade.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
make_docx(package / "paper_anonymized.docx")
_rc, report = run_dir(package)
a4 = checks_by_id(report)["A4"]
assert a4["status"] == "not_checked"
assert "text-form" in a4["detail"]
def test_A7_not_satisfied_by_blind_named_supplement(tmp_path):
# codex P1: a declared double-blind package whose only blind-named file is
# an ancillary CSV has NO blind manuscript variant — A7 must still fail.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
(package / "blind_survey.csv").write_text("q,a\n", encoding="utf-8")
profile = tmp_path / "double.yaml"
profile.write_text(
"blind_review: double\ndeclared_by: scholar\n", encoding="utf-8")
rc = run([str(package), "--venue-profile", str(profile)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
assert rc == 1
assert checks_by_id(report)["A7"]["status"] == "fail"
def test_untriggered_A4_is_never_strict_eligible(tmp_path):
# codex P1: A4's eligibility comes ONLY from the explicit profile
# declaration — including on the untriggered/not_applicable path.
_rc, report, _ = run_on("clean", tmp_path)
assert checks_by_id(report)["A4"]["strict_eligible"] is False
def test_A6_token_match_does_not_flag_substrings(tmp_path):
# codex P2: token-to-token matching — `Smith` must not flag
# `blacksmith_notes.md`.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
make_docx(package / "paper.docx", creator="Jordan Smith")
make_docx(package / "paper_anonymized.docx")
(package / "blacksmith_notes.md").write_text("x\n", encoding="utf-8")
_rc, report = run_dir(package)
a6 = checks_by_id(report)["A6"]
assert a6["status"] == "pass", a6["detail"]
def test_oversized_docx_part_is_not_checked(tmp_path, monkeypatch):
# codex P2: unbounded zip reads are a zip-bomb exposure; oversized parts
# are reported as unreadable incompleteness, never scanned or passed.
import verify_submission_package as v
monkeypatch.setattr(v, "_MAX_XML_PART_BYTES", 64)
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("Body.\n", encoding="utf-8")
make_docx(package / "paper_anonymized.docx",
creator="x" * 200) # inflates core.xml past the test cap
_rc, report = run_dir(package)
a2 = checks_by_id(report)["A2"]
assert a2["status"] == "not_checked"
assert "unreadable" in a2["detail"]
def test_profile_rejects_nonboolean_acknowledgments_forbidden_field(tmp_path):
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("# x\n", encoding="utf-8")
profile = tmp_path / "p.yaml"
profile.write_text(
"declared_by: scholar\nacknowledgments_forbidden_in_blind: maybe\n",
encoding="utf-8")
assert run([str(package), "--venue-profile", str(profile)]) == 2
def test_schema_accepts_not_applicable():
# (Unknown statuses are rejected by test_schema_rejects_unknown_status.)
ok = _minimal_report(status="not_applicable")
jsonschema.validate(ok, load_schema())
def test_venue_profile_fixtures_validate_against_schema():
profile_schema = json.loads(
(REPO_ROOT / "shared" / "contracts" / "submission"
/ "venue_profile.schema.json").read_text(encoding="utf-8"))
for name in ("full.yaml", "tight.yaml"):
profile = yaml.safe_load(
(FIXTURES / "profiles" / name).read_text(encoding="utf-8"))
jsonschema.validate(profile, profile_schema)
# --- Round 2: fail / warn / NOT-CHECKED paths + exit codes -------------------
def test_orphan_intext_citation_fails_C1_exit_1(tmp_path):
rc, report, _ = run_on("orphan_intext", tmp_path)
assert rc == 1
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "fail"
assert "ghost2024" in by_id["C1"]["detail"]
assert by_id["C1"]["location"] == "paper.md"
# The orphan is deterministic-classed on the joined marker path.
assert by_id["C1"]["signal_class"] == "deterministic"
assert by_id["C1"]["strict_eligible"] is True
jsonschema.validate(report, load_schema())
def test_uncited_reference_entry_warns_C2_exit_0(tmp_path):
# §3.3: uncited reference entry = warn (some venues allow further-reading
# entries) — advisory, never a fail exit (3 = Family B not checked).
rc, report, _ = run_on("uncited_reference", tmp_path)
assert rc == 3
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "pass"
assert by_id["C2"]["status"] == "warn"
assert "chenlee2023" in by_id["C2"]["detail"]
def test_markers_without_join_source_not_checked_exit_3(tmp_path):
# §3.3 + §8 join test: markers present, passport supplies a corpus (a
# reference list) but NO citation_verification_summary — never a guessed
# comparison.
passport = FIXTURES / "passports" / "corpus_only.yaml"
rc, report, _ = run_on("marker_no_join", tmp_path,
extra_args=["--passport", str(passport)])
assert rc == 3
by_id = checks_by_id(report)
for cid in ("C1", "C2"):
assert by_id[cid]["status"] == "not_checked"
assert "missing prose-reference join" in by_id[cid]["detail"]
assert report["header"]["not_checked_count"] == 7 # 2 C + 5 B (no profile)
assert report["header"]["extraction_path"] == "none"
jsonschema.validate(report, load_schema())
def test_join_map_resolves_the_no_join_case(tmp_path):
# The explicit scholar-supplied join map is a valid join source (§3.3) and
# joins the prose slug to the corpus citation_key.
passport = FIXTURES / "passports" / "corpus_only.yaml"
join = tmp_path / "join.yaml"
join.write_text("smith-feedback-2024: smith2024\n", encoding="utf-8")
rc, report, _ = run_on(
"marker_no_join", tmp_path,
extra_args=["--passport", str(passport), "--join-map", str(join)])
assert rc == 3 # Family C green; B not checked (no profile)
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "pass"
assert by_id["C2"]["status"] == "pass"
assert report["header"]["extraction_path"] == "joined_marker"
def test_missing_package_dir_is_usage_error(tmp_path):
assert run([str(tmp_path / "does-not-exist")]) == 2
def test_unparseable_passport_is_usage_error(tmp_path):
bad = tmp_path / "bad.yaml"
bad.write_text("just a string\n", encoding="utf-8")
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text("# x\n", encoding="utf-8")
assert run([str(package), "--passport", str(bad)]) == 2
# --- Round 3: fallback extraction, summary join, fingerprint -----------------
def test_fallback_latex_cite_extraction_is_heuristic_best_effort(tmp_path):
# §3.3: post-converted sources fall back to \cite{} extraction; the header
# downgrades to best-effort and the whole path is heuristic-classed
# (advisory-only) — even a true orphan fail is NOT strict-eligible.
rc, report, _ = run_on("fallback_latex", tmp_path)
assert rc == 1
assert report["header"]["extraction_path"] == "best_effort"
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "fail"
assert "ghost2024" in by_id["C1"]["detail"]
assert "smith2024" not in by_id["C1"]["detail"]
for cid in ("C1", "C2"):
assert by_id[cid]["signal_class"] == "heuristic"
assert by_id[cid]["strict_eligible"] is False
assert by_id["C2"]["status"] == "pass"
jsonschema.validate(report, load_schema())
def test_fallback_authoryear_extraction_matches_bib_metadata(tmp_path):
rc, report, _ = run_on("fallback_authoryear", tmp_path)
assert rc == 1
assert report["header"]["extraction_path"] == "best_effort"
by_id = checks_by_id(report)
# Only the unmatched (Nowhere, 2020) is an orphan; Smith (2024) narrative
# and (Chen & Lee, 2023) parenthetical both join to bib metadata.
assert by_id["C1"]["status"] == "fail"
assert "nowhere" in by_id["C1"]["detail"].lower()
assert "smith" not in by_id["C1"]["detail"].lower()
assert "chen" not in by_id["C1"]["detail"].lower()
# Both bib entries were cited, so C2 passes — and the references section
# itself was not scanned as in-text prose.
assert by_id["C2"]["status"] == "pass"
assert by_id["C1"]["signal_class"] == "heuristic"
def test_summary_join_consumes_real_prose_join(tmp_path):
# The prose slug (smith-feedback-2024) differs from the citation_key
# (smith2024): a pass proves the citation_verification_summary join was
# consumed, not an identity guess (§3.3).
passport = FIXTURES / "passports" / "summary_join.yaml"
rc, report, _ = run_on("summary_join", tmp_path,
extra_args=["--passport", str(passport)])
assert rc == 3 # Family C green; B not checked (no profile)
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "pass"
assert by_id["C2"]["status"] == "pass"
assert report["header"]["extraction_path"] == "joined_marker"
for cid in ("C1", "C2"):
assert by_id[cid]["signal_class"] == "deterministic"
def test_no_machine_readable_reference_list_not_checked(tmp_path):
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text(
"Smith (2024) said things.\n", encoding="utf-8")
rc = run([str(package)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
assert rc == 3
by_id = checks_by_id(report)
for cid in ("C1", "C2"):
assert by_id[cid]["status"] == "not_checked"
assert "no machine-readable reference list" in by_id[cid]["detail"]
def test_fingerprint_follows_audit_snapshot_convention_excluding_report(tmp_path):
# §10 open item 3 (adjudicated at slice 1): `:`
# lines, byte-sorted, trailing newline, fingerprint = sha256 of the
# manifest text; the report file itself is excluded. Pinned here by an
# independent reimplementation.
import hashlib
_, report, package_dir = run_on("clean", tmp_path)
lines = []
for p in sorted(package_dir.rglob("*")):
if not p.is_file() or p.name == REPORT_BASENAME:
continue
digest = hashlib.sha256(p.read_bytes()).hexdigest()
lines.append(f"{p.relative_to(package_dir).as_posix()}:{digest}")
lines.sort()
expected = hashlib.sha256(
("\n".join(lines) + "\n").encode("utf-8")).hexdigest()
assert report["header"]["package_fingerprint"] == expected
def test_fingerprint_stable_across_reruns_with_report_present(tmp_path):
# Second run sees the first run's report inside the package dir; the
# exclusion keeps the fingerprint stable (freshness guard usable, §5.2).
_, first, package_dir = run_on("clean", tmp_path)
run([str(package_dir)])
second = json.loads(
(package_dir / REPORT_BASENAME).read_text(encoding="utf-8"))
assert (second["header"]["package_fingerprint"]
== first["header"]["package_fingerprint"])
# --- Codex review round: P1 partial-join identity guess + P2s ----------------
def test_partial_summary_join_never_falls_back_to_identity(tmp_path):
# P1: a marker slug ABSENT from the join source must never be compared via
# an identity guess — even (especially) when the slug coincidentally equals
# a citation_key in the reference list (§3.3 "never a guessed comparison").
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text(
"Joined (Smith, 2024) .\n"
"Unjoined but key-shaped (Smith, 2024) .\n",
encoding="utf-8")
passport = FIXTURES / "passports" / "summary_join.yaml"
rc = run([str(package), "--passport", str(passport)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
assert rc == 1
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "fail"
assert "no join entry" in by_id["C1"]["detail"]
assert "smith2024" in by_id["C1"]["detail"]
def test_C2_is_never_strict_eligible(tmp_path):
# P2: C2's worst outcome is warn, which is advisory-only and never
# policy-promotable (§5.3) — so the check itself is not strict-eligible,
# even on the deterministic path.
_, report, _ = run_on("clean", tmp_path)
by_id = checks_by_id(report)
assert by_id["C1"]["strict_eligible"] is True
assert by_id["C2"]["strict_eligible"] is False
def test_custom_report_out_inside_package_excluded_from_fingerprint(tmp_path):
# P2: a --report-out path inside the package must be excluded from the
# fingerprint like the default basename, or reruns self-reference.
package = tmp_path / "clean"
shutil.copytree(FIXTURES / "clean", package)
out = package / "custom_report.json"
run([str(package), "--report-out", str(out)])
first = json.loads(out.read_text(encoding="utf-8"))
run([str(package), "--report-out", str(out)])
second = json.loads(out.read_text(encoding="utf-8"))
assert (first["header"]["package_fingerprint"]
== second["header"]["package_fingerprint"])
def test_authoryear_fallback_tolerates_page_locators(tmp_path):
# P3: `Smith (2024, p. 12)` / `(Chen & Lee, 2023, pp. 45–67)` are common
# locator forms; missing them creates avoidable fallback false orphans.
package = tmp_path / "pkg"
package.mkdir()
(package / "paper.md").write_text(
"Smith (2024, p. 12) framed it; details follow "
"(Chen & Lee, 2023, pp. 45–67).\n", encoding="utf-8")
shutil.copy(FIXTURES / "fallback_authoryear" / "references.bib",
package / "references.bib")
rc = run([str(package)])
report = json.loads(
(package / REPORT_BASENAME).read_text(encoding="utf-8"))
by_id = checks_by_id(report)
assert rc == 3 # Family C green; B not checked (no profile)
assert by_id["C1"]["status"] == "pass"
assert by_id["C2"]["status"] == "pass"
def test_schema_rejects_warn_with_strict_eligible():
# P2: warn is advisory-only and never policy-promotable — tightened
# structurally like the heuristic exclusion.
bad = _minimal_report(status="warn", strict_eligible=True)
with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(bad, load_schema())
ok = _minimal_report(status="warn", strict_eligible=False)
jsonschema.validate(ok, load_schema())
# --- Report schema structural contract --------------------------------------
def _minimal_report(**check_overrides):
check = {
"id": "C1",
"family": "reference_integrity",
"signal_class": "deterministic",
"strict_eligible": True,
"status": "pass",
"detail": "ok",
"location": None,
}
check.update(check_overrides)
return {
"header": {
"extraction_path": "joined_marker",
"not_checked_count": 0,
"package_fingerprint": "0" * 64,
"inputs_fingerprint": "0" * 64,
"policy_slug": None,
},
"checks": [check],
}
def test_schema_rejects_heuristic_strict_eligible():
# §3.1/§6: heuristic checks are advisory-only STRUCTURALLY — the schema
# itself forbids the promotion, not just the emitter.
bad = _minimal_report(signal_class="heuristic", strict_eligible=True)
with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(bad, load_schema())
ok = _minimal_report(signal_class="heuristic", strict_eligible=False)
jsonschema.validate(ok, load_schema())
def test_schema_binds_check_id_prefix_to_family():
# The id prefix encodes the family (spec §3 tables); the contract binds
# them so a later-slice emitter cannot ship mismatched pairs.
bad = _minimal_report(id="C1", family="blind_review_residue")
with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(bad, load_schema())
bad2 = _minimal_report(id="A1", family="reference_integrity")
with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(bad2, load_schema())
def test_schema_rejects_unknown_status():
bad = _minimal_report(status="skipped")
with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(bad, load_schema())
# --- Slice 4: terminality (--policy / --check-freshness, spec §5.2/§5.3/§8) --
#
# Terminal signals are STDOUT TOKENS (TERMINAL-BLOCK / VERIFICATION-INCOMPLETE
# / STALE-REPORT), never raw exit codes — exit 1 also carries nonterminal
# advisory/heuristic fails (gate-1 P1). The evaluator keys on STATUS
# (fail / not_checked), never on the strict_eligible bit alone, so
# not_applicable can never compose into a block (slice-3 schema pin).
def _strict_run(fixture_name, tmp_path, capsys, extra_args=()):
rc, report, package_dir = run_on(
fixture_name, tmp_path,
extra_args=["--policy", "strict", *extra_args])
out = capsys.readouterr().out
return rc, report, package_dir, out
def test_strict_eligible_fail_blocks_with_token(tmp_path, capsys):
# orphan_intext fails C1 on the joined marker path (deterministic class).
rc, report, _, out = _strict_run("orphan_intext", tmp_path, capsys)
assert rc == 1
assert "TERMINAL-BLOCK policy=submission_package" in out
assert report["header"]["policy_slug"] == "strict"
def test_strict_heuristic_fail_never_promotes(tmp_path, capsys):
# fallback_latex fails C1 on the FALLBACK path (heuristic class). Under
# strict the heuristic fail must NEVER appear as a TERMINAL-BLOCK
# (§3.1/§6, structural exclusion). The run exits 4, not 1: the
# profileless Family B not_checked outranks the nonterminal fail —
# itself evidence of the fail-closed precedence (a heuristic fail does
# not short-circuit the incompleteness verdict the way a strict fail
# would).
rc, report, _, out = _strict_run("fallback_latex", tmp_path, capsys)
by_id = checks_by_id(report)
assert by_id["C1"]["status"] == "fail"
assert by_id["C1"]["signal_class"] == "heuristic"
assert "TERMINAL-BLOCK" not in out
assert rc == 4 # VERIFICATION-INCOMPLETE from Family B, not the C1 fail
assert "C1" not in next(
l for l in out.splitlines() if "VERIFICATION-INCOMPLETE" in l)
def test_evaluate_policy_unit_contract():
# Unit pin of the evaluator's whole decision table: token + exit per
# policy×status×strict_eligible combination, including precedence (a
# strict fail outranks incomplete) and the not_applicable
# never-composes rule. The advisory/strict divergence lives inside the
# evaluator (it takes the resolved policy), so non-strict policies are
# part of the table.
from verify_submission_package import evaluate_policy
def report_of(*checks):
return {"checks": [
{"id": cid, "strict_eligible": se, "status": status}
for cid, se, status in checks
]}
# Heuristic (strict_eligible=False) fail alone under strict: exit 1,
# NO token.
token, code = evaluate_policy(report_of(("A5", False, "fail"),
("C2", True, "pass")), "strict")
assert token is None and code == 1
# Strict-eligible fail under strict: token + exit 1.
token, code = evaluate_policy(report_of(("B4", True, "fail")), "strict")
assert code == 1 and "TERMINAL-BLOCK policy=submission_package" in token
assert "B4" in token
# The SAME report under advisory / under no policy: never a token —
# the divergence is the evaluator's own contract, not a caller guard.
for policy in ("advisory", None):
token, code = evaluate_policy(report_of(("B4", True, "fail")), policy)
assert token is None and code == 1
# Strict-eligible not_checked under strict: incomplete token + exit 4.
token, code = evaluate_policy(
report_of(("B1", True, "not_checked")), "strict")
assert code == 4 and "VERIFICATION-INCOMPLETE" in token
# ... and under advisory: plain exit 3, no token.
token, code = evaluate_policy(
report_of(("B1", True, "not_checked")), "advisory")
assert token is None and code == 3
# Precedence: strict fail wins over incomplete (block now, the rerun
# after remediation surfaces the rest).
token, code = evaluate_policy(report_of(("B4", True, "fail"),
("B1", True, "not_checked")),
"strict")
assert code == 1 and "TERMINAL-BLOCK" in token
# not_applicable never composes into anything (slice-3 schema pin),
# heuristic not_checked doesn't either.
token, code = evaluate_policy(report_of(("A1", True, "not_applicable"),
("A5", False, "not_checked"),
("C2", True, "pass")), "strict")
assert token is None and code == 3
# All green under strict: clean 0.
token, code = evaluate_policy(report_of(("C1", True, "pass")), "strict")
assert token is None and code == 0
def test_strict_eligible_not_checked_is_verification_incomplete(
tmp_path, capsys):
# venue_clean without a profile: Family B is strict-eligible not_checked
# → fail-closed exit 4 + token (§5.2: a missing input must not silently
# waive the class the scholar opted into blocking on).
rc, report, _, out = _strict_run("venue_clean", tmp_path, capsys)
assert rc == 4
assert "VERIFICATION-INCOMPLETE" in out
assert "TERMINAL-BLOCK" not in out
assert report["header"]["policy_slug"] == "strict"
def test_advisory_same_not_checked_stays_exit_3(tmp_path, capsys):
# Explicit advisory: byte-identical slice-3 behavior except the stamp.
rc, report, _, = run_on("venue_clean", tmp_path,
extra_args=["--policy", "advisory"])
out = capsys.readouterr().out
assert rc == 3
assert "VERIFICATION-INCOMPLETE" not in out
assert "TERMINAL-BLOCK" not in out
assert report["header"]["policy_slug"] == "advisory"
def test_no_policy_flag_stamps_null(tmp_path):
# Standalone unevaluated run: argparse default is None (never "default
# advisory" — gate-1 P1), the stamp stays null.
_, report, _ = run_on("venue_clean", tmp_path)
assert report["header"]["policy_slug"] is None
def test_strict_not_applicable_never_blocks(tmp_path, capsys):
# clean has no anonymized variant and no blind_review declaration: Family
# A is not_applicable (untriggered). Under strict that must NOT read as
# incomplete or block — the evaluator keys on status, and not_applicable
# is neither fail nor not_checked. (Family B absent-profile not_checked
# still yields exit 4 here, proving the discrimination is per-status.)
rc, report, _, out = _strict_run("clean", tmp_path, capsys)
by_id = checks_by_id(report)
assert by_id["A1"]["status"] == "not_applicable"
assert rc == 4 # from Family B not_checked, NOT from Family A
assert "VERIFICATION-INCOMPLETE" in out
token_line = next(l for l in out.splitlines()
if "VERIFICATION-INCOMPLETE" in l)
for aid in (f"A{i}" for i in range(1, 8)):
assert aid not in token_line
def test_strict_full_profile_clean_package_exits_0(tmp_path, capsys):
# The strict happy path: everything checked, everything green.
profile = FIXTURES / "profiles" / "full.yaml"
rc, report, _, out = _strict_run(
"venue_clean", tmp_path, capsys,
extra_args=["--venue-profile", str(profile)])
assert rc == 0
assert "TERMINAL-BLOCK" not in out
assert "VERIFICATION-INCOMPLETE" not in out
assert report["header"]["policy_slug"] == "strict"
# --- Slice 4: freshness guard (--check-freshness, §5.2) ----------------------
def _fresh_args(policy="advisory"):
return ["--check-freshness", "--policy", policy]
def test_freshness_mutated_package_is_stale(tmp_path, capsys):
_, _, package_dir = run_on("venue_clean", tmp_path,
extra_args=["--policy", "advisory"])
manuscript = next(p for p in package_dir.iterdir()
if p.suffix == ".md" and p.name != "provenance_summary.md")
manuscript.write_text(
manuscript.read_text(encoding="utf-8") + "\nDrifted.\n",
encoding="utf-8")
capsys.readouterr()
rc = run([str(package_dir), *_fresh_args()])
out = capsys.readouterr().out
assert rc == 5
assert "STALE-REPORT" in out
def test_freshness_policy_mismatch_is_stale(tmp_path, capsys):
# Report stamped advisory, orchestrator now wants strict: stale, rerun.
_, _, package_dir = run_on("venue_clean", tmp_path,
extra_args=["--policy", "advisory"])
capsys.readouterr()
rc = run([str(package_dir), *_fresh_args(policy="strict")])
out = capsys.readouterr().out
assert rc == 5
assert "STALE-REPORT" in out
def test_freshness_null_stamped_report_never_fresh(tmp_path, capsys):
# A standalone (unevaluated, policy_slug=null) report never satisfies
# pipeline freshness — gate-1 P1 (null must not impersonate advisory).
# The reason token is pinned: null_policy_slug ("you handed the pipeline
# a standalone report") is a different remediation from policy_mismatch
# ("the policy changed since stamping") — without the dedicated branch
# the null case would collapse into the mismatch reason.
_, _, package_dir = run_on("venue_clean", tmp_path) # no --policy
capsys.readouterr()
rc = run([str(package_dir), *_fresh_args()])
out = capsys.readouterr().out
assert rc == 5
assert "STALE-REPORT reason=null_policy_slug" in out
def test_freshness_missing_report_is_stale(tmp_path, capsys):
package_dir = tmp_path / "pkg"
package_dir.mkdir()
(package_dir / "paper.md").write_text("Body.\n", encoding="utf-8")
rc = run([str(package_dir), *_fresh_args()])
out = capsys.readouterr().out
assert rc == 5
assert "STALE-REPORT" in out
def test_freshness_requires_policy(tmp_path, capsys):
# --check-freshness without --policy is a usage error (exit 2): freshness
# is always relative to an expected policy, never free-floating.
_, _, package_dir = run_on("venue_clean", tmp_path,
extra_args=["--policy", "advisory"])
capsys.readouterr()
rc = run([str(package_dir), "--check-freshness"])
err = capsys.readouterr().err
assert rc == 2
assert "policy" in err.lower()
def test_freshness_does_not_rerun_checks(tmp_path, capsys):
# Freshness must not rewrite the report: mtime-stable bytes.
_, _, package_dir = run_on("venue_clean", tmp_path,
extra_args=["--policy", "advisory"])
report_path = package_dir / REPORT_BASENAME
before = report_path.read_bytes()
capsys.readouterr()
run([str(package_dir), *_fresh_args()])
assert report_path.read_bytes() == before
def test_provenance_summary_outside_fingerprint(tmp_path, capsys):
# gate-1 P1 self-staleness: provenance_summary.md is the pipeline's own
# advisory carrier (D4 appends to it AFTER the report is stamped) — it is
# excluded from the fingerprint, so mutating it does NOT stale the report.
package_dir = tmp_path / "venue_clean"
shutil.copytree(FIXTURES / "venue_clean", package_dir)
(package_dir / "provenance_summary.md").write_text(
"# Provenance\n", encoding="utf-8")
rc, _ = run_dir(package_dir, extra_args=["--policy", "advisory"])
(package_dir / "provenance_summary.md").write_text(
"# Provenance\n\n## Submission Package Advisories\n\n- B2: ...\n",
encoding="utf-8")
capsys.readouterr()
rc = run([str(package_dir), *_fresh_args()])
out = capsys.readouterr().out
# Fresh (no STALE token); the exit code is the re-emitted underlying
# verdict — 3 here (profileless Family B not_checked), NOT a stale 5.
assert rc == 3
assert "STALE-REPORT" not in out
assert "report fresh" in out
def test_advisory_run_is_byte_equivalent_for_package_files(tmp_path):
# §11 byte-equivalence: an explicit-advisory run mutates NOTHING in the
# package except adding the report file.
package_dir = tmp_path / "venue_clean"
shutil.copytree(FIXTURES / "venue_clean", package_dir)
before = {
p.relative_to(package_dir).as_posix(): p.read_bytes()
for p in package_dir.rglob("*") if p.is_file()
}
profile = FIXTURES / "profiles" / "full.yaml"
run_dir(package_dir, extra_args=["--policy", "advisory",
"--venue-profile", str(profile)])
after = {
p.relative_to(package_dir).as_posix(): p.read_bytes()
for p in package_dir.rglob("*") if p.is_file()
}
assert set(after) - set(before) == {REPORT_BASENAME}
for rel, content in before.items():
assert after[rel] == content, f"{rel} mutated by an advisory run"
# --- Slice 4 gate-2 review round: freshness re-emit + external inputs -------
def test_freshness_fresh_strict_report_reemits_terminal_token(
tmp_path, capsys):
# Gate-2 P1: freshness alone means "the report is trustworthy", not "the
# package passed". A fresh strict report that recorded a blocking fail
# must re-emit its terminal verdict on reuse — otherwise the verdict
# silently evaporates on resume and the orchestrator (gating on tokens)
# reads the reuse as a pass.
_, _, package_dir = run_on("orphan_intext", tmp_path,
extra_args=["--policy", "strict"])
capsys.readouterr()
rc = run([str(package_dir), "--check-freshness", "--policy", "strict"])
out = capsys.readouterr().out
assert rc == 1
assert "report fresh" in out
assert "TERMINAL-BLOCK policy=submission_package" in out
def test_freshness_fresh_advisory_report_reemits_underlying_code(
tmp_path, capsys):
# Same contract on the advisory side: a fresh advisory report with
# not_checked rows re-emits exit 3, not a flat 0.
_, _, package_dir = run_on("venue_clean", tmp_path,
extra_args=["--policy", "advisory"])
capsys.readouterr()
rc = run([str(package_dir), *_fresh_args()])
out = capsys.readouterr().out
assert rc == 3
assert "report fresh" in out
assert "STALE-REPORT" not in out
def test_freshness_changed_venue_profile_is_stale(tmp_path, capsys):
# Gate-2 P1: Family B verdicts depend on the venue profile — a report
# produced under one profile must never read as fresh under another
# (a lenient-profile report would otherwise waive a stricter profile's
# limits without any check ever running against them).
full = FIXTURES / "profiles" / "full.yaml"
tight = FIXTURES / "profiles" / "tight.yaml"
_, _, package_dir = run_on(
"venue_clean", tmp_path,
extra_args=["--policy", "advisory", "--venue-profile", str(full)])
capsys.readouterr()
rc = run([str(package_dir), "--check-freshness", "--policy", "advisory",
"--venue-profile", str(tight)])
out = capsys.readouterr().out
assert rc == 5
assert "STALE-REPORT reason=inputs_mismatch" in out
def test_freshness_same_venue_profile_stays_fresh(tmp_path, capsys):
full = FIXTURES / "profiles" / "full.yaml"
_, _, package_dir = run_on(
"venue_clean", tmp_path,
extra_args=["--policy", "advisory", "--venue-profile", str(full)])
capsys.readouterr()
rc = run([str(package_dir), "--check-freshness", "--policy", "advisory",
"--venue-profile", str(full)])
out = capsys.readouterr().out
assert rc == 0
assert "report fresh" in out
def test_freshness_dropped_input_is_stale(tmp_path, capsys):
# Running with a profile, then checking freshness WITHOUT one, is an
# inputs change (declared → absent) and must be stale.
full = FIXTURES / "profiles" / "full.yaml"
_, _, package_dir = run_on(
"venue_clean", tmp_path,
extra_args=["--policy", "advisory", "--venue-profile", str(full)])
capsys.readouterr()
rc = run([str(package_dir), *_fresh_args()])
out = capsys.readouterr().out
assert rc == 5
assert "STALE-REPORT reason=inputs_mismatch" in out
def test_freshness_thinned_roster_is_stale(tmp_path, capsys):
# Final-round review P2: the report file is excluded from the package
# fingerprint, so a hand-edited report with checks thinned out (here:
# emptied) would otherwise read as fresh and re-evaluate to a clean
# exit — the roster guard build_report enforces at write time must
# hold on reuse too.
_, _, package_dir = run_on("orphan_intext", tmp_path,
extra_args=["--policy", "strict"])
report_path = package_dir / REPORT_BASENAME
doctored = json.loads(report_path.read_text(encoding="utf-8"))
doctored["checks"] = []
report_path.write_text(json.dumps(doctored), encoding="utf-8")
capsys.readouterr()
rc = run([str(package_dir), "--check-freshness", "--policy", "strict"])
out = capsys.readouterr().out
assert rc == 5
assert "STALE-REPORT reason=roster_mismatch" in out
assert "report fresh" not in out