"""The assumptions the npm-side Skill installer makes about this repository. `tools/skills-installer/` is JavaScript, and its own behaviour is gated by `node tools/skills-installer/selftest.mjs`. What that self-test cannot own is the *other* half of the contract: the shape of `skills/` and the packaging manifest that decides what a published package contains. Both live on this side of the language boundary and both drift silently — a Skill added without `SKILL.md`, a `bin` path renamed, a `files` glob that stops matching — so they are asserted here, in the suite that runs on every commit with nothing but Python installed. Deliberately NOT here: anything that would require Node. A test that shells out to `node` reports success by skipping on the machine that does not have it, which is the failure mode the marker policy exists to prevent. """ from __future__ import annotations import json import re from pathlib import Path import pytest from scripts import render_skill_install_sections as install_sections pytestmark = pytest.mark.skills ROOT = Path(__file__).resolve().parents[1] SKILLS = ROOT / "skills" PACKAGE_JSON = ROOT / "package.json" #: The installer's discovery rule, restated: a directory is a Skill when it #: holds this file, and a collection when it holds the marker below with its #: members one level down. Both names are also what `openai4s/skills_loader/ #: loader.py` looks for; a rename on either side must break something. SKILL_MARKER = "SKILL.md" COLLECTION_MARKER = "COLLECTION.json" def _package() -> dict: return json.loads(PACKAGE_JSON.read_text(encoding="utf-8")) def _skill_dirs() -> list[Path]: found: list[Path] = [] for child in sorted(SKILLS.iterdir()): if not child.is_dir() or child.name.startswith("."): continue if (child / COLLECTION_MARKER).is_file(): found.extend( member for member in sorted(child.iterdir()) if member.is_dir() and (member / SKILL_MARKER).is_file() ) continue if (child / SKILL_MARKER).is_file(): found.append(child) return found def test_the_published_description_counts_the_skills_it_actually_ships(): """The count npm shows a stranger is the one nothing was checking. Three hand-written counts of the same library drifted apart: this description said 603, `install.mjs` said 602, and the tree held 604 — none of them wrong when they were written, all of them wrong together. A number in prose that no gate reads has a shelf life, so this is the gate: adding a Skill now fails here until the published description follows. The comment in `install.mjs` dropped its number instead, because nothing can check a comment and a second uncheckable count is what produced this. """ description = _package()["description"] stated = re.search(r"(\d+)\s+scientific recipes", description) assert stated, f"description no longer states a recipe count: {description!r}" assert int(stated.group(1)) == len(_skill_dirs()), ( f"package.json advertises {stated.group(1)} recipes; " f"skills/ holds {len(_skill_dirs())}" ) #: Every prose sentence that states a live Skill count, as (path, regex, which #: counts its groups carry). Historical figures — the v0.2.0 release-note lines, #: the tag's pack size in `docs/TODO.md` — are deliberately not matched: they #: were true at the tag and are meant to stay as written. COUNT_SITES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( ("README.md", r"\*\*🔬 (\d+) bundled Skills\*\*", ("total",)), ("README.md", r"(\d+) curated OpenAI4S recipes", ("curated",)), ("README.md", r"plus all (\d+) recipes from the pinned", ("collection",)), ("README.md", r"The (\d+) bundled Skills are recipes", ("total",)), ("README.md", r"# the (\d+) curated Skills", ("curated",)), ("README.md", r"# the (\d+) pinned bioinformatics recipes", ("collection",)), ("README.md", r"you already have all (\d+)", ("total",)), ("README_zh.md", r"\*\*🔬 (\d+) 个内置 Skill\*\*", ("total",)), ("README_zh.md", r"(\d+) 份由 OpenAI4S 筛选维护", ("curated",)), ("README_zh.md", r"内置的 (\d+) 个 Skill 是配方", ("total",)), ("README_zh.md", r"# (\d+) 个精选 Skill", ("curated",)), ("README_zh.md", r"# (\d+) 个固定版生信配方", ("collection",)), ("README_zh.md", r"那 (\d+) 个你已经全有了", ("total",)), ( "CLAUDE.md", r"(\d+) bundled Skills: (\d+) curated OpenAI4S directories plus the pinned (\d+)-recipe", ("total", "curated", "collection"), ), ("docs/skills.md", r"## Bundled Skills \((\d+)\)", ("total",)), ( "docs/skills.md", r"two maintenance tiers: (\d+) curated OpenAI4S Skills", ("curated",), ), ("docs/skills.md", r"read-only import of all (\d+) MIT-licensed", ("collection",)), ("docs/skills.md", r"### Curated OpenAI4S Skills \((\d+)\)", ("curated",)), ("docs/skills.md", r"# the (\d+) curated Skills", ("curated",)), ("docs/skills.md", r"# the (\d+) pinned recipes", ("collection",)), ( "docs/skills.md", r"(\d+) Skills and a bundled Skill takes precedence", ("total",), ), ( "skills/README.md", r"This tree exposes (\d+) bundled Skills: (\d+) curated OpenAI4S recipes plus the (\d+)", ("total", "curated", "collection"), ), ( "skills/README_zh.md", r"共暴露 (\d+) 个内置 Skill:(\d+) 份由 OpenAI4S 筛选维护", ("total", "curated"), ), ("skills/README_zh.md", r"GPTomics/bioSkills 全部 (\d+) 份配方", ("collection",)), ("tools/skills-installer/README.md", r"# the (\d+) curated Skills", ("curated",)), ( "tools/skills-installer/README.md", r"# the (\d+) pinned recipes", ("collection",), ), ( "tools/skills-installer/README.md", r"all (\d+) Skills, and `openai4s/skills_loader", ("total",), ), ("tools/skills-installer/README_zh.md", r"# (\d+) 个精选 Skill", ("curated",)), ( "tools/skills-installer/README_zh.md", r"# (\d+) 个固定版第三方配方", ("collection",), ), ( "tools/skills-installer/README_zh.md", r"wheel 已经带上了全部 (\d+) 个 Skill", ("total",), ), ("docs/TODO.md", r"current source tree has (\d+)\s+Skills", ("total",)), ( "docs/TODO_zh.md", r"当前源码树有 (\d+) 个 Skill", ("total",), ), ) def test_every_prose_count_matches_the_tree(): """The number `package.json` is held to is stated in thirty other places. The count gate above pins one JSON field; the same 604, 43 and 561 sit in the root README (the PyPI long_description), `CLAUDE.md`, the Skills doc, the `skills/` index, the installer README and the TODO — where the next Skill directory would leave them wrong with `package.json` green. Each site is matched by a sentence-specific pattern so a sentence that stops stating the count fails too, rather than silently dropping out of the gate; a second collection would change what "collection" means here and must revisit the table. """ dirs = _skill_dirs() curated = [skill for skill in dirs if skill.parent == SKILLS] expected = { "total": len(dirs), "curated": len(curated), "collection": len(dirs) - len(curated), } problems = [] for rel, pattern, counts in COUNT_SITES: text = (ROOT / rel).read_text(encoding="utf-8") matches = list(re.finditer(pattern, text)) if not matches: problems.append(f"{rel}: no sentence matches {pattern!r} any more") for match in matches: for stated, name in zip(match.groups(), counts): if int(stated) != expected[name]: problems.append( f"{rel}: says {stated} where the tree's {name} count is " f"{expected[name]} ({match.group(0)[:60]!r})" ) assert not problems, "\n".join(problems) def test_every_skill_page_carries_its_own_install_section(): """The page a reader lands on must install the Skill it describes. `skills/README.md` promises that every Skill page repeats the Install section with its own name filled in. That is a claim about ninety generated pages, and a generated page with no generator is a snapshot: the next `skills//` lands with a README pair, passes the directory-README gate and the count gate above, and has no section — or a hand edit forks one page's wording from the other eighty-nine. `render_skill_install_sections --check` is the mechanism; this test is what makes it run on every commit. """ stale = install_sections.check() assert not stale, ( "Install sections out of date; run " "`python scripts/render_skill_install_sections.py`: " + ", ".join(stale) ) def test_the_installer_can_name_every_bundled_skill(): """Every Skill the loader exposes must be one the installer can copy. The installer addresses a Skill by its directory name and proves it is a Skill by `SKILL.md`. A bundled directory that satisfies the loader through some other route would be listed by the daemon and unreachable from `npx github:PKU-YuanGroup/OpenAI4S install ` -- present in one catalogue and absent from the other, with nothing saying so. """ dirs = _skill_dirs() assert len(dirs) >= 500, f"only {len(dirs)} Skills discovered under {SKILLS}" for skill in dirs: assert (skill / SKILL_MARKER).is_file() assert ( skill.name == skill.name.strip() ), f"padded directory name: {skill.name!r}" assert "/" not in skill.name and "\\" not in skill.name def test_every_skill_document_starts_with_frontmatter(): """The installer reads `name` and `description` out of the leading `---` block to list a Skill. A document without one still installs, but lists as a bare directory name with no summary -- a catalogue entry that says nothing, which is how a user picks the wrong recipe.""" missing = [ str(skill.relative_to(ROOT)) for skill in _skill_dirs() if not (skill / SKILL_MARKER) .read_text(encoding="utf-8", errors="replace") .lstrip() .startswith("---") ] assert not missing, missing def test_the_skill_tree_contains_no_symlinks(): """The installer refuses link members when it extracts an archive, and copies file-by-file when it does not. A symlink inside `skills/` would be resolved and its *target's* bytes written into the user's directory -- a file that came from outside the tree the manifest claims to describe.""" links = [ str(path.relative_to(ROOT)) for path in SKILLS.rglob("*") if path.is_symlink() ] assert not links, links def test_the_npm_manifest_points_at_a_command_that_exists(): package = _package() assert package["name"] == "@pku-yuangroup/openai4s-skills" lock = json.loads((ROOT / "package-lock.json").read_text(encoding="utf-8")) assert lock["name"] == package["name"] assert lock["packages"][""]["name"] == package["name"] assert package["bin"] == {"openai4s-skills": "tools/skills-installer/cli.mjs"} binaries = package["bin"] assert binaries, "the package declares no command, so `npx` has nothing to run" for name, target in binaries.items(): path = ROOT / target assert path.is_file(), f"bin {name} -> {target} does not exist" first = path.read_text(encoding="utf-8").splitlines()[0] assert first.startswith("#!"), f"bin {name} has no shebang" def test_the_npm_manifest_ships_the_command_and_the_skills(): """`files` is the entire contract between this tree and what `npx` gets. It is a list of globs with nothing checking it against the repository, and the way it fails is not a crash: drop `skills/` and the published command still runs, finds nothing bundled, and silently falls back to downloading a 100 MB source tarball on every install. """ package = _package() shipped = package["files"] target = next(iter(package["bin"].values())) assert any( target.startswith(entry.rstrip("/") + "/") or entry == target for entry in shipped if not entry.startswith("!") ), f"`files` does not ship {target}" assert any( entry.rstrip("/") == "skills" for entry in shipped if not entry.startswith("!") ), "`files` does not ship skills/" def test_the_npm_package_is_not_shipped_inside_the_python_wheel(): """Two distributions, one tree. `tools/` is Node and must stay out of the wheel: `pip install openai4s` promising a stdlib-only core and then carrying JavaScript is a claim the packaging would quietly break.""" pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") include = pyproject.split("[tool.setuptools.packages.find]", 1)[1] include = include.split("[", 1)[0] assert '"tools' not in include, "tools/ is being packaged into the wheel" def test_node_and_python_agree_on_the_supported_node_floor(): """The CLI refuses to run below the floor `package.json` advertises. Two numbers that can disagree are one number and one lie.""" package = _package() declared = package["engines"]["node"] cli = (ROOT / "tools" / "skills-installer" / "cli.mjs").read_text(encoding="utf-8") assert "MIN_NODE_MAJOR = 18" in cli assert declared == ">=18", declared def test_recipe_added_after_npm_020_uses_github_installation(): """A current-only recipe must not advertise a failing published install.""" name = "single-cell-rna-analysis" for lang, notice in ( ("README.md", "npm 0.2.0 does not include this recipe"), ("README_zh.md", "npm 0.2.0 不包含本配方"), ): text = (SKILLS / name / lang).read_text(encoding="utf-8") assert notice in " ".join(text.split()) assert f"npx github:PKU-YuanGroup/OpenAI4S install {name}" in text assert f"npx @pku-yuangroup/openai4s-skills install {name}" not in text def _version_key(version: str) -> tuple[int, ...]: return tuple(int(part) for part in version.split(".")) def _next_release_steps(version: str) -> set[str]: """The versions one release step after `version`: next patch, minor, major.""" major, minor, patch = _version_key(version) return { f"{major}.{minor}.{patch + 1}", f"{major}.{minor + 1}.0", f"{major + 1}.0.0", } def test_every_npm_pin_names_a_published_release(): """A pinned install command is only honest if that version is on npm. `package.json` moves to the next version before the registry has it, because `npm publish` reads the version from there. The docs cannot follow it until the publish has happened. So pins are held to the list of releases recorded as published, and `package.json` may be ahead of that list by exactly one release step (the next patch, minor or major version after the newest published one). Bumping `package.json` therefore cannot drag the pins along, a version that skips a release is refused, and switching a pin before the version is recorded as published fails here. """ import subprocess published = install_sections.PUBLISHED_NPM_VERSIONS assert published, "no npm release is recorded as published" declared = _package()["version"] newest = max(published, key=_version_key) assert declared in published or declared in _next_release_steps(newest), ( f"package.json declares {declared}, which is neither published " f"({published}) nor one release step after {newest} " f"({sorted(_next_release_steps(newest))})" ) try: listed = subprocess.run( ["git", "ls-files", "-z"], cwd=ROOT, capture_output=True, check=True ).stdout.decode("utf-8") except (OSError, subprocess.CalledProcessError): pytest.skip("not a git checkout") pin = re.compile(r"openai4s-skills@(\d+\.\d+\.\d+)") unpublished = [] for rel in filter(None, listed.split("\0")): if rel.startswith("skills/bioskills/") or not rel.endswith( (".md", ".mjs", ".py", ".json", ".txt") ): continue path = ROOT / rel if not path.is_file(): continue text = path.read_text(encoding="utf-8", errors="replace") for match in pin.finditer(text): if match.group(1) not in published: unpublished.append(f"{rel}: {match.group(0)}") assert not unpublished, ( "install pins name npm versions not recorded as published in " f"PUBLISHED_NPM_VERSIONS: {unpublished}" )