"""Docs guard: what `README.md` tells a human to type must be real (phase 01 task 04). Documentation rots in two ways this file makes impossible: 1. **Pseudo-commands.** A fenced block that reads plausibly but cannot run — an invented script, a tool that is not part of this project, a placeholder like `make deploy`. Every command line inside a `bash`/`sh`/`console`/`shell` fence of `README.md` or `docs/*.md` must start with a known-safe prefix, and whatever it launches through `uv run` must be a console script or a dependency declared in `pyproject.toml`. That is what keeps the Quickstart true after a rename — and what stops a promise (e.g. the phase-02 `undercooked-levels` CLI) from being written as a runnable command before it exists. 2. **Config drift.** A variable documented but not implemented, or implemented but never documented. AGENTS.md §5 requires the `Settings` field, the `ENV_KEYS` entry, the `.env.example` line and the README row to land together; here the documentation side is compared against `config.py` and `.env.example`, so they cannot disagree. """ from __future__ import annotations import re import tomllib from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path import pytest from undercooked.config import ENV_KEYS pytestmark = pytest.mark.unit REPO_ROOT = Path(__file__).resolve().parents[2] README = REPO_ROOT / "README.md" ENV_EXAMPLE = REPO_ROOT / ".env.example" PYPROJECT = REPO_ROOT / "pyproject.toml" #: Markdown documents whose command blocks a human may copy. DOCUMENTED_MARKDOWN: tuple[Path, ...] = (README, *sorted((REPO_ROOT / "docs").glob("*.md"))) #: Shell fences that promise "type this". COMMAND_FENCE_LANGUAGES = frozenset({"bash", "sh", "console", "shell"}) #: The only things a documented command line may start with. Anything else is prose #: wearing a code fence, and it will be deleted rather than run. SAFE_PREFIXES = ("uv ", "podman ", "bash .agents/validate.sh", "curl ", "cd ", "#") #: `FOO=bar cmd …` is still a command; only documented variables may lead a line. ENV_ASSIGNMENT = re.compile(r"""^(?:[A-Z][A-Z0-9_]*=(?:"[^"]*"|'[^']*'|\S)+\s+)+""") #: Every `UNDERCOOKED_*` / `DEBUGPY*` name written anywhere in prose, a table or a fence. #: A name has to start with a real character, so the `UNDERCOOKED_*` wildcard in prose is #: not read as a variable nobody implemented. DOCUMENTED_VARIABLE = re.compile(r"\bUNDERCOOKED_[A-Z0-9_]*[A-Z0-9]\b|\bDEBUGPY(?:_[A-Z0-9]+)?\b") #: `uv run` options that consume the token after them, so the program name is further along. UV_VALUE_OPTIONS = frozenset({"--extra", "--python", "--with", "--directory", "--project", "-m"}) #: `KEY=value` lines of an env template. _TEMPLATE_VARIABLE = re.compile(r"^([A-Z][A-Z0-9_]*)\s*=") @dataclass(frozen=True, slots=True) class Command: """One logical command line from a fenced block (continuations joined).""" document: str line: int text: str def __str__(self) -> str: return f"{self.document}:{self.line}: {self.text}" def _strip_assignment(command: str) -> tuple[tuple[str, ...], str]: """Split `A=1 B=2 uv run …` into its variable names and the command underneath.""" names: list[str] = [] while match := ENV_ASSIGNMENT.match(command): names.extend(part.split("=", 1)[0] for part in match.group(0).split()) command = command[match.end(0) :].lstrip() return tuple(names), command def fenced_commands(markdown: str, *, document: str = "") -> list[Command]: """Every command inside a `bash`-family fence, with the line it started on. A trailing-backslash continuation becomes one `Command`, because only the first line of a wrapped command carries the program name. """ commands: list[Command] = [] fence: str | None = None pending: list[str] = [] pending_line = 0 for number, raw in enumerate(markdown.splitlines(), start=1): marker = re.match(r"^\s*```(\w*)\s*$", raw) if marker: fence = None if fence is not None else marker.group(1).lower() if pending: commands.append(Command(document, pending_line, " ".join(pending))) pending = [] continue if fence is None or fence not in COMMAND_FENCE_LANGUAGES: continue stripped = raw.strip() if not stripped: if pending: commands.append(Command(document, pending_line, " ".join(pending))) pending = [] continue if not pending: pending_line = number if stripped.endswith("\\"): pending.append(stripped[:-1].strip()) continue pending.append(stripped) commands.append(Command(document, pending_line, " ".join(pending))) pending = [] if pending: # pragma: no cover - a document ending mid-continuation commands.append(Command(document, pending_line, " ".join(pending))) return commands def uv_run_program(command: str) -> str | None: """The program a `uv run …` line launches, with uv's own options skipped; else `None`. That program must be declared in `pyproject.toml`, which is what makes a doc line inventing a script fail the build. """ if not command.startswith("uv run"): return None tokens = command.split()[2:] index = 0 while index < len(tokens): token = tokens[index] if token in UV_VALUE_OPTIONS: index += 2 continue if token.startswith("-"): index += 1 continue return token.lower() return None def declared_uv_programs() -> frozenset[str]: """Program names `uv run` may launch: this project's scripts plus its dependencies.""" def names(requirements: list[str]) -> set[str]: return {re.split(r"[<>=!;\[ ]", name)[0].strip().lower() for name in requirements} data = tomllib.loads(PYPROJECT.read_text(encoding="utf-8")) project = data["project"] packages = names(list(project.get("dependencies", []))) for extra in project.get("optional-dependencies", {}).values(): packages |= names(list(extra)) return frozenset(set(project.get("scripts", {})) | packages | {"python"}) def template_variables() -> set[str]: """Names declared in the committed `.env.example` template (comments ignored).""" return { match.group(1) for line in ENV_EXAMPLE.read_text(encoding="utf-8").splitlines() if (match := _TEMPLATE_VARIABLE.match(line.strip())) } def variables_in(path: Path) -> set[str]: """Every configuration variable named anywhere in one markdown file.""" return set(DOCUMENTED_VARIABLE.findall(path.read_text(encoding="utf-8"))) def _fail(header: str, offenders: list[str]) -> None: pytest.fail(f"{header}\n" + "\n".join(f" {offender}" for offender in offenders)) def unsafe_commands(commands: Iterable[Command]) -> list[str]: """Commands whose first token is none of `SAFE_PREFIXES`, once env assignments are peeled.""" offenders: list[str] = [] for command in commands: _, bare = _strip_assignment(command.text) if not bare.startswith(SAFE_PREFIXES): offenders.append(str(command)) return offenders def undeclared_programs(commands: Iterable[Command], allowed: frozenset[str]) -> list[str]: """`uv run ` lines whose `` pyproject.toml does not declare.""" offenders: list[str] = [] for command in commands: _, bare = _strip_assignment(command.text) program = uv_run_program(bare) if program is not None and program not in allowed: offenders.append(f"{command} -> {program!r} is not in pyproject.toml") return offenders def documented_but_unimplemented(documented: set[str], implemented: set[str]) -> list[str]: """Variables the docs name and `config.py` does not read.""" return sorted(documented - implemented) def implemented_but_undocumented(implemented: set[str], documented: set[str]) -> list[str]: """Variables `config.py` reads and the docs never mention.""" return sorted(set(implemented) - documented) # --------------------------------------------------------------------------- command blocks def test_readme_has_command_blocks_worth_checking() -> None: """Non-vacuous: if the Quickstart stops being a bash fence, this has to notice.""" commands = fenced_commands(README.read_text(encoding="utf-8"), document="README.md") assert len(commands) >= 8, f"README.md has lost its command blocks: {commands}" assert any(command.text.startswith("uv run undercooked") for command in commands) assert any(command.text.startswith("uv run pytest") for command in commands) def test_documented_command_lines_start_with_a_known_prefix() -> None: """No pseudo-commands in the docs a player or an operator copy-pastes from.""" offenders: list[str] = [] for document in DOCUMENTED_MARKDOWN: commands = fenced_commands(document.read_text(encoding="utf-8"), document=document.name) offenders.extend(unsafe_commands(commands)) if offenders: _fail(f"documented commands must start with one of {SAFE_PREFIXES}:", offenders) def test_documented_commands_only_name_declared_programs() -> None: """`uv run ` may only launch a script or dependency of this project.""" allowed = declared_uv_programs() offenders: list[str] = [] for document in DOCUMENTED_MARKDOWN: commands = fenced_commands(document.read_text(encoding="utf-8"), document=document.name) offenders.extend(undeclared_programs(commands, allowed)) if offenders: _fail(f"`uv run` may only run {sorted(allowed)}:", offenders) def test_command_fence_is_a_fence_and_not_a_table() -> None: """Pins the extractor: fenced bash is a command, indented prose and other fences are not.""" sample = "\n".join( [ "text `not-a-command --yolo`", # 1: inline code is not a command "", "```bash", # 3 "uv run undercooked \\", # 4 \\ the continuation joins the next line " --port 8137", # 5 "", # 6: a blank line ends a command, it never joins across it "uv run pytest -q", # 7 "# a comment", # 8 "uv run ruff check \\", # 9 \\ a continuation cut off by a blank line still counts "", "uv run pyright \\", # 11 \\ and one cut off by the closing fence counts too "```", "", "```text", # 14: not a command fence "made-up-command --yolo", "```", ] ) assert fenced_commands(sample) == [ Command("", 4, "uv run undercooked --port 8137"), Command("", 7, "uv run pytest -q"), Command("", 8, "# a comment"), Command("", 9, "uv run ruff check"), Command("", 11, "uv run pyright"), ] assert str(Command("README.md", 12, "uv run undercooked")) == "README.md:12: uv run undercooked" def test_leading_environment_assignments_are_peeled_off() -> None: names, bare = _strip_assignment("DEBUGPY=1 UNDERCOOKED_PORT=8137 uv run undercooked") assert names == ("DEBUGPY", "UNDERCOOKED_PORT") assert bare == "uv run undercooked" @pytest.mark.parametrize( ("command", "program"), [ ("uv run pytest -q", "pytest"), ("uv run --extra dev ruff check .", "ruff"), ("uv run undercooked --reload", "undercooked"), ("uv run python -c 'print(1)'", "python"), ], ) def test_the_uv_run_extractor_finds_the_program(command: str, program: str) -> None: assert uv_run_program(command) == program @pytest.mark.parametrize("command", ["podman build .", "uv sync --extra dev", "cd .", "uv run", ""]) def test_the_uv_run_extractor_returns_none_for_everything_else(command: str) -> None: assert uv_run_program(command) is None def test_the_declared_programs_cover_the_real_toolchain() -> None: """Derived from pyproject.toml, so a dependency removal breaks the docs loudly.""" allowed = declared_uv_programs() assert {"undercooked", "pytest", "ruff", "pyright", "playwright", "python"} <= allowed assert "undercooked-levels" in allowed # declared now, importable from phase 02 # --------------------------------------------------------------------------- variable names def test_every_variable_named_in_the_docs_is_implemented() -> None: """README and docs may only name a variable `config.py` or `.env.example` defines.""" implemented = set(ENV_KEYS.values()) | template_variables() offenders = [ f"{document.name}: {name}" for document in DOCUMENTED_MARKDOWN for name in documented_but_unimplemented(variables_in(document), implemented) ] if offenders: _fail("documented but not implemented:", offenders) def test_every_implemented_variable_is_documented() -> None: """The reverse direction: a knob nobody documented is a knob nobody can find.""" documented = variables_in(README) | variables_in(ENV_EXAMPLE) missing = implemented_but_undocumented(set(ENV_KEYS.values()), documented) if missing: _fail("config.py defines variables neither README.md nor .env.example mentions:", missing) def test_readme_and_env_template_name_the_same_variables() -> None: """The two operator-facing lists cannot diverge from each other.""" documented, template = variables_in(README), variables_in(ENV_EXAMPLE) only_readme = sorted(documented - template) only_template = sorted(template - documented) assert not only_readme, f"named in README.md but absent from .env.example: {only_readme}" assert not only_template, f"named in .env.example but absent from README.md: {only_template}" # --------------------------------------------------------------------------- the gate is documented def test_readme_documents_the_gate_and_the_cdn_rule() -> None: """The commands AGENTS.md calls binding are findable in the human-facing docs.""" readme = README.read_text(encoding="utf-8") assert "bash .agents/validate.sh" in readme, "the pipeline gate is not documented in README" assert re.search(r"no external cdns", readme, re.IGNORECASE), "A12 is not stated in README" def test_a_pseudo_command_is_reported() -> None: """The prefix rule actually fires: planted prose-commands are named, real ones are not.""" sample = "\n".join( [ "```bash", # 1 "uv run undercooked --port 8137", # 2 "DEBUGPY=1 uv run undercooked", # 3: an env lead is still a command "make deploy-to-staging", # 4 "kubectl port-forward svc/undercooked 8000", # 5 "# a comment is a valid line", # 6 "```", ] ) offenders = unsafe_commands(fenced_commands(sample, document="")) assert offenders == [ ":4: make deploy-to-staging", ":5: kubectl port-forward svc/undercooked 8000", ] def test_an_invented_program_is_reported() -> None: """A doc line cannot run a script pyproject.toml does not declare.""" sample = "\n".join( [ "```bash", "uv run --frozen undercooked-levels validate", "uv run made-up-cli validate", "```", ] ) commands = fenced_commands(sample, document="") offenders = undeclared_programs(commands, frozenset({"undercooked-levels"})) assert len(offenders) == 1, offenders assert "made-up-cli" in offenders[0] def test_the_variable_diffs_work_in_both_directions() -> None: """The two drift directions are tested, not just asserted to be empty on today's docs.""" implemented = {"UNDERCOOKED_PORT", "DEBUGPY"} documented = {"UNDERCOOKED_PORT", "DEBUGPY", "UNDERCOOKED_MADE_UP", "DEBUGPY_PORT"} assert documented_but_unimplemented(documented, implemented) == [ "DEBUGPY_PORT", "UNDERCOOKED_MADE_UP", ] assert implemented_but_undocumented(implemented, documented) == [] assert implemented_but_undocumented({"UNDERCOOKED_NEW_KNOB"}, documented) == [ "UNDERCOOKED_NEW_KNOB" ] def test_the_failure_helper_lists_every_offender() -> None: """`_fail` is the message an executor reads; it has to enumerate, not truncate.""" with pytest.raises(pytest.fail.Exception, match="two"): _fail("broken:", ["one", "two"]) def test_agents_md_only_names_test_paths_that_exist() -> None: """Every concrete `tests/**.py|md` path AGENTS.md cites is a real file.""" agents = (REPO_ROOT / "AGENTS.md").read_text(encoding="utf-8") cited = re.findall(r"tests/[A-Za-z0-9_./-]+\.(?:py|md)", agents) assert "tests/unit/test_no_external_assets.py" in cited assert cited, "AGENTS.md cites no test files at all" missing = sorted({path for path in cited if not (REPO_ROOT / path).is_file()}) assert missing == [], f"AGENTS.md points at files that do not exist: {missing}"