"""Integration test: phase 102 — extensionless files named in ``BOR_IMPORT_EXTENSIONS`` import end to end. The owner's defect: ``Dockerfile`` / ``Containerfile`` never synced even with ``dockerfile`` / ``containerfile`` in the env, because the importer matched on the dotted suffix only (``Path("Dockerfile").suffix`` is ``""``). Phase 102's rule (D1/D2): a suffix-less file is in scope iff its lowercased FULL filename equals a token of the set — exact name, case-insensitive — and the ``formats`` counter keys it by the matched token, never ``unknown``. Proven here against the story-dedicated fixture directory ``tests/fixtures/extensionless_kb/`` (the shared ``extension_kb`` fixture of phase 56 stays pinned by its own suite): * positive — ``md,dockerfile,containerfile`` walks ``Dockerfile``, ``Containerfile`` and ``notes.md`` (the ``Makefile`` negative control stays out): rows + embedded chunks + the deterministic mock ``SUMMARY_MODE`` digest for both build files (non-markdown → the phase-30 ``lite`` path), ``formats == {"dockerfile": 1, "containerfile": 1, "md": 1}`` with NO ``unknown`` key; * case — an on-disk ``DOCKERFILE`` imports under the ``dockerfile`` token, its row keeps the on-disk case; * negative — ``md`` alone: only the control note imports; * prune — a token removed from the env drops its extensionless rows on the next ``prune=True`` run (D4 — the A9 junk precedent). The LLM is the deterministic mock server (``tests/e2e/mock_llm.py``) on a scratch port — the integration analogue of the e2e ``mock_llm`` fixture (the phase-56 integration pattern). Runs against the local compose Postgres (the ``db`` fixture from ``tests/conftest.py``); only the distinctive ``extensionless_kb`` / ``casekb`` source rows are created and deleted, so the rest of the shared KB is untouched. Runs (DB must be up: ``podman compose up -d db``): uv run pytest tests/integration/test_import_extensionless.py -v """ from __future__ import annotations import asyncio import os import socket import subprocess import sys import time from collections.abc import Iterator from pathlib import Path from typing import Any import httpx import pytest from sqlalchemy import select from sqlalchemy.orm import Session from app.config import Settings from app.models import Document from app.rag.importer import import_sources from app.rag.llm import LLMClient REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "extensionless_kb" SOURCE = FIXTURES.name # "extensionless_kb" — distinctive, never asserted by count DOCKER_REL = "Dockerfile" CONTAINER_REL = "Containerfile" NOTES_REL = "notes.md" DOCKER_SENTINEL = "DOCKERFILE-PROBE-SENTINEL-7a3e" CONTAINER_SENTINEL = "CONTAINERFILE-PROBE-SENTINEL-4b1c" #: The mock's SUMMARY_MODE tokenizes the document (``[a-z0-9]+``) — the #: hyphenated sentinels land in the digest in their tokenized forms. DOCKER_SENTINEL_TOKENS = "dockerfile probe sentinel 7a3e" CONTAINER_SENTINEL_TOKENS = "containerfile probe sentinel 4b1c" def _wait_http(url: str, timeout: float = 30.0) -> None: deadline = time.monotonic() + timeout last_err = "unknown" while time.monotonic() < deadline: try: httpx.get(url, timeout=2.0) return except Exception as e: # noqa: BLE001 — retry until deadline last_err = str(e) time.sleep(0.2) raise RuntimeError(f"mock LLM at {url} did not come up: {last_err}") @pytest.fixture(scope="module") def mock_llm_port() -> Iterator[int]: """The deterministic mock LLM (``tests/e2e/mock_llm.py``) on a free scratch port — same server as the e2e ``mock_llm`` fixture, but private to this file (integration tests otherwise run network-free).""" sock = socket.socket() sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] sock.close() env = dict(os.environ) env.pop("DEBUGPY", None) proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "tests.e2e.mock_llm:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=REPO, env=env, ) try: _wait_http(f"http://127.0.0.1:{port}/v1/models") yield port finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() def _settings(mock_port: int, extensions: str) -> Settings: kwargs: dict[str, Any] = { "_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", "import_extensions": extensions, } return Settings(**kwargs) # pyright: ignore[reportCallIssue] def _cleanup_source(db: Session, source: str) -> None: for doc in db.scalars(select(Document).where(Document.source == source)).all(): db.delete(doc) db.commit() def test_name_token_files_import_end_to_end(mock_llm_port: int, db: Session) -> None: """``md,dockerfile,containerfile`` over the fixture imports the two name-token build files + the markdown control: rows, embedded chunks, the mock ``SUMMARY_MODE`` digest for each build file, and the ``formats`` counter keyed by the matched token — never ``unknown``. The ``Makefile`` (no token) stays out.""" settings = _settings(mock_llm_port, "md,dockerfile,containerfile") assert settings.import_extension_set == {".md", ".dockerfile", ".containerfile"} summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db)) try: # Exactly the three in-scope files walk — the Makefile negative # control and nothing else. assert ( summary.files, summary.added, summary.unchanged, summary.updated, summary.errors ) == (3, 3, 0, 0, 0) # D2: extensionless files count under their matched token. assert summary.formats == {"dockerfile": 1, "containerfile": 1, "md": 1} assert "unknown" not in summary.formats # Phase 30: both build files are non-markdown → lite summaries; # the markdown control never is. assert (summary.summaries, summary.summary_errors) == (2, 0) for rel, sentinel, tokens in ( (DOCKER_REL, DOCKER_SENTINEL, DOCKER_SENTINEL_TOKENS), (CONTAINER_REL, CONTAINER_SENTINEL, CONTAINER_SENTINEL_TOKENS), ): doc = db.scalar( select(Document).where(Document.source == SOURCE, Document.path == rel) ) assert doc is not None, f"{rel} was not imported by name token" # Suffix-less: the title is the file stem — the whole name. assert doc.title == rel # Plain-text chunking: the content (incl. the sentinel) is # chunked and every content chunk is embedded at the 768-dim # contract. content = [c for c in doc.chunks if not c.is_summary] assert content, f"{rel} has no content chunks" assert all( c.embedding is not None and len(c.embedding) == 768 for c in content ) assert any(sentinel in c.content for c in content) # Mock SUMMARY_MODE digest: byte-stable, the tokenized # sentinel inside it, plus the code-appended pointer line. assert doc.summary is not None assert doc.summary.startswith("This document covers") assert tokens in doc.summary assert f"Source: {SOURCE}/{rel}" in doc.summary schunks = [c for c in doc.chunks if c.is_summary] assert len(schunks) == 1 and schunks[0].position == -1 assert schunks[0].embedding is not None # The markdown control doc imported too — but markdown never # gets a summary (phase 30). note = db.scalar( select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL) ) assert note is not None assert note.summary is None assert [c for c in note.chunks if not c.is_summary] # The negative control: no token names Makefile exactly — never # walked. assert ( db.scalar( select(Document).where(Document.source == SOURCE, Document.path == "Makefile") ) is None ) finally: _cleanup_source(db, SOURCE) def test_uppercase_extensionless_name_imports( mock_llm_port: int, db: Session, tmp_path: Path ) -> None: """Case-insensitive exact name (D1): an on-disk ``DOCKERFILE`` imports under the ``dockerfile`` token, and its row keeps the on-disk case in both ``path`` and ``title``.""" root = tmp_path / "casekb" root.mkdir() (root / "DOCKERFILE").write_text("FROM alpine\nRUN apk add --no-cache curl\n") settings = _settings(mock_llm_port, "md,dockerfile") summary = asyncio.run(import_sources([root], LLMClient(settings), session=db)) try: assert (summary.files, summary.added, summary.errors) == (1, 1, 0) assert summary.formats == {"dockerfile": 1} doc = db.scalar( select(Document).where(Document.source == "casekb", Document.path == "DOCKERFILE") ) assert doc is not None, "DOCKERFILE did not import under the dockerfile token" assert doc.title == "DOCKERFILE" assert [c for c in doc.chunks if not c.is_summary] # Non-markdown → the phase-30 lite path ran. assert summary.summaries == 1 assert doc.summary is not None assert doc.summary.startswith("This document covers") finally: _cleanup_source(db, "casekb") def test_without_tokens_the_extensionless_files_stay_out( mock_llm_port: int, db: Session ) -> None: """``md`` alone: the control note imports; ``Dockerfile`` / ``Containerfile`` (their names are not tokens in this scope) and the ``Makefile`` are all out of scope — no value of the scope walks them.""" settings = _settings(mock_llm_port, "md") assert settings.import_extension_set == {".md"} summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db)) try: assert ( summary.files, summary.added, summary.unchanged, summary.updated, summary.errors ) == (1, 1, 0, 0, 0) assert summary.formats == {"md": 1} assert summary.summaries == 0 for rel in (DOCKER_REL, CONTAINER_REL, "Makefile"): assert ( db.scalar( select(Document).where(Document.source == SOURCE, Document.path == rel) ) is None ), f"{rel} was imported although its name is not a token in `md`" assert ( db.scalar( select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL) ) is not None ) finally: _cleanup_source(db, SOURCE) def test_prune_removes_extensionless_rows_when_the_token_is_removed( mock_llm_port: int, db: Session ) -> None: """D4: a file that stops matching — the token leaves the env — leaves ``seen`` and is deleted by the next ``prune=True`` run (the A9 junk precedent); the in-scope ``notes.md`` row survives.""" with_tokens = _settings(mock_llm_port, "md,dockerfile,containerfile") asyncio.run(import_sources([FIXTURES], LLMClient(with_tokens), session=db)) try: md_only = _settings(mock_llm_port, "md") summary = asyncio.run( import_sources([FIXTURES], LLMClient(md_only), session=db, prune=True) ) assert summary.pruned == 2, "the two extensionless rows must be pruned" assert summary.unchanged == 1, "notes.md must survive the prune" for rel in (DOCKER_REL, CONTAINER_REL): assert ( db.scalar( select(Document).where(Document.source == SOURCE, Document.path == rel) ) is None ), f"{rel} survived although its token left the env" assert ( db.scalar( select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL) ) is not None ) finally: _cleanup_source(db, SOURCE)