brain-of-reese/tests/integration/test_import_extensionless.py

"""Integration test: phase 102 — extensionless files named in
``BOR_IMPORT_EXTENSIONS`` import end to end.

The owner's defect: ``Dockerfile`` / ``Containerfile`` never synced even
with ``dockerfile`` / ``containerfile`` in the env, because the importer
matched on the dotted suffix only (``Path("Dockerfile").suffix`` is
``""``). Phase 102's rule (D1/D2): a suffix-less file is in scope iff its
lowercased FULL filename equals a token of the set — exact name,
case-insensitive — and the ``formats`` counter keys it by the matched
token, never ``unknown``.

Proven here against the story-dedicated fixture directory
``tests/fixtures/extensionless_kb/`` (the shared ``extension_kb``
fixture of phase 56 stays pinned by its own suite):

* positive — ``md,dockerfile,containerfile`` walks ``Dockerfile``,
  ``Containerfile`` and ``notes.md`` (the ``Makefile`` negative control
  stays out): rows + embedded chunks + the deterministic mock
  ``SUMMARY_MODE`` digest for EVERY doc (phase 118, A2: markdown
  included — the phase-30 ``lite`` path), ``formats == {"dockerfile": 1,
  "containerfile": 1, "md": 1}`` with NO ``unknown`` key;
* case — an on-disk ``DOCKERFILE`` imports under the ``dockerfile``
  token, its row keeps the on-disk case;
* negative — ``md`` alone: only the control note imports;
* prune — a token removed from the env drops its extensionless rows on
  the next ``prune=True`` run (D4 — the A9 junk precedent).

The LLM is the deterministic mock server (``tests/e2e/mock_llm.py``) on
a scratch port — the integration analogue of the e2e ``mock_llm``
fixture (the phase-56 integration pattern). Runs against the local
compose Postgres (the ``db`` fixture from ``tests/conftest.py``); only
the distinctive ``extensionless_kb`` / ``casekb`` source rows are
created and deleted, so the rest of the shared KB is untouched.

Runs (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/integration/test_import_extensionless.py -v
"""
from __future__ import annotations

import asyncio
import os
import socket
import subprocess
import sys
import time
from collections.abc import Iterator
from pathlib import Path
from typing import Any

import httpx
import pytest
from sqlalchemy import select
from sqlalchemy.orm import Session

from app.config import Settings
from app.models import Document
from app.rag.importer import import_sources
from app.rag.llm import LLMClient

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "extensionless_kb"
SOURCE = FIXTURES.name  # "extensionless_kb" — distinctive, never asserted by count
DOCKER_REL = "Dockerfile"
CONTAINER_REL = "Containerfile"
NOTES_REL = "notes.md"
DOCKER_SENTINEL = "DOCKERFILE-PROBE-SENTINEL-7a3e"
CONTAINER_SENTINEL = "CONTAINERFILE-PROBE-SENTINEL-4b1c"
#: The mock's SUMMARY_MODE tokenizes the document (``[a-z0-9]+``) — the
#: hyphenated sentinels land in the digest in their tokenized forms.
DOCKER_SENTINEL_TOKENS = "dockerfile probe sentinel 7a3e"
CONTAINER_SENTINEL_TOKENS = "containerfile probe sentinel 4b1c"


def _wait_http(url: str, timeout: float = 30.0) -> None:
    deadline = time.monotonic() + timeout
    last_err = "unknown"
    while time.monotonic() < deadline:
        try:
            httpx.get(url, timeout=2.0)
            return
        except Exception as e:  # noqa: BLE001 — retry until deadline
            last_err = str(e)
            time.sleep(0.2)
    raise RuntimeError(f"mock LLM at {url} did not come up: {last_err}")


@pytest.fixture(scope="module")
def mock_llm_port() -> Iterator[int]:
    """The deterministic mock LLM (``tests/e2e/mock_llm.py``) on a free
    scratch port — same server as the e2e ``mock_llm`` fixture, but
    private to this file (integration tests otherwise run network-free)."""
    sock = socket.socket()
    sock.bind(("127.0.0.1", 0))
    port = sock.getsockname()[1]
    sock.close()
    env = dict(os.environ)
    env.pop("DEBUGPY", None)
    proc = subprocess.Popen(
        [sys.executable, "-m", "uvicorn", "tests.e2e.mock_llm:app",
         "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
        cwd=REPO,
        env=env,
    )
    try:
        _wait_http(f"http://127.0.0.1:{port}/v1/models")
        yield port
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()


def _settings(mock_port: int, extensions: str) -> Settings:
    kwargs: dict[str, Any] = {
        "_env_file": None,
        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
        "import_extensions": extensions,
    }
    return Settings(**kwargs)  # pyright: ignore[reportCallIssue]


def _cleanup_source(db: Session, source: str) -> None:
    for doc in db.scalars(select(Document).where(Document.source == source)).all():
        db.delete(doc)
    db.commit()


def test_name_token_files_import_end_to_end(mock_llm_port: int, db: Session) -> None:
    """``md,dockerfile,containerfile`` over the fixture imports the two
    name-token build files + the markdown control: rows, embedded
    chunks, the mock ``SUMMARY_MODE`` digest for each build file, and
    the ``formats`` counter keyed by the matched token — never
    ``unknown``. The ``Makefile`` (no token) stays out."""
    settings = _settings(mock_llm_port, "md,dockerfile,containerfile")
    assert settings.import_extension_set == {".md", ".dockerfile", ".containerfile"}
    summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
    try:
        # Exactly the three in-scope files walk — the Makefile negative
        # control and nothing else.
        assert (
            summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
        ) == (3, 3, 0, 0, 0)
        # D2: extensionless files count under their matched token.
        assert summary.formats == {"dockerfile": 1, "containerfile": 1, "md": 1}
        assert "unknown" not in summary.formats
        # Phase 118 (A2): EVERY doc gets a lite summary — both build
        # files AND the markdown control.
        assert (summary.summaries, summary.summary_errors) == (3, 0)

        for rel, sentinel, tokens in (
            (DOCKER_REL, DOCKER_SENTINEL, DOCKER_SENTINEL_TOKENS),
            (CONTAINER_REL, CONTAINER_SENTINEL, CONTAINER_SENTINEL_TOKENS),
        ):
            doc = db.scalar(
                select(Document).where(Document.source == SOURCE, Document.path == rel)
            )
            assert doc is not None, f"{rel} was not imported by name token"
            # Suffix-less: the title is the file stem — the whole name.
            assert doc.title == rel
            # Plain-text chunking: the content (incl. the sentinel) is
            # chunked and every content chunk is embedded at the 768-dim
            # contract.
            content = [c for c in doc.chunks if not c.is_summary]
            assert content, f"{rel} has no content chunks"
            assert all(
                c.embedding is not None and len(c.embedding) == 768 for c in content
            )
            assert any(sentinel in c.content for c in content)
            # Mock SUMMARY_MODE digest: byte-stable, the tokenized
            # sentinel inside it, plus the code-appended pointer line.
            assert doc.summary is not None
            assert doc.summary.startswith("This document covers")
            assert tokens in doc.summary
            assert f"Source: {SOURCE}/{rel}" in doc.summary
            schunks = [c for c in doc.chunks if c.is_summary]
            assert len(schunks) == 1 and schunks[0].position == -1
            assert schunks[0].embedding is not None

        # The markdown control doc imported too — AND got the mock
        # ``SUMMARY_MODE`` digest (phase 118, A2: markdown summarized).
        note = db.scalar(
            select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL)
        )
        assert note is not None
        assert note.summary is not None
        assert note.summary.startswith("This document covers extensionless fixture notes")
        assert f"Source: {SOURCE}/{NOTES_REL}" in note.summary
        note_schunks = [c for c in note.chunks if c.is_summary]
        assert len(note_schunks) == 1 and note_schunks[0].position == -1
        assert note_schunks[0].embedding is not None
        assert [c for c in note.chunks if not c.is_summary]

        # The negative control: no token names Makefile exactly — never
        # walked.
        assert (
            db.scalar(
                select(Document).where(Document.source == SOURCE, Document.path == "Makefile")
            )
            is None
        )
    finally:
        _cleanup_source(db, SOURCE)


def test_uppercase_extensionless_name_imports(
    mock_llm_port: int, db: Session, tmp_path: Path
) -> None:
    """Case-insensitive exact name (D1): an on-disk ``DOCKERFILE``
    imports under the ``dockerfile`` token, and its row keeps the
    on-disk case in both ``path`` and ``title``."""
    root = tmp_path / "casekb"
    root.mkdir()
    (root / "DOCKERFILE").write_text("FROM alpine\nRUN apk add --no-cache curl\n")
    settings = _settings(mock_llm_port, "md,dockerfile")
    summary = asyncio.run(import_sources([root], LLMClient(settings), session=db))
    try:
        assert (summary.files, summary.added, summary.errors) == (1, 1, 0)
        assert summary.formats == {"dockerfile": 1}
        doc = db.scalar(
            select(Document).where(Document.source == "casekb", Document.path == "DOCKERFILE")
        )
        assert doc is not None, "DOCKERFILE did not import under the dockerfile token"
        assert doc.title == "DOCKERFILE"
        assert [c for c in doc.chunks if not c.is_summary]
        # Non-markdown → the phase-30 lite path ran.
        assert summary.summaries == 1
        assert doc.summary is not None
        assert doc.summary.startswith("This document covers")
    finally:
        _cleanup_source(db, "casekb")


def test_without_tokens_the_extensionless_files_stay_out(
    mock_llm_port: int, db: Session
) -> None:
    """``md`` alone: the control note imports; ``Dockerfile`` /
    ``Containerfile`` (their names are not tokens in this scope) and the
    ``Makefile`` are all out of scope — no value of the scope walks
    them."""
    settings = _settings(mock_llm_port, "md")
    assert settings.import_extension_set == {".md"}
    summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
    try:
        assert (
            summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
        ) == (1, 1, 0, 0, 0)
        assert summary.formats == {"md": 1}
        # Phase 118 (A2): the md-only run still summarizes the note.
        assert summary.summaries == 1
        for rel in (DOCKER_REL, CONTAINER_REL, "Makefile"):
            assert (
                db.scalar(
                    select(Document).where(Document.source == SOURCE, Document.path == rel)
                )
                is None
            ), f"{rel} was imported although its name is not a token in `md`"
        assert (
            db.scalar(
                select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL)
            )
            is not None
        )
    finally:
        _cleanup_source(db, SOURCE)


def test_prune_removes_extensionless_rows_when_the_token_is_removed(
    mock_llm_port: int, db: Session
) -> None:
    """D4: a file that stops matching — the token leaves the env —
    leaves ``seen`` and is deleted by the next ``prune=True`` run (the
    A9 junk precedent); the in-scope ``notes.md`` row survives."""
    with_tokens = _settings(mock_llm_port, "md,dockerfile,containerfile")
    asyncio.run(import_sources([FIXTURES], LLMClient(with_tokens), session=db))
    try:
        md_only = _settings(mock_llm_port, "md")
        summary = asyncio.run(
            import_sources([FIXTURES], LLMClient(md_only), session=db, prune=True)
        )
        assert summary.pruned == 2, "the two extensionless rows must be pruned"
        assert summary.unchanged == 1, "notes.md must survive the prune"
        for rel in (DOCKER_REL, CONTAINER_REL):
            assert (
                db.scalar(
                    select(Document).where(Document.source == SOURCE, Document.path == rel)
                )
                is None
            ), f"{rel} survived although its token left the env"
        assert (
            db.scalar(
                select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL)
            )
            is not None
        )
    finally:
        _cleanup_source(db, SOURCE)