feat(import): user-extensible BOR_IMPORT_EXTENSIONS — any well-formed extension, A9 family stays the default

tests/integration/test_import_extensions_env.py

@@ -0,0 +1,190 @@
+"""Integration test: phase 56 — ``BOR_IMPORT_EXTENSIONS`` is user-extensible.
+
+Proves a NOVEL (non-A9) extension flows through the import machinery
+(config → walk → delta → chunk → mock ``SUMMARY_MODE`` digest) against
+the story-dedicated fixture directory ``tests/fixtures/extension_kb/``
+(the shared ``docs`` / ``summary_kb`` fixtures stay pinned by their own
+suites). The LLM is the deterministic mock server (``tests/e2e/mock_llm.py``)
+on a scratch port — the integration analogue of the e2e ``mock_llm``
+fixture — so the summary is the byte-stable ``SUMMARY_MODE`` digest and
+the vectors are genuine token-overlap embeddings. Runs against the local
+compose Postgres (the ``db`` fixture from ``tests/conftest.py``); only
+the distinctive ``extension_kb`` source rows are created and deleted, so
+the rest of the shared KB is untouched.
+
+Runs (DB must be up: ``podman compose up -d db``):
+
+    uv run pytest tests/integration/test_import_extensions_env.py -v
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import socket
+import subprocess
+import sys
+import time
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+import httpx
+import pytest
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.models import Document
+from app.rag.importer import import_sources
+from app.rag.llm import LLMClient
+
+REPO = Path(__file__).resolve().parents[2]
+FIXTURES = REPO / "tests" / "fixtures" / "extension_kb"
+SOURCE = FIXTURES.name  # "extension_kb" — distinctive, never asserted by count
+SH_REL = "homelab/scripts/uptime.sh"
+MD_REL = "homelab/notes/note.md"
+SENTINEL = "UPTIME-PROBE-SENTINEL-9c2f"
+#: The mock's SUMMARY_MODE tokenizes the document (``[a-z0-9]+``) — the
+#: hyphenated sentinel lands in the digest in its tokenized form.
+SENTINEL_TOKENS = "uptime probe sentinel 9c2f"
+
+
+def _wait_http(url: str, timeout: float = 30.0) -> None:
+    deadline = time.monotonic() + timeout
+    last_err = "unknown"
+    while time.monotonic() < deadline:
+        try:
+            httpx.get(url, timeout=2.0)
+            return
+        except Exception as e:  # noqa: BLE001 — retry until deadline
+            last_err = str(e)
+            time.sleep(0.2)
+    raise RuntimeError(f"mock LLM at {url} did not come up: {last_err}")
+
+
+@pytest.fixture(scope="module")
+def mock_llm_port() -> Iterator[int]:
+    """The deterministic mock LLM (``tests/e2e/mock_llm.py``) on a free
+    scratch port — same server as the e2e ``mock_llm`` fixture, but
+    private to this file (integration tests otherwise run network-free)."""
+    sock = socket.socket()
+    sock.bind(("127.0.0.1", 0))
+    port = sock.getsockname()[1]
+    sock.close()
+    env = dict(os.environ)
+    env.pop("DEBUGPY", None)
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "uvicorn", "tests.e2e.mock_llm:app",
+         "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
+        cwd=REPO,
+        env=env,
+    )
+    try:
+        _wait_http(f"http://127.0.0.1:{port}/v1/models")
+        yield port
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+
+def _settings(mock_port: int, extensions: str) -> Settings:
+    kwargs: dict[str, Any] = {
+        "_env_file": None,
+        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
+        "import_extensions": extensions,
+    }
+    return Settings(**kwargs)  # pyright: ignore[reportCallIssue]
+
+
+def _cleanup_source(db: Session, source: str) -> None:
+    for doc in db.scalars(select(Document).where(Document.source == source)).all():
+        db.delete(doc)
+    db.commit()
+
+
+def test_novel_extension_imports_end_to_end(mock_llm_port: int, db: Session) -> None:
+    """``md,sh`` (a novel extension) imports the ``.sh`` file end to
+    end: row + plain-text chunks + the mock ``SUMMARY_MODE`` digest,
+    with the markdown control doc imported as well."""
+    settings = _settings(mock_llm_port, "md,sh")
+    assert settings.import_extension_set == {".md", ".sh"}
+    summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
+    try:
+        assert (
+            summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
+        ) == (2, 2, 0, 0, 0)
+        assert summary.formats == {"sh": 1, "md": 1}
+        # The .sh file is non-markdown → exactly one lite summary (phase 30).
+        assert (summary.summaries, summary.summary_errors) == (1, 0)
+
+        sh = db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == SH_REL
+            )
+        )
+        assert sh is not None, "the novel .sh extension was not imported"
+        # Non-markdown: the title comes from the file stem (a ``#`` line
+        # is a comment, not a heading).
+        assert sh.title == "uptime"
+        # Plain-text chunking: the content (incl. the sentinel) is
+        # chunked and every content chunk is embedded at the 768-dim
+        # contract.
+        content = [c for c in sh.chunks if not c.is_summary]
+        assert content, "the .sh file has no content chunks"
+        assert all(
+            c.embedding is not None and len(c.embedding) == 768 for c in content
+        )
+        assert any(SENTINEL in c.content for c in content)
+        # Mock SUMMARY_MODE digest: byte-stable, the tokenized sentinel
+        # inside it, plus the code-appended pointer line.
+        assert sh.summary is not None
+        assert sh.summary.startswith("This document covers")
+        assert SENTINEL_TOKENS in sh.summary
+        assert f"Source: {SOURCE}/{SH_REL}" in sh.summary
+        schunks = [c for c in sh.chunks if c.is_summary]
+        assert len(schunks) == 1 and schunks[0].position == -1
+        assert schunks[0].embedding is not None
+
+        # The markdown control doc imported too — but markdown never
+        # gets a summary (phase 30).
+        note = db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == MD_REL
+            )
+        )
+        assert note is not None
+        assert note.summary is None
+        assert [c for c in note.chunks if not c.is_summary]
+    finally:
+        _cleanup_source(db, SOURCE)
+
+
+def test_narrowing_to_md_still_excludes_the_novel_extension(
+    mock_llm_port: int, db: Session
+) -> None:
+    """``md`` (the A9-era narrowing, preserved as a special case): the
+    ``.sh`` file is out of scope, only the control note imports."""
+    settings = _settings(mock_llm_port, "md")
+    assert settings.import_extension_set == {".md"}
+    summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
+    try:
+        assert (
+            summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
+        ) == (1, 1, 0, 0, 0)
+        assert summary.formats == {"md": 1}
+        assert summary.summaries == 0
+        assert db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == SH_REL
+            )
+        ) is None
+        assert db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == MD_REL
+            )
+        ) is not None
+    finally:
+        _cleanup_source(db, SOURCE)