feat(import): user-extensible BOR_IMPORT_EXTENSIONS — any well-formed extension, A9 family stays the default

tests/e2e/test_import_extensions_env.py

@@ -0,0 +1,163 @@
+"""Phase 56 E2E (Playwright): a NOVEL extension (``.sh``) flows config →
+import → chunks → mock summary → Sources page.
+
+TODO.md L6: "Allow the user to specify extensions to be read in .env,
+don't hard-code working extensions." The subject is the env-driven
+extension scope (``import_extensions="md,sh"``); the story-dedicated
+fixture (``tests/fixtures/extension_kb/``) is seeded in-process against
+the deterministic mock LLM — the phase-02 seeding-thread pattern, the
+fixture, not the subject of the tests.
+
+Run in isolation (DB must be up: ``podman compose up -d db``):
+
+    uv run pytest tests/e2e/test_import_extensions_env.py -v --no-cov
+
+DB isolation: the fixture's source name (``extension_kb``) is
+distinctive — the suite never asserts on absolute row counts and
+deletes the rows it creates in a ``finally`` (other suites' documents
+stay untouched in the shared E2E database).
+"""
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterator
+from pathlib import Path
+from threading import Thread
+from typing import Any
+
+import pytest
+from playwright.sync_api import Page, expect
+from sqlalchemy import select
+
+from app.config import Settings
+from app.db import SessionLocal
+from app.models import Document
+from app.rag.importer import ImportSummary, import_sources
+from app.rag.llm import LLMClient
+from e2e.auth_helpers import login
+
+REPO = Path(__file__).resolve().parents[2]
+FIXTURES = REPO / "tests" / "fixtures" / "extension_kb"
+SOURCE = FIXTURES.name  # "extension_kb" — distinctive, never asserted by count
+SH_REL = "homelab/scripts/uptime.sh"
+MD_REL = "homelab/notes/note.md"
+SENTINEL = "UPTIME-PROBE-SENTINEL-9c2f"
+
+
+async def _import_fixtures(mock_port: int, extensions: str) -> ImportSummary:
+    kwargs: dict[str, Any] = {
+        "_env_file": None,
+        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
+        "import_extensions": extensions,
+    }
+    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
+    return await import_sources([FIXTURES], LLMClient(settings))
+
+
+def _run_in_thread(coro: Any) -> Any:
+    """Run a coroutine on a worker thread.
+
+    Playwright's sync API keeps an asyncio loop running on the test
+    thread, so ``asyncio.run`` cannot be called directly from a test
+    body.
+    """
+    box: dict[str, Any] = {}
+
+    def runner() -> None:
+        try:
+            box["value"] = asyncio.run(coro)
+        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
+            box["error"] = e
+
+    t = Thread(target=runner)
+    t.start()
+    t.join()
+    if "error" in box:
+        raise box["error"]
+    return box["value"]
+
+
+def _delete_source_rows() -> None:
+    """Delete every row of this suite's distinctive source (chunks
+    cascade with the document rows)."""
+    with SessionLocal() as db:
+        for doc in db.scalars(select(Document).where(Document.source == SOURCE)).all():
+            db.delete(doc)
+        db.commit()
+
+
+@pytest.fixture(autouse=True)
+def extension_kb(mock_llm: int, db_ready: None) -> Iterator[ImportSummary]:
+    """Seed the fixture with the NOVEL scope (``md,sh``) for one test
+    and delete every row it creates afterwards (DB isolation — see the
+    module docstring)."""
+    _delete_source_rows()  # idempotent: leftovers from a crashed run
+    summary = _run_in_thread(_import_fixtures(mock_llm, "md,sh"))
+    try:
+        yield summary
+    finally:
+        _delete_source_rows()
+
+
+def test_admin_sources_lists_the_novel_extension(
+    page: Page, app_url: str, extension_kb: ImportSummary
+) -> None:
+    # The seed saw exactly the two fixture files in their formats — the
+    # novel .sh extension walked, chunked, and summarized.
+    assert extension_kb.formats == {"sh": 1, "md": 1}
+    assert (extension_kb.added, extension_kb.errors) == (2, 0)
+
+    login(page, app_url)  # phase 16: the catalog is admin-only
+    # The novel .sh document is listed; the path cell carries the full
+    # path (the column is ellipsized — the title attribute is the pin).
+    row = page.locator("#docs-tbody tr", has_text=SH_REL)
+    expect(row).to_have_count(1)
+    link = row.locator("td:nth-child(2) a.doc-link")
+    expect(link).to_have_count(1)
+    expect(link).to_have_attribute("title", SH_REL)
+    # The markdown control doc is listed too (never asserted by count —
+    # other suites' documents may share the shared E2E database).
+    expect(page.locator("#docs-tbody tr", has_text=MD_REL)).to_have_count(1)
+
+    # Format badge: the row's path link opens the same-page modal and
+    # its meta row shows the .sh format (house assertion style —
+    # test_document_viewer.py asserts the same locator for yaml/md).
+    before = len(page.context.pages)
+    link.click()
+    assert len(page.context.pages) == before, "clicking a row link must not open a new tab"
+    expect(page.locator("#doc-modal-meta .doc-source-badge")).to_have_text(SOURCE)
+    expect(page.locator("#doc-modal-meta .format-badge")).to_have_text("sh")
+    # Non-markdown content renders as escaped monospace text in a pre —
+    # the sentinel proves it is THIS document's content.
+    pre = page.locator("#doc-modal-content pre.doc-raw")
+    expect(pre).to_have_count(1)
+    expect(pre).to_contain_text(SENTINEL)
+    # Still on the Sources page: no navigation happened.
+    assert page.url == app_url + "/sources.html", f"navigated away: {page.url}"
+
+
+def test_anonymous_sources_gate_and_no_api_docs(
+    page: Page, app_url: str, extension_kb: ImportSummary
+) -> None:
+    """A fresh anonymous context (function-scoped ``page`` = new
+    browser context, no cookies): the sign-in gate renders and the page
+    never calls ``/api/docs`` — the phase-16 pin, regression-checked
+    with the novel-extension KB seeded."""
+    api_docs_calls: list[str] = []
+    page.on(
+        "request",
+        lambda r: api_docs_calls.append(r.url) if "/api/docs" in r.url else None,
+    )
+
+    page.goto(f"{app_url}/sources.html")
+    # The gate, with its sign-in link — not a redirect.
+    gate = page.locator("#sources-gate")
+    expect(gate).to_be_visible()
+    expect(gate).to_contain_text("Sign in to view the full catalog")
+    expect(gate.locator("a[href='/login.html?next=/sources.html']")).to_have_count(1)
+    # Stat cards + table hidden…
+    expect(page.locator("#stat-cards")).to_be_hidden()
+    expect(page.locator("#docs-table")).to_be_hidden()
+    expect(page.locator("#sources-empty")).to_be_hidden()
+    # …and NO /api/docs call was ever made.
+    assert api_docs_calls == [], f"anonymous sources page called /api/docs: {api_docs_calls}"

tests/fixtures/extension_kb/homelab/notes/note.md

@@ -0,0 +1,6 @@
+# Extension fixture note
+
+A small markdown control document for the phase 56 extension-kb fixture.
+It exists so the `md` scope and the novel `sh` scope are told apart when
+the importer walks `tests/fixtures/extension_kb/` — with
+`import_extensions="md"` only this file should land in the index.

tests/fixtures/extension_kb/homelab/scripts/uptime.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# UPTIME-PROBE-SENTINEL-9c2f — phase 56 fixture marker: this token exists
+# nowhere else, so the extension_kb rows are unambiguous in the shared DB.
+#
+# uptime.sh — homelab service probe: polls the core services and posts a
+# ntfy alert on the first failure. A novel (.sh) file on purpose — it
+# only imports when BOR_IMPORT_EXTENSIONS names the sh extension.
+
+set -euo pipefail
+
+ALERT_TOPIC="homelab-alerts"
+NTFY_URL="https://ntfy.reeseapps.com"
+CHECKS=(
+  "k3s|https://10.0.1.10:6443/healthz"
+  "gitlab|https://gitlab.reeseapps.com/-/health_check"
+  "ntfy|https://ntfy.reeseapps.com/health"
+)
+
+probe() {
+  local name="$1" url="$2"
+  curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url"
+}
+
+main() {
+  local line name code
+  for line in "${CHECKS[@]}"; do
+    name="${line%%|*}"
+    code="$(probe "$name" "${line#*|}")"
+    if [[ "$code" != "200" ]]; then
+      echo "uptime: $name answered $code (expected 200)" >&2
+      curl -s -X POST "$NTFY_URL/$ALERT_TOPIC" \
+        -H "Title: homelab check failed" \
+        -d "$name is down (HTTP $code)"
+    fi
+  done
+  echo "uptime: round complete"
+}
+
+main "$@"

tests/integration/test_import_extensions_env.py

@@ -0,0 +1,190 @@
+"""Integration test: phase 56 — ``BOR_IMPORT_EXTENSIONS`` is user-extensible.
+
+Proves a NOVEL (non-A9) extension flows through the import machinery
+(config → walk → delta → chunk → mock ``SUMMARY_MODE`` digest) against
+the story-dedicated fixture directory ``tests/fixtures/extension_kb/``
+(the shared ``docs`` / ``summary_kb`` fixtures stay pinned by their own
+suites). The LLM is the deterministic mock server (``tests/e2e/mock_llm.py``)
+on a scratch port — the integration analogue of the e2e ``mock_llm``
+fixture — so the summary is the byte-stable ``SUMMARY_MODE`` digest and
+the vectors are genuine token-overlap embeddings. Runs against the local
+compose Postgres (the ``db`` fixture from ``tests/conftest.py``); only
+the distinctive ``extension_kb`` source rows are created and deleted, so
+the rest of the shared KB is untouched.
+
+Runs (DB must be up: ``podman compose up -d db``):
+
+    uv run pytest tests/integration/test_import_extensions_env.py -v
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import socket
+import subprocess
+import sys
+import time
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+import httpx
+import pytest
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.models import Document
+from app.rag.importer import import_sources
+from app.rag.llm import LLMClient
+
+REPO = Path(__file__).resolve().parents[2]
+FIXTURES = REPO / "tests" / "fixtures" / "extension_kb"
+SOURCE = FIXTURES.name  # "extension_kb" — distinctive, never asserted by count
+SH_REL = "homelab/scripts/uptime.sh"
+MD_REL = "homelab/notes/note.md"
+SENTINEL = "UPTIME-PROBE-SENTINEL-9c2f"
+#: The mock's SUMMARY_MODE tokenizes the document (``[a-z0-9]+``) — the
+#: hyphenated sentinel lands in the digest in its tokenized form.
+SENTINEL_TOKENS = "uptime probe sentinel 9c2f"
+
+
+def _wait_http(url: str, timeout: float = 30.0) -> None:
+    deadline = time.monotonic() + timeout
+    last_err = "unknown"
+    while time.monotonic() < deadline:
+        try:
+            httpx.get(url, timeout=2.0)
+            return
+        except Exception as e:  # noqa: BLE001 — retry until deadline
+            last_err = str(e)
+            time.sleep(0.2)
+    raise RuntimeError(f"mock LLM at {url} did not come up: {last_err}")
+
+
+@pytest.fixture(scope="module")
+def mock_llm_port() -> Iterator[int]:
+    """The deterministic mock LLM (``tests/e2e/mock_llm.py``) on a free
+    scratch port — same server as the e2e ``mock_llm`` fixture, but
+    private to this file (integration tests otherwise run network-free)."""
+    sock = socket.socket()
+    sock.bind(("127.0.0.1", 0))
+    port = sock.getsockname()[1]
+    sock.close()
+    env = dict(os.environ)
+    env.pop("DEBUGPY", None)
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "uvicorn", "tests.e2e.mock_llm:app",
+         "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
+        cwd=REPO,
+        env=env,
+    )
+    try:
+        _wait_http(f"http://127.0.0.1:{port}/v1/models")
+        yield port
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+
+def _settings(mock_port: int, extensions: str) -> Settings:
+    kwargs: dict[str, Any] = {
+        "_env_file": None,
+        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
+        "import_extensions": extensions,
+    }
+    return Settings(**kwargs)  # pyright: ignore[reportCallIssue]
+
+
+def _cleanup_source(db: Session, source: str) -> None:
+    for doc in db.scalars(select(Document).where(Document.source == source)).all():
+        db.delete(doc)
+    db.commit()
+
+
+def test_novel_extension_imports_end_to_end(mock_llm_port: int, db: Session) -> None:
+    """``md,sh`` (a novel extension) imports the ``.sh`` file end to
+    end: row + plain-text chunks + the mock ``SUMMARY_MODE`` digest,
+    with the markdown control doc imported as well."""
+    settings = _settings(mock_llm_port, "md,sh")
+    assert settings.import_extension_set == {".md", ".sh"}
+    summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
+    try:
+        assert (
+            summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
+        ) == (2, 2, 0, 0, 0)
+        assert summary.formats == {"sh": 1, "md": 1}
+        # The .sh file is non-markdown → exactly one lite summary (phase 30).
+        assert (summary.summaries, summary.summary_errors) == (1, 0)
+
+        sh = db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == SH_REL
+            )
+        )
+        assert sh is not None, "the novel .sh extension was not imported"
+        # Non-markdown: the title comes from the file stem (a ``#`` line
+        # is a comment, not a heading).
+        assert sh.title == "uptime"
+        # Plain-text chunking: the content (incl. the sentinel) is
+        # chunked and every content chunk is embedded at the 768-dim
+        # contract.
+        content = [c for c in sh.chunks if not c.is_summary]
+        assert content, "the .sh file has no content chunks"
+        assert all(
+            c.embedding is not None and len(c.embedding) == 768 for c in content
+        )
+        assert any(SENTINEL in c.content for c in content)
+        # Mock SUMMARY_MODE digest: byte-stable, the tokenized sentinel
+        # inside it, plus the code-appended pointer line.
+        assert sh.summary is not None
+        assert sh.summary.startswith("This document covers")
+        assert SENTINEL_TOKENS in sh.summary
+        assert f"Source: {SOURCE}/{SH_REL}" in sh.summary
+        schunks = [c for c in sh.chunks if c.is_summary]
+        assert len(schunks) == 1 and schunks[0].position == -1
+        assert schunks[0].embedding is not None
+
+        # The markdown control doc imported too — but markdown never
+        # gets a summary (phase 30).
+        note = db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == MD_REL
+            )
+        )
+        assert note is not None
+        assert note.summary is None
+        assert [c for c in note.chunks if not c.is_summary]
+    finally:
+        _cleanup_source(db, SOURCE)
+
+
+def test_narrowing_to_md_still_excludes_the_novel_extension(
+    mock_llm_port: int, db: Session
+) -> None:
+    """``md`` (the A9-era narrowing, preserved as a special case): the
+    ``.sh`` file is out of scope, only the control note imports."""
+    settings = _settings(mock_llm_port, "md")
+    assert settings.import_extension_set == {".md"}
+    summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
+    try:
+        assert (
+            summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
+        ) == (1, 1, 0, 0, 0)
+        assert summary.formats == {"md": 1}
+        assert summary.summaries == 0
+        assert db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == SH_REL
+            )
+        ) is None
+        assert db.scalar(
+            select(Document).where(
+                Document.source == SOURCE, Document.path == MD_REL
+            )
+        ) is not None
+    finally:
+        _cleanup_source(db, SOURCE)

tests/unit/test_config.py

@@ -8,7 +8,7 @@ import pytest
 from pydantic import ValidationError
 from pydantic_settings import SettingsError
 
-from app.config import _ALLOWED_IMPORT_EXTENSIONS, Settings  # pyright: ignore[reportPrivateUsage]
+from app.config import _DEFAULT_IMPORT_EXTENSIONS, Settings  # pyright: ignore[reportPrivateUsage]
 
 
 def _settings(**kwargs: Any) -> Settings:
@@ -57,15 +57,15 @@ NEW_A9_FORMATS = (
 )
 
 
-def test_allowed_import_extensions_contains_all_seventeen_formats() -> None:
-    """The validator's base set is the full A9 set: the original seven
-    plus the ten added 2026-08-27 (quadlet family + ``j2``). The
-    never-widen contract bounds :py:data:`import_extensions` against
-    exactly this set."""
+def test_default_import_extensions_is_the_full_a9_family() -> None:
+    """Phase 56: the built-in default is the full A9 set — the original
+    seven plus the ten added 2026-08-27 (quadlet family + ``j2``). It is
+    the default and the ``.env.example`` example, NOT a ceiling: the
+    validator accepts any well-formed extension beyond it."""
     assert {
         "md", "markdown", "txt", "yaml", "yml", "json", "py",
         *NEW_A9_FORMATS,
-    } == _ALLOWED_IMPORT_EXTENSIONS
+    } == _DEFAULT_IMPORT_EXTENSIONS
 
 
 def test_default_import_extensions_include_the_ten_new_formats() -> None:
@@ -151,34 +151,51 @@ def test_import_extensions_env_override_is_a_csv_list(monkeypatch) -> None:
     assert s.import_extension_set == {".md", ".yml"}
 
 
-def test_import_extensions_rejects_unknown_format(monkeypatch) -> None:
-    """A typo in the CSV fails at startup (loudly), not by silently
-    walking zero files."""
-    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,docx")
-    with pytest.raises(ValidationError, match="docx"):
-        _settings()
+def test_import_extensions_accepts_novel_extension(monkeypatch) -> None:
+    """Phase 56 (owner permission 2026-08-31): the A9 family is the
+    default, not the ceiling — a novel well-formed extension (``sh``) is
+    accepted and simply becomes importable."""
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,sh")
+    s = _settings()
+    assert s.import_extension_set == {".md", ".sh"}
 
 
-def test_import_extensions_rejects_empty(monkeypatch) -> None:
-    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "   ")
-    with pytest.raises(ValidationError):
-        _settings()
+def test_import_extensions_normalizes_case_and_leading_dot(monkeypatch) -> None:
+    """Case and a leading dot are both tolerated (unchanged tolerance)."""
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "MD,.Py")
+    s = _settings()
+    assert s.import_extension_set == {".md", ".py"}
+
+
+def test_import_extensions_rejects_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A blank list would silently import nothing — fail loudly at
+    startup, naming the field (empty, whitespace-only, and comma-only
+    all parse to zero formats)."""
+    for value in ("", "   ", ",,"):
+        monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", value)
+        with pytest.raises(ValidationError, match="import_extensions"):
+            _settings()
 
 
 def test_import_extensions_validator_accepts_new_a9_formats(monkeypatch) -> None:
-    """A9 revised 2026-08-27: the new names are first-class — the
-    never-widen contract now holds against the widened base set, so a
-    narrowing CSV with quadlet/jinja names is accepted."""
+    """A9 revised 2026-08-27: quadlet/jinja names are first-class default
+    formats — a CSV using them (a narrowing of the default family) is
+    accepted."""
     monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,container,j2")
     s = _settings()
     assert s.import_extension_set == {".md", ".container", ".j2"}
 
 
-def test_import_extensions_validator_still_rejects_unknown(monkeypatch) -> None:
-    """Truly unknown extensions still fail loudly at startup (the
-    validator is intact — only the allowed base set widened)."""
-    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,xyz")
-    with pytest.raises(ValidationError, match="xyz"):
+def test_import_extensions_rejects_malformed_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The shape guard (``^[a-z0-9]{1,16}$``) is the typo guard — it
+    keeps punctuation and path-ish values out of the set, naming the
+    offending token(s), while any extension a file could actually be
+    suffixed with still goes through."""
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,sh!")
+    with pytest.raises(ValidationError, match="sh!"):
+        _settings()
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,../x")
+    with pytest.raises(ValidationError, match=r"/x"):
         _settings()