phase: 102_extensionless_filenames
All verification complete — every gate green, no defects found in previously completed work. **Phase 102 final verification pass — report** Verified (all three task files present in `complete/`; working-tree implementation matches D1–D5 design): - `match_extension` choke point in `app/rag/importer.py` (walk + `formats` counter), `doc_format` name-token badge in `app/api/docs.py`, config/`.env.example` docs, fixture `tests/fixtures/extensionless_kb/`, integration + E2E suites — all present and correct - Completion criteria: end-to-end sync (✓ integration + E2E), case matrix incl. `mydockerfile`/`Dockerfile.dev`/`.dockerfile` exclusions (✓ unit), `formats=dockerfile:1` not `unknown` (✓ log-line assertion), badge `dockerfile`/`containerfile` + `text` fallback + suffixed unchanged (✓ unit/integration/E2E), prune-on-token-removal (✓ `pruned==2`), suffixed-path rule byte-identical (✓ single-line swap, existing cases untouched) Test / lint results (exact commands): - `uv run pytest --cov=app --cov-report=term-missing` → 2084 passed, **99%** coverage (>90% gate) - `uv run pytest tests/e2e/test_extensionless_import.py -v --no-cov` → 2 passed, isolated, DB up - Regressions isolated: `test_import_documents` 3✓, `test_import_extensions_env` 2✓, `test_quadlet_jinja_import` 4✓, `test_document_viewer` 7✓, `test_kb_tree` 8✓ - `uv run ruff check .` → clean; `uv run pyright` → 0 errors, 0 warnings Notable: commit intentionally not made (harness commits the phase); 102's task files already sit in `complete/`, overview stays in `todo/` for the harness. Next pending phases: 98, 99, 103, 104, 105 (numeric next after 102: `103_suggestions_session_openers`).
This commit is contained in:
@@ -0,0 +1,245 @@
|
||||
"""Phase 102 E2E (Playwright): extensionless files (``Dockerfile``,
|
||||
``Containerfile``) sync when their name is a ``BOR_IMPORT_EXTENSIONS``
|
||||
token — import → chunks → mock ``lite`` summary → drill-down tree →
|
||||
viewer badge, plus the negative control and the anonymous gate.
|
||||
|
||||
The owner's defect: files without extensions never got synced, so
|
||||
``Dockerfile`` / ``Containerfile`` were skipped even with
|
||||
``dockerfile,containerfile`` in the env. Phase 102's rule: a suffix-less
|
||||
file imports iff its lowercased FULL filename equals a token (exact
|
||||
name, case-insensitive), the import ``formats=`` counter keys it by the
|
||||
matched token, and the viewer badge is truthful (``dockerfile``, not
|
||||
the generic ``text``).
|
||||
|
||||
The subject is the extensionless scope (``import_extensions=
|
||||
"md,dockerfile,containerfile"``); the story-dedicated fixture
|
||||
(``tests/fixtures/extensionless_kb/``) is seeded in-process against the
|
||||
deterministic mock LLM — the phase-02/56 seeding-thread pattern — with
|
||||
``Dockerfile`` / ``Containerfile`` carrying their own unique sentinels
|
||||
and ``Makefile`` as the negative control (no token names it).
|
||||
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_extensionless_import.py -v --no-cov
|
||||
|
||||
DB isolation: the fixture's source name (``extensionless_kb``) is
|
||||
distinctive — the suite never asserts on absolute row counts and
|
||||
deletes the rows it creates in a ``finally`` (other suites' documents
|
||||
stay untouched in the shared E2E database).
|
||||
|
||||
App-under-test scope pin (the phase-61 leak-guard pattern): the viewer
|
||||
badge is rendered by the APP from its own ``get_settings().
|
||||
import_extension_set`` (``GET /api/documents/content``), so the app
|
||||
process must carry exactly the scope under test — a module-level
|
||||
``BOR_IMPORT_EXTENSIONS`` override (process env ranks above the
|
||||
operator's local gitignored ``.env`` when the ``app_server`` fixture
|
||||
snapshots ``os.environ``). In isolation (AGENTS.md rule 9) the session
|
||||
app is spawned by this file's first test, so the pin lands determinis-
|
||||
tically.
|
||||
|
||||
Phase 97 adaptation: the catalog is the DRILL-DOWN TREE — the fixture's
|
||||
files live at the SOURCE level (no subfolders), so a single drill
|
||||
reaches every asserted row; the drill is the only change, the asserted
|
||||
rows/links/modal are unchanged.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Document
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
from e2e.auth_helpers import login
|
||||
from tests.e2e.mock_llm import TOKEN_RE
|
||||
|
||||
#: The app-under-test scope pin (see the module docstring) — exactly the
|
||||
#: scope this suite proves, no matter what the operator's local ``.env``
|
||||
#: carries. Must run before the session ``app_server`` fixture snapshots
|
||||
#: ``os.environ`` (module import precedes fixture setup).
|
||||
os.environ["BOR_IMPORT_EXTENSIONS"] = "md,dockerfile,containerfile"
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "extensionless_kb"
|
||||
SOURCE = FIXTURES.name # "extensionless_kb" — distinctive, never asserted by count
|
||||
DOCKER_REL = "Dockerfile"
|
||||
CONTAINER_REL = "Containerfile"
|
||||
NOTES_REL = "notes.md"
|
||||
MAKE_REL = "Makefile" # negative control — no token in the scope under test
|
||||
DOCKER_SENTINEL = "DOCKERFILE-PROBE-SENTINEL-7a3e"
|
||||
|
||||
|
||||
async def _import_fixtures(mock_port: int, extensions: str) -> ImportSummary:
|
||||
kwargs: dict[str, Any] = {
|
||||
"_env_file": None,
|
||||
"llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
|
||||
"import_extensions": extensions,
|
||||
}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return await import_sources([FIXTURES], LLMClient(settings))
|
||||
|
||||
|
||||
def _run_in_thread(coro: Any) -> Any:
|
||||
"""Run a coroutine on a worker thread.
|
||||
|
||||
Playwright's sync API keeps an asyncio loop running on the test
|
||||
thread, so ``asyncio.run`` cannot be called directly from a test
|
||||
body.
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(coro)
|
||||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||
box["error"] = e
|
||||
|
||||
t = Thread(target=runner)
|
||||
t.start()
|
||||
t.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _delete_source_rows() -> None:
|
||||
"""Delete every row of this suite's distinctive source (chunks
|
||||
cascade with the document rows)."""
|
||||
with SessionLocal() as db:
|
||||
for doc in db.scalars(select(Document).where(Document.source == SOURCE)).all():
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _drill(page: Page, *names: str) -> None:
|
||||
"""Drill one level at a time (phase 97 — client-side, no fetch,
|
||||
no URL change): each name is the EXACT text of the source/folder
|
||||
link at the current level."""
|
||||
for name in names:
|
||||
page.click(f'#folders-tbody a.folder-link:text-is("{name}")')
|
||||
|
||||
|
||||
def _go_top(page: Page) -> None:
|
||||
"""Back to the top level: the breadcrumb's top-level link (call
|
||||
between drills only — the breadcrumb is hidden at the top)."""
|
||||
page.locator("#kb-crumb a.kb-crumb-link").first.click()
|
||||
|
||||
|
||||
def _summary_lines(path: str) -> tuple[str, str]:
|
||||
"""(digest line, pointer line) of the stored phase-30 summary for a
|
||||
fixture file — mirrors the mock ``SUMMARY_MODE`` branch (first 24
|
||||
tokens of the document content) plus the code pointer line (the
|
||||
``test_summary_in_viewer.py`` house helper)."""
|
||||
content = (FIXTURES / path).read_text(encoding="utf-8")
|
||||
digest = " ".join(TOKEN_RE.findall(content.lower())[:24])
|
||||
expected = f"This document covers {digest}.\nSource: {SOURCE}/{path}"
|
||||
digest_line, pointer_line = expected.split("\n", 1)
|
||||
return digest_line, pointer_line
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def extensionless_kb(mock_llm: int, db_ready: None) -> Iterator[ImportSummary]:
|
||||
"""Seed the fixture with the extensionless scope
|
||||
(``md,dockerfile,containerfile``) for one test and delete every row
|
||||
it creates afterwards (DB isolation — see the module docstring)."""
|
||||
_delete_source_rows() # idempotent: leftovers from a crashed run
|
||||
summary = _run_in_thread(_import_fixtures(mock_llm, "md,dockerfile,containerfile"))
|
||||
try:
|
||||
yield summary
|
||||
finally:
|
||||
_delete_source_rows()
|
||||
|
||||
|
||||
def test_admin_tree_lists_name_token_files_and_viewer_badge(
|
||||
page: Page, app_url: str, extensionless_kb: ImportSummary
|
||||
) -> None:
|
||||
# The seed saw exactly the three in-scope files in their formats —
|
||||
# the two extensionless build files walked by name token, chunked,
|
||||
# and summarized; the Makefile negative control stayed out.
|
||||
assert extensionless_kb.formats == {"dockerfile": 1, "containerfile": 1, "md": 1}
|
||||
assert "unknown" not in extensionless_kb.formats
|
||||
assert (extensionless_kb.added, extensionless_kb.errors) == (3, 0)
|
||||
|
||||
login(page, app_url) # phase 16: the catalog is admin-only
|
||||
# All three fixture files live at the SOURCE level (no subfolders —
|
||||
# one drill reaches them, phase 97).
|
||||
_drill(page, SOURCE)
|
||||
for name in (DOCKER_REL, CONTAINER_REL, NOTES_REL):
|
||||
expect(page.locator("#docs-tbody tr", has_text=name)).to_have_count(1)
|
||||
# The negative control: no token names Makefile exactly — absent.
|
||||
expect(page.locator("#docs-tbody tr", has_text=MAKE_REL)).to_have_count(0)
|
||||
|
||||
# The Dockerfile row's path link opens the SAME-PAGE modal and its
|
||||
# meta row shows the D3 badge (``dockerfile`` — not the generic
|
||||
# ``text`` fallback) + the source badge + the stem title.
|
||||
row = page.locator("#docs-tbody tr", has_text=DOCKER_REL)
|
||||
link = row.locator("td:nth-child(2) a.doc-link")
|
||||
expect(link).to_have_count(1)
|
||||
expect(link).to_have_attribute("title", DOCKER_REL)
|
||||
before = len(page.context.pages)
|
||||
link.click()
|
||||
assert len(page.context.pages) == before, "clicking a row link must not open a new tab"
|
||||
expect(page.locator("#doc-modal-meta .doc-source-badge")).to_have_text(SOURCE)
|
||||
expect(page.locator("#doc-modal-meta .format-badge")).to_have_text("dockerfile")
|
||||
expect(page.locator("#doc-modal-title")).to_have_text(DOCKER_REL)
|
||||
|
||||
# Non-markdown content renders as escaped monospace text in a pre —
|
||||
# the sentinel proves it is THIS document's content.
|
||||
pre = page.locator("#doc-modal-content pre.doc-raw")
|
||||
expect(pre).to_have_count(1)
|
||||
expect(pre).to_contain_text(DOCKER_SENTINEL)
|
||||
|
||||
# The phase-30 summary line renders above the content (house
|
||||
# assertion style — test_summary_in_viewer.py): the deterministic
|
||||
# mock digest + the code pointer line, in the labeled panel.
|
||||
digest_line, pointer_line = _summary_lines(DOCKER_REL)
|
||||
panel = page.locator("#doc-modal .doc-summary")
|
||||
expect(panel).to_have_count(1)
|
||||
expect(panel).to_be_visible()
|
||||
expect(panel).to_have_attribute("aria-label", "Summary")
|
||||
expect(panel).to_contain_text(digest_line)
|
||||
expect(panel).to_contain_text(pointer_line)
|
||||
|
||||
# Still on the Sources page: no navigation happened.
|
||||
assert page.url == app_url + "/sources.html", f"navigated away: {page.url}"
|
||||
|
||||
|
||||
def test_anonymous_sources_gate_and_no_api_docs(
|
||||
page: Page, app_url: str, extensionless_kb: ImportSummary
|
||||
) -> None:
|
||||
"""A fresh anonymous context (function-scoped ``page`` = new
|
||||
browser context, no cookies): the sign-in gate renders and the page
|
||||
never calls ``/api/docs`` — the phase-16 pin, regression-checked
|
||||
with the extensionless KB seeded; the API itself 403s anonymous
|
||||
callers."""
|
||||
api_docs_calls: list[str] = []
|
||||
page.on(
|
||||
"request",
|
||||
lambda r: api_docs_calls.append(r.url) if "/api/docs" in r.url else None,
|
||||
)
|
||||
|
||||
page.goto(f"{app_url}/sources.html")
|
||||
# The gate, with its sign-in link — not a redirect.
|
||||
gate = page.locator("#sources-gate")
|
||||
expect(gate).to_be_visible()
|
||||
expect(gate).to_contain_text("Sign in to view the full catalog")
|
||||
expect(gate.locator("a[href='/login.html?next=/sources.html']")).to_have_count(1)
|
||||
# Stat cards + table hidden…
|
||||
expect(page.locator("#stat-cards")).to_be_hidden()
|
||||
expect(page.locator("#docs-table")).to_be_hidden()
|
||||
expect(page.locator("#sources-empty")).to_be_hidden()
|
||||
# …and NO /api/docs call was ever made.
|
||||
assert api_docs_calls == [], f"anonymous sources page called /api/docs: {api_docs_calls}"
|
||||
# And the API gate itself: an anonymous GET /api/docs 403s (phase 16
|
||||
# — the router sits behind require_admin).
|
||||
assert page.request.get(f"{app_url}/api/docs").status == 403
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# CONTAINERFILE-PROBE-SENTINEL-4b1c — phase 102 fixture marker: this
|
||||
# token exists nowhere else (the Dockerfile sibling carries its own).
|
||||
#
|
||||
# Containerfile — the local Postgres 17 + pgvector image the compose
|
||||
# stack builds for the Brain of Reese database service.
|
||||
|
||||
FROM postgres:17
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential git \
|
||||
&& git clone --branch v0.7.4 https://github.com/pgvector/pgvector /tmp/pgvector \
|
||||
&& make -C /tmp/pgvector install \
|
||||
&& rm -rf /tmp/pgvector \
|
||||
&& apt-get purge -y build-essential git
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
# DOCKERFILE-PROBE-SENTINEL-7a3e — phase 102 fixture marker: this token
|
||||
# exists nowhere else, so the extensionless_kb rows are unambiguous in
|
||||
# the shared database.
|
||||
#
|
||||
# Dockerfile — the Brain of Reese app image: a slim Python runtime that
|
||||
# serves the FastAPI app behind the homelab reverse proxy.
|
||||
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml uv.lock ./
|
||||
COPY app ./app
|
||||
RUN pip install --no-cache-dir uv && uv sync --frozen --no-dev
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["uv", "run", "uvicorn", "app.main:app", "--host", "0.0.0.0"]
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
# Makefile — phase 102 negative control: extensionless, and no token in
|
||||
# the env under test names it exactly, so the importer must skip it
|
||||
# (the exact-name rule never reaches for a compound or unrelated name).
|
||||
.PHONY: up down ps
|
||||
up:
|
||||
podman compose up -d
|
||||
down:
|
||||
podman compose down
|
||||
ps:
|
||||
podman compose ps
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
# Extensionless fixture notes
|
||||
|
||||
A short markdown control document for the phase 102 extensionless-kb
|
||||
fixture. With `import_extensions="md,dockerfile,containerfile"` the
|
||||
importer walks this file plus the name-token build files (`Dockerfile`,
|
||||
`Containerfile`); with `md` alone, this note is the only import.
|
||||
@@ -35,6 +35,7 @@ from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
import app.api.docs as docs_api
|
||||
from app.config import get_settings
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.llm import EmbeddingError
|
||||
@@ -555,3 +556,71 @@ def test_content_format_from_suffix(client, db) -> None:
|
||||
finally:
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_content_format_name_token_extensionless(
|
||||
client, db, monkeypatch
|
||||
) -> None:
|
||||
"""Phase 102: the endpoint's ``format`` carries the name token for an
|
||||
extensionless document whose lowercased full filename is a
|
||||
``BOR_IMPORT_EXTENSIONS`` token (``Dockerfile`` → ``dockerfile``),
|
||||
``text`` for an extensionless name that is NOT configured, and every
|
||||
suffixed value is unchanged (display never depends on the import
|
||||
list — ``notes/README.dev`` badges ``dev`` even though ``dev`` is not
|
||||
a configured token). The endpoint reads the real ``get_settings()``,
|
||||
so the token lands via the house env-override pattern (the default
|
||||
list does not contain ``dockerfile``)."""
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,dockerfile")
|
||||
|
||||
_seed_doc(
|
||||
db,
|
||||
path="services/api/Dockerfile",
|
||||
title="Dockerfile",
|
||||
content="FROM alpine\nCMD [\"/bin/sh\"]",
|
||||
chunks=0,
|
||||
)
|
||||
r = client.get(
|
||||
"/api/documents/content",
|
||||
params={"source": "Homelab", "path": "services/api/Dockerfile"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["format"] == "dockerfile"
|
||||
|
||||
# Suffix precedence under the same override — the import list is
|
||||
# ignored for suffixed paths.
|
||||
_seed_doc(
|
||||
db,
|
||||
path="notes/README.dev",
|
||||
title="README.dev",
|
||||
content="dev note",
|
||||
chunks=0,
|
||||
)
|
||||
r = client.get(
|
||||
"/api/documents/content",
|
||||
params={"source": "Homelab", "path": "notes/README.dev"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["format"] == "dev"
|
||||
|
||||
# Extensionless name NOT in the configured set → the ``text``
|
||||
# fallback (the existing README pin above covers the same case
|
||||
# under the default settings).
|
||||
_seed_doc(
|
||||
db,
|
||||
path="Containerfile",
|
||||
title="Containerfile",
|
||||
content="FROM alpine",
|
||||
chunks=0,
|
||||
)
|
||||
r = client.get(
|
||||
"/api/documents/content",
|
||||
params={"source": "Homelab", "path": "Containerfile"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["format"] == "text"
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Integration test: phase 102 — extensionless files named in
|
||||
``BOR_IMPORT_EXTENSIONS`` import end to end.
|
||||
|
||||
The owner's defect: ``Dockerfile`` / ``Containerfile`` never synced even
|
||||
with ``dockerfile`` / ``containerfile`` in the env, because the importer
|
||||
matched on the dotted suffix only (``Path("Dockerfile").suffix`` is
|
||||
``""``). Phase 102's rule (D1/D2): a suffix-less file is in scope iff its
|
||||
lowercased FULL filename equals a token of the set — exact name,
|
||||
case-insensitive — and the ``formats`` counter keys it by the matched
|
||||
token, never ``unknown``.
|
||||
|
||||
Proven here against the story-dedicated fixture directory
|
||||
``tests/fixtures/extensionless_kb/`` (the shared ``extension_kb``
|
||||
fixture of phase 56 stays pinned by its own suite):
|
||||
|
||||
* positive — ``md,dockerfile,containerfile`` walks ``Dockerfile``,
|
||||
``Containerfile`` and ``notes.md`` (the ``Makefile`` negative control
|
||||
stays out): rows + embedded chunks + the deterministic mock
|
||||
``SUMMARY_MODE`` digest for both build files (non-markdown → the
|
||||
phase-30 ``lite`` path), ``formats == {"dockerfile": 1,
|
||||
"containerfile": 1, "md": 1}`` with NO ``unknown`` key;
|
||||
* case — an on-disk ``DOCKERFILE`` imports under the ``dockerfile``
|
||||
token, its row keeps the on-disk case;
|
||||
* negative — ``md`` alone: only the control note imports;
|
||||
* prune — a token removed from the env drops its extensionless rows on
|
||||
the next ``prune=True`` run (D4 — the A9 junk precedent).
|
||||
|
||||
The LLM is the deterministic mock server (``tests/e2e/mock_llm.py``) on
|
||||
a scratch port — the integration analogue of the e2e ``mock_llm``
|
||||
fixture (the phase-56 integration pattern). Runs against the local
|
||||
compose Postgres (the ``db`` fixture from ``tests/conftest.py``); only
|
||||
the distinctive ``extensionless_kb`` / ``casekb`` source rows are
|
||||
created and deleted, so the rest of the shared KB is untouched.
|
||||
|
||||
Runs (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/integration/test_import_extensionless.py -v
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import Document
|
||||
from app.rag.importer import import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "extensionless_kb"
|
||||
SOURCE = FIXTURES.name # "extensionless_kb" — distinctive, never asserted by count
|
||||
DOCKER_REL = "Dockerfile"
|
||||
CONTAINER_REL = "Containerfile"
|
||||
NOTES_REL = "notes.md"
|
||||
DOCKER_SENTINEL = "DOCKERFILE-PROBE-SENTINEL-7a3e"
|
||||
CONTAINER_SENTINEL = "CONTAINERFILE-PROBE-SENTINEL-4b1c"
|
||||
#: The mock's SUMMARY_MODE tokenizes the document (``[a-z0-9]+``) — the
|
||||
#: hyphenated sentinels land in the digest in their tokenized forms.
|
||||
DOCKER_SENTINEL_TOKENS = "dockerfile probe sentinel 7a3e"
|
||||
CONTAINER_SENTINEL_TOKENS = "containerfile probe sentinel 4b1c"
|
||||
|
||||
|
||||
def _wait_http(url: str, timeout: float = 30.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
last_err = "unknown"
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
httpx.get(url, timeout=2.0)
|
||||
return
|
||||
except Exception as e: # noqa: BLE001 — retry until deadline
|
||||
last_err = str(e)
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError(f"mock LLM at {url} did not come up: {last_err}")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mock_llm_port() -> Iterator[int]:
|
||||
"""The deterministic mock LLM (``tests/e2e/mock_llm.py``) on a free
|
||||
scratch port — same server as the e2e ``mock_llm`` fixture, but
|
||||
private to this file (integration tests otherwise run network-free)."""
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
sock.close()
|
||||
env = dict(os.environ)
|
||||
env.pop("DEBUGPY", None)
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-m", "uvicorn", "tests.e2e.mock_llm:app",
|
||||
"--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
|
||||
cwd=REPO,
|
||||
env=env,
|
||||
)
|
||||
try:
|
||||
_wait_http(f"http://127.0.0.1:{port}/v1/models")
|
||||
yield port
|
||||
finally:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
|
||||
|
||||
def _settings(mock_port: int, extensions: str) -> Settings:
|
||||
kwargs: dict[str, Any] = {
|
||||
"_env_file": None,
|
||||
"llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
|
||||
"import_extensions": extensions,
|
||||
}
|
||||
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _cleanup_source(db: Session, source: str) -> None:
|
||||
for doc in db.scalars(select(Document).where(Document.source == source)).all():
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_name_token_files_import_end_to_end(mock_llm_port: int, db: Session) -> None:
|
||||
"""``md,dockerfile,containerfile`` over the fixture imports the two
|
||||
name-token build files + the markdown control: rows, embedded
|
||||
chunks, the mock ``SUMMARY_MODE`` digest for each build file, and
|
||||
the ``formats`` counter keyed by the matched token — never
|
||||
``unknown``. The ``Makefile`` (no token) stays out."""
|
||||
settings = _settings(mock_llm_port, "md,dockerfile,containerfile")
|
||||
assert settings.import_extension_set == {".md", ".dockerfile", ".containerfile"}
|
||||
summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
|
||||
try:
|
||||
# Exactly the three in-scope files walk — the Makefile negative
|
||||
# control and nothing else.
|
||||
assert (
|
||||
summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
|
||||
) == (3, 3, 0, 0, 0)
|
||||
# D2: extensionless files count under their matched token.
|
||||
assert summary.formats == {"dockerfile": 1, "containerfile": 1, "md": 1}
|
||||
assert "unknown" not in summary.formats
|
||||
# Phase 30: both build files are non-markdown → lite summaries;
|
||||
# the markdown control never is.
|
||||
assert (summary.summaries, summary.summary_errors) == (2, 0)
|
||||
|
||||
for rel, sentinel, tokens in (
|
||||
(DOCKER_REL, DOCKER_SENTINEL, DOCKER_SENTINEL_TOKENS),
|
||||
(CONTAINER_REL, CONTAINER_SENTINEL, CONTAINER_SENTINEL_TOKENS),
|
||||
):
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == rel)
|
||||
)
|
||||
assert doc is not None, f"{rel} was not imported by name token"
|
||||
# Suffix-less: the title is the file stem — the whole name.
|
||||
assert doc.title == rel
|
||||
# Plain-text chunking: the content (incl. the sentinel) is
|
||||
# chunked and every content chunk is embedded at the 768-dim
|
||||
# contract.
|
||||
content = [c for c in doc.chunks if not c.is_summary]
|
||||
assert content, f"{rel} has no content chunks"
|
||||
assert all(
|
||||
c.embedding is not None and len(c.embedding) == 768 for c in content
|
||||
)
|
||||
assert any(sentinel in c.content for c in content)
|
||||
# Mock SUMMARY_MODE digest: byte-stable, the tokenized
|
||||
# sentinel inside it, plus the code-appended pointer line.
|
||||
assert doc.summary is not None
|
||||
assert doc.summary.startswith("This document covers")
|
||||
assert tokens in doc.summary
|
||||
assert f"Source: {SOURCE}/{rel}" in doc.summary
|
||||
schunks = [c for c in doc.chunks if c.is_summary]
|
||||
assert len(schunks) == 1 and schunks[0].position == -1
|
||||
assert schunks[0].embedding is not None
|
||||
|
||||
# The markdown control doc imported too — but markdown never
|
||||
# gets a summary (phase 30).
|
||||
note = db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL)
|
||||
)
|
||||
assert note is not None
|
||||
assert note.summary is None
|
||||
assert [c for c in note.chunks if not c.is_summary]
|
||||
|
||||
# The negative control: no token names Makefile exactly — never
|
||||
# walked.
|
||||
assert (
|
||||
db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == "Makefile")
|
||||
)
|
||||
is None
|
||||
)
|
||||
finally:
|
||||
_cleanup_source(db, SOURCE)
|
||||
|
||||
|
||||
def test_uppercase_extensionless_name_imports(
|
||||
mock_llm_port: int, db: Session, tmp_path: Path
|
||||
) -> None:
|
||||
"""Case-insensitive exact name (D1): an on-disk ``DOCKERFILE``
|
||||
imports under the ``dockerfile`` token, and its row keeps the
|
||||
on-disk case in both ``path`` and ``title``."""
|
||||
root = tmp_path / "casekb"
|
||||
root.mkdir()
|
||||
(root / "DOCKERFILE").write_text("FROM alpine\nRUN apk add --no-cache curl\n")
|
||||
settings = _settings(mock_llm_port, "md,dockerfile")
|
||||
summary = asyncio.run(import_sources([root], LLMClient(settings), session=db))
|
||||
try:
|
||||
assert (summary.files, summary.added, summary.errors) == (1, 1, 0)
|
||||
assert summary.formats == {"dockerfile": 1}
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == "casekb", Document.path == "DOCKERFILE")
|
||||
)
|
||||
assert doc is not None, "DOCKERFILE did not import under the dockerfile token"
|
||||
assert doc.title == "DOCKERFILE"
|
||||
assert [c for c in doc.chunks if not c.is_summary]
|
||||
# Non-markdown → the phase-30 lite path ran.
|
||||
assert summary.summaries == 1
|
||||
assert doc.summary is not None
|
||||
assert doc.summary.startswith("This document covers")
|
||||
finally:
|
||||
_cleanup_source(db, "casekb")
|
||||
|
||||
|
||||
def test_without_tokens_the_extensionless_files_stay_out(
|
||||
mock_llm_port: int, db: Session
|
||||
) -> None:
|
||||
"""``md`` alone: the control note imports; ``Dockerfile`` /
|
||||
``Containerfile`` (their names are not tokens in this scope) and the
|
||||
``Makefile`` are all out of scope — no value of the scope walks
|
||||
them."""
|
||||
settings = _settings(mock_llm_port, "md")
|
||||
assert settings.import_extension_set == {".md"}
|
||||
summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
|
||||
try:
|
||||
assert (
|
||||
summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
|
||||
) == (1, 1, 0, 0, 0)
|
||||
assert summary.formats == {"md": 1}
|
||||
assert summary.summaries == 0
|
||||
for rel in (DOCKER_REL, CONTAINER_REL, "Makefile"):
|
||||
assert (
|
||||
db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == rel)
|
||||
)
|
||||
is None
|
||||
), f"{rel} was imported although its name is not a token in `md`"
|
||||
assert (
|
||||
db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
finally:
|
||||
_cleanup_source(db, SOURCE)
|
||||
|
||||
|
||||
def test_prune_removes_extensionless_rows_when_the_token_is_removed(
|
||||
mock_llm_port: int, db: Session
|
||||
) -> None:
|
||||
"""D4: a file that stops matching — the token leaves the env —
|
||||
leaves ``seen`` and is deleted by the next ``prune=True`` run (the
|
||||
A9 junk precedent); the in-scope ``notes.md`` row survives."""
|
||||
with_tokens = _settings(mock_llm_port, "md,dockerfile,containerfile")
|
||||
asyncio.run(import_sources([FIXTURES], LLMClient(with_tokens), session=db))
|
||||
try:
|
||||
md_only = _settings(mock_llm_port, "md")
|
||||
summary = asyncio.run(
|
||||
import_sources([FIXTURES], LLMClient(md_only), session=db, prune=True)
|
||||
)
|
||||
assert summary.pruned == 2, "the two extensionless rows must be pruned"
|
||||
assert summary.unchanged == 1, "notes.md must survive the prune"
|
||||
for rel in (DOCKER_REL, CONTAINER_REL):
|
||||
assert (
|
||||
db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == rel)
|
||||
)
|
||||
is None
|
||||
), f"{rel} survived although its token left the env"
|
||||
assert (
|
||||
db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == NOTES_REL)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
finally:
|
||||
_cleanup_source(db, SOURCE)
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
Python side:
|
||||
* ``doc_format`` — format from the path suffix (incl. ``.markdown`` and the
|
||||
no-suffix fallback);
|
||||
no-suffix fallback) + the phase-102 extensionless name-token rule
|
||||
(``Dockerfile`` → ``dockerfile`` with a configured token, suffixes still
|
||||
unconditional, no-arg calls byte-identical to the suffix-only rule);
|
||||
* the content endpoint's 200/404 mapping — tested WITHOUT a database by
|
||||
stubbing the session via FastAPI's dependency override (unknown pairs and
|
||||
traversal-style paths map to 404 ``{detail: "document not found"}``;
|
||||
@@ -74,6 +76,43 @@ def test_doc_format_from_suffix(path: str, expected: str) -> None:
|
||||
assert doc_format(path) == expected
|
||||
|
||||
|
||||
def test_doc_format_no_args_extensionless_falls_back_to_text() -> None:
|
||||
"""The no-arg contract: default ``extensions=frozenset()`` keeps the
|
||||
pre-phase-102 result for every path — an extensionless name that LOOKS
|
||||
like a configured token still badges ``text`` without the token set."""
|
||||
assert doc_format("Dockerfile") == "text"
|
||||
assert doc_format("README") == "text"
|
||||
|
||||
|
||||
#: A token set in the dotted form ``import_extension_set`` passes it.
|
||||
_TOKEN_EXTS = frozenset({".md", ".dev", ".dockerfile", ".containerfile"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "extensions", "expected"),
|
||||
[
|
||||
# Name-token branch (phase 102): suffix-less name in the set.
|
||||
("services/api/Dockerfile", _TOKEN_EXTS, "dockerfile"),
|
||||
("DOCKERFILE", _TOKEN_EXTS, "dockerfile"), # case-insensitive name
|
||||
("Containerfile", _TOKEN_EXTS, "containerfile"),
|
||||
# Suffix precedence: display never depends on the import list.
|
||||
("Dockerfile.dev", _TOKEN_EXTS, "dev"), # suffixed → its suffix rules
|
||||
("readme.rst", _TOKEN_EXTS, "rst"), # out-of-scope suffix still badges
|
||||
("notes/README.dev", _TOKEN_EXTS, "dev"),
|
||||
("dockerfile.bak", _TOKEN_EXTS, "bak"), # lookalike → suffix, not name
|
||||
("kubernetes.md", frozenset(), "md"), # empty set: suffix unconditional
|
||||
# Extensionless names NOT in the set fall back to text — exact
|
||||
# name only, no partial names.
|
||||
("README", _TOKEN_EXTS, "text"),
|
||||
("mydockerfile", _TOKEN_EXTS, "text"),
|
||||
],
|
||||
)
|
||||
def test_doc_format_with_token_set(
|
||||
path: str, extensions: frozenset[str], expected: str
|
||||
) -> None:
|
||||
assert doc_format(path, extensions) == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content endpoint mapping — stubbed session, no database required
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -27,6 +27,7 @@ from app.rag.importer import (
|
||||
_store_summary,
|
||||
import_sources,
|
||||
iter_importable_files,
|
||||
match_extension,
|
||||
)
|
||||
from app.rag.llm import EmbeddingError
|
||||
from tests.fakes import FakeEmbedder
|
||||
@@ -59,6 +60,15 @@ class _CapEmbedder(FakeEmbedder):
|
||||
return await super().embed(texts)
|
||||
|
||||
|
||||
def _embedder_with_extensions(extensions: str) -> FakeEmbedder:
|
||||
"""A :class:`FakeEmbedder` whose settings carry a custom
|
||||
``BOR_IMPORT_EXTENSIONS`` CSV (phase 102 — the import scope the walk
|
||||
and the ``formats`` counter read from ``llm.settings``)."""
|
||||
llm = FakeEmbedder()
|
||||
llm.settings = Settings(_env_file=None, import_extensions=extensions) # pyright: ignore[reportCallIssue]
|
||||
return llm
|
||||
|
||||
|
||||
def _cleanup_source(db, source: str) -> None:
|
||||
for doc in db.scalars(select(Document).where(Document.source == source)).all():
|
||||
db.delete(doc)
|
||||
@@ -199,6 +209,48 @@ def test_iter_importable_files_respects_custom_extension_filter(tmp_path: Path)
|
||||
assert found == {"a.md"}
|
||||
|
||||
|
||||
def test_match_extension_matrix() -> None:
|
||||
"""Phase 102, D1 — the matching matrix: the A9 dotted-suffix rule first,
|
||||
then the extensionless exact-name rule (case-insensitive), exact name
|
||||
only — no partial matching, suffixed files governed by their suffix."""
|
||||
exts = frozenset({".md", ".dockerfile"})
|
||||
# Rule 1 — the dotted suffix (case-insensitive, as today):
|
||||
assert match_extension(Path("kubernetes.md"), frozenset({".md"})) == "md"
|
||||
assert match_extension(Path("Kubernetes.MD"), frozenset({".md"})) == "md"
|
||||
# Rule 2 — extensionless files by exact lowercased FULL filename:
|
||||
assert match_extension(Path("Dockerfile"), exts) == "dockerfile"
|
||||
assert match_extension(Path("DOCKERFILE"), exts) == "dockerfile"
|
||||
# …only when the token is actually in the set:
|
||||
assert match_extension(Path("Dockerfile"), frozenset({".md"})) is None
|
||||
# Exact name only — compound lookalikes never match the token:
|
||||
assert match_extension(Path("mydockerfile"), exts) is None
|
||||
# A suffixed file is governed by its suffix, never its name:
|
||||
assert match_extension(Path("Dockerfile.dev"), exts) is None
|
||||
assert match_extension(Path("Dockerfile.dev"), frozenset({".md", ".dev"})) == "dev"
|
||||
# An out-of-scope suffix is out of scope, name be damned:
|
||||
assert match_extension(Path("readme.rst"), frozenset({".md"})) is None
|
||||
|
||||
|
||||
def test_iter_importable_files_walks_extensionless_name_tokens(tmp_path: Path) -> None:
|
||||
"""Phase 102, D1 — an extensionless file walks iff its lowercased full
|
||||
name is a token (``md,dockerfile,containerfile`` here): lookalikes and
|
||||
the dot-prefixed hidden file stay skipped by the pre-existing rules."""
|
||||
root = tmp_path / "build"
|
||||
root.mkdir()
|
||||
for name in (
|
||||
"Dockerfile",
|
||||
"Containerfile",
|
||||
"mydockerfile", # compound name — never matches the `dockerfile` token
|
||||
"Dockerfile.dev", # governed by its .dev suffix (not a token here)
|
||||
".dockerfile", # dot-prefixed FILE — the hidden-component rule
|
||||
"notes.md",
|
||||
):
|
||||
(root / name).write_text(f"# {name}\n\nbody {name}\n")
|
||||
exts = frozenset({".md", ".dockerfile", ".containerfile"})
|
||||
found = [p.name for p in iter_importable_files(root, exts)]
|
||||
assert found == ["Containerfile", "Dockerfile", "notes.md"]
|
||||
|
||||
|
||||
def test_excluded_dirs_match_plan_anchor_a9() -> None:
|
||||
assert {
|
||||
".venv", "node_modules", ".git", "__pycache__", ".pytest_cache", "dist", "build"
|
||||
@@ -711,6 +763,36 @@ def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 102: extensionless name-token import ----------
|
||||
|
||||
|
||||
def test_formats_counter_counts_extensionless_name_token(
|
||||
db, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""Phase 102, D2 — an imported ``Dockerfile`` counts under
|
||||
``dockerfile`` in ``summary.formats`` and the PLAN §9 line, never
|
||||
``unknown``."""
|
||||
root = tmp_path / "extless"
|
||||
root.mkdir()
|
||||
(root / "Dockerfile").write_text("FROM alpine\n\nCMD [\"/bin/sh\"]\n")
|
||||
(root / "notes.md").write_text("# Notes\n\nbody\n")
|
||||
llm = _embedder_with_extensions("md,dockerfile")
|
||||
try:
|
||||
with caplog.at_level(logging.INFO, logger="app.importer"):
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.files == 2 and summary.added == 2
|
||||
assert summary.formats == {"dockerfile": 1, "md": 1}
|
||||
line = next(
|
||||
r.getMessage()
|
||||
for r in caplog.records
|
||||
if "import: summary files=" in r.getMessage()
|
||||
)
|
||||
assert line.endswith("formats=dockerfile:1,md:1")
|
||||
assert "unknown" not in line
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 64 (task 01): optional per-file progress hook ----------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user