phase: 102_extensionless_filenames
All verification complete — every gate green, no defects found in previously completed work. **Phase 102 final verification pass — report** Verified (all three task files present in `complete/`; working-tree implementation matches D1–D5 design): - `match_extension` choke point in `app/rag/importer.py` (walk + `formats` counter), `doc_format` name-token badge in `app/api/docs.py`, config/`.env.example` docs, fixture `tests/fixtures/extensionless_kb/`, integration + E2E suites — all present and correct - Completion criteria: end-to-end sync (✓ integration + E2E), case matrix incl. `mydockerfile`/`Dockerfile.dev`/`.dockerfile` exclusions (✓ unit), `formats=dockerfile:1` not `unknown` (✓ log-line assertion), badge `dockerfile`/`containerfile` + `text` fallback + suffixed unchanged (✓ unit/integration/E2E), prune-on-token-removal (✓ `pruned==2`), suffixed-path rule byte-identical (✓ single-line swap, existing cases untouched) Test / lint results (exact commands): - `uv run pytest --cov=app --cov-report=term-missing` → 2084 passed, **99%** coverage (>90% gate) - `uv run pytest tests/e2e/test_extensionless_import.py -v --no-cov` → 2 passed, isolated, DB up - Regressions isolated: `test_import_documents` 3✓, `test_import_extensions_env` 2✓, `test_quadlet_jinja_import` 4✓, `test_document_viewer` 7✓, `test_kb_tree` 8✓ - `uv run ruff check .` → clean; `uv run pyright` → 0 errors, 0 warnings Notable: commit intentionally not made (harness commits the phase); 102's task files already sit in `complete/`, overview stays in `todo/` for the harness. Next pending phases: 98, 99, 103, 104, 105 (numeric next after 102: `103_suggestions_session_openers`).
This commit is contained in:
@@ -2,7 +2,9 @@
|
||||
|
||||
Python side:
|
||||
* ``doc_format`` — format from the path suffix (incl. ``.markdown`` and the
|
||||
no-suffix fallback);
|
||||
no-suffix fallback) + the phase-102 extensionless name-token rule
|
||||
(``Dockerfile`` → ``dockerfile`` with a configured token, suffixes still
|
||||
unconditional, no-arg calls byte-identical to the suffix-only rule);
|
||||
* the content endpoint's 200/404 mapping — tested WITHOUT a database by
|
||||
stubbing the session via FastAPI's dependency override (unknown pairs and
|
||||
traversal-style paths map to 404 ``{detail: "document not found"}``;
|
||||
@@ -74,6 +76,43 @@ def test_doc_format_from_suffix(path: str, expected: str) -> None:
|
||||
assert doc_format(path) == expected
|
||||
|
||||
|
||||
def test_doc_format_no_args_extensionless_falls_back_to_text() -> None:
|
||||
"""The no-arg contract: default ``extensions=frozenset()`` keeps the
|
||||
pre-phase-102 result for every path — an extensionless name that LOOKS
|
||||
like a configured token still badges ``text`` without the token set."""
|
||||
assert doc_format("Dockerfile") == "text"
|
||||
assert doc_format("README") == "text"
|
||||
|
||||
|
||||
#: A token set in the dotted form ``import_extension_set`` passes it.
|
||||
_TOKEN_EXTS = frozenset({".md", ".dev", ".dockerfile", ".containerfile"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "extensions", "expected"),
|
||||
[
|
||||
# Name-token branch (phase 102): suffix-less name in the set.
|
||||
("services/api/Dockerfile", _TOKEN_EXTS, "dockerfile"),
|
||||
("DOCKERFILE", _TOKEN_EXTS, "dockerfile"), # case-insensitive name
|
||||
("Containerfile", _TOKEN_EXTS, "containerfile"),
|
||||
# Suffix precedence: display never depends on the import list.
|
||||
("Dockerfile.dev", _TOKEN_EXTS, "dev"), # suffixed → its suffix rules
|
||||
("readme.rst", _TOKEN_EXTS, "rst"), # out-of-scope suffix still badges
|
||||
("notes/README.dev", _TOKEN_EXTS, "dev"),
|
||||
("dockerfile.bak", _TOKEN_EXTS, "bak"), # lookalike → suffix, not name
|
||||
("kubernetes.md", frozenset(), "md"), # empty set: suffix unconditional
|
||||
# Extensionless names NOT in the set fall back to text — exact
|
||||
# name only, no partial names.
|
||||
("README", _TOKEN_EXTS, "text"),
|
||||
("mydockerfile", _TOKEN_EXTS, "text"),
|
||||
],
|
||||
)
|
||||
def test_doc_format_with_token_set(
|
||||
path: str, extensions: frozenset[str], expected: str
|
||||
) -> None:
|
||||
assert doc_format(path, extensions) == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content endpoint mapping — stubbed session, no database required
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -27,6 +27,7 @@ from app.rag.importer import (
|
||||
_store_summary,
|
||||
import_sources,
|
||||
iter_importable_files,
|
||||
match_extension,
|
||||
)
|
||||
from app.rag.llm import EmbeddingError
|
||||
from tests.fakes import FakeEmbedder
|
||||
@@ -59,6 +60,15 @@ class _CapEmbedder(FakeEmbedder):
|
||||
return await super().embed(texts)
|
||||
|
||||
|
||||
def _embedder_with_extensions(extensions: str) -> FakeEmbedder:
|
||||
"""A :class:`FakeEmbedder` whose settings carry a custom
|
||||
``BOR_IMPORT_EXTENSIONS`` CSV (phase 102 — the import scope the walk
|
||||
and the ``formats`` counter read from ``llm.settings``)."""
|
||||
llm = FakeEmbedder()
|
||||
llm.settings = Settings(_env_file=None, import_extensions=extensions) # pyright: ignore[reportCallIssue]
|
||||
return llm
|
||||
|
||||
|
||||
def _cleanup_source(db, source: str) -> None:
|
||||
for doc in db.scalars(select(Document).where(Document.source == source)).all():
|
||||
db.delete(doc)
|
||||
@@ -199,6 +209,48 @@ def test_iter_importable_files_respects_custom_extension_filter(tmp_path: Path)
|
||||
assert found == {"a.md"}
|
||||
|
||||
|
||||
def test_match_extension_matrix() -> None:
|
||||
"""Phase 102, D1 — the matching matrix: the A9 dotted-suffix rule first,
|
||||
then the extensionless exact-name rule (case-insensitive), exact name
|
||||
only — no partial matching, suffixed files governed by their suffix."""
|
||||
exts = frozenset({".md", ".dockerfile"})
|
||||
# Rule 1 — the dotted suffix (case-insensitive, as today):
|
||||
assert match_extension(Path("kubernetes.md"), frozenset({".md"})) == "md"
|
||||
assert match_extension(Path("Kubernetes.MD"), frozenset({".md"})) == "md"
|
||||
# Rule 2 — extensionless files by exact lowercased FULL filename:
|
||||
assert match_extension(Path("Dockerfile"), exts) == "dockerfile"
|
||||
assert match_extension(Path("DOCKERFILE"), exts) == "dockerfile"
|
||||
# …only when the token is actually in the set:
|
||||
assert match_extension(Path("Dockerfile"), frozenset({".md"})) is None
|
||||
# Exact name only — compound lookalikes never match the token:
|
||||
assert match_extension(Path("mydockerfile"), exts) is None
|
||||
# A suffixed file is governed by its suffix, never its name:
|
||||
assert match_extension(Path("Dockerfile.dev"), exts) is None
|
||||
assert match_extension(Path("Dockerfile.dev"), frozenset({".md", ".dev"})) == "dev"
|
||||
# An out-of-scope suffix is out of scope, name be damned:
|
||||
assert match_extension(Path("readme.rst"), frozenset({".md"})) is None
|
||||
|
||||
|
||||
def test_iter_importable_files_walks_extensionless_name_tokens(tmp_path: Path) -> None:
|
||||
"""Phase 102, D1 — an extensionless file walks iff its lowercased full
|
||||
name is a token (``md,dockerfile,containerfile`` here): lookalikes and
|
||||
the dot-prefixed hidden file stay skipped by the pre-existing rules."""
|
||||
root = tmp_path / "build"
|
||||
root.mkdir()
|
||||
for name in (
|
||||
"Dockerfile",
|
||||
"Containerfile",
|
||||
"mydockerfile", # compound name — never matches the `dockerfile` token
|
||||
"Dockerfile.dev", # governed by its .dev suffix (not a token here)
|
||||
".dockerfile", # dot-prefixed FILE — the hidden-component rule
|
||||
"notes.md",
|
||||
):
|
||||
(root / name).write_text(f"# {name}\n\nbody {name}\n")
|
||||
exts = frozenset({".md", ".dockerfile", ".containerfile"})
|
||||
found = [p.name for p in iter_importable_files(root, exts)]
|
||||
assert found == ["Containerfile", "Dockerfile", "notes.md"]
|
||||
|
||||
|
||||
def test_excluded_dirs_match_plan_anchor_a9() -> None:
|
||||
assert {
|
||||
".venv", "node_modules", ".git", "__pycache__", ".pytest_cache", "dist", "build"
|
||||
@@ -711,6 +763,36 @@ def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 102: extensionless name-token import ----------
|
||||
|
||||
|
||||
def test_formats_counter_counts_extensionless_name_token(
|
||||
db, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""Phase 102, D2 — an imported ``Dockerfile`` counts under
|
||||
``dockerfile`` in ``summary.formats`` and the PLAN §9 line, never
|
||||
``unknown``."""
|
||||
root = tmp_path / "extless"
|
||||
root.mkdir()
|
||||
(root / "Dockerfile").write_text("FROM alpine\n\nCMD [\"/bin/sh\"]\n")
|
||||
(root / "notes.md").write_text("# Notes\n\nbody\n")
|
||||
llm = _embedder_with_extensions("md,dockerfile")
|
||||
try:
|
||||
with caplog.at_level(logging.INFO, logger="app.importer"):
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.files == 2 and summary.added == 2
|
||||
assert summary.formats == {"dockerfile": 1, "md": 1}
|
||||
line = next(
|
||||
r.getMessage()
|
||||
for r in caplog.records
|
||||
if "import: summary files=" in r.getMessage()
|
||||
)
|
||||
assert line.endswith("formats=dockerfile:1,md:1")
|
||||
assert "unknown" not in line
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# ---------- phase 64 (task 01): optional per-file progress hook ----------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user