feat(import): user-extensible BOR_IMPORT_EXTENSIONS — any well-formed extension, A9 family stays the default
This commit is contained in:
+7
-2
@@ -40,8 +40,13 @@ BOR_RRF_K=60 # Reciprocal Rank Fusion damping constant
|
|||||||
# --- Agent document tools (phase 37: grounded turns may list + read) ---
|
# --- Agent document tools (phase 37: grounded turns may list + read) ---
|
||||||
# BOR_AGENT_MAX_ROUNDS=10 # hard cap on agent tool rounds per turn (0 = no tools)
|
# BOR_AGENT_MAX_ROUNDS=10 # hard cap on agent tool rounds per turn (0 = no tools)
|
||||||
|
|
||||||
# --- Import scope (A9 formats; may only narrow, never widen) ---
|
# --- Import scope (A9 default; ANY well-formed extension is allowed) ---
|
||||||
# BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,container,network,volume,image,pod,kube,swap,os,endpoint,j2
|
# Comma-separated file extensions (lowercase, no dot) the importer reads.
|
||||||
|
# Any extension is allowed — the value below is the built-in default (the
|
||||||
|
# A9 family: the original seven + the quadlet family + jinja ``j2``); add
|
||||||
|
# your own (e.g. md,sh,toml) or narrow it (e.g. md). A blank list or a
|
||||||
|
# malformed token (e.g. md,sh!) fails startup loudly, naming the value.
|
||||||
|
BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,container,network,volume,image,pod,kube,swap,os,endpoint,j2
|
||||||
# BOR_SUGGESTIONS=["How is my Kubernetes cluster set up?"] # JSON list of onboarding chips
|
# BOR_SUGGESTIONS=["How is my Kubernetes cluster set up?"] # JSON list of onboarding chips
|
||||||
|
|
||||||
# --- Import sources (git; phase 28, admin-managed since phase 35) ---
|
# --- Import sources (git; phase 28, admin-managed since phase 35) ---
|
||||||
|
|||||||
+30
-16
@@ -6,18 +6,22 @@ Every setting can be overridden with an environment variable prefixed
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
|
||||||
from pydantic import field_validator
|
from pydantic import field_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
#: The A9 import formats (PLAN anchor A9, revised 2026-08-21; revised
|
#: The built-in DEFAULT import formats (PLAN anchor A9, revised 2026-08-21;
|
||||||
#: 2026-08-27, owner permission — the full Podman quadlet family
|
#: revised 2026-08-27, owner permission — the full Podman quadlet family
|
||||||
#: ``container, network, volume, image, pod, kube, swap, os, endpoint``
|
#: ``container, network, volume, image, pod, kube, swap, os, endpoint``
|
||||||
#: plus Jinja templates ``j2`` join the allowed set, chunked as plain
|
#: plus Jinja templates ``j2`` join the default, chunked as plain text).
|
||||||
#: text). ``BOR_IMPORT_EXTENSIONS`` may narrow — but never widen — this
|
#: This is the default scope AND the ``.env.example`` example — it is NOT
|
||||||
#: set.
|
#: a ceiling: ``BOR_IMPORT_EXTENSIONS`` may name **any** well-formed
|
||||||
_ALLOWED_IMPORT_EXTENSIONS: frozenset[str] = frozenset(
|
#: extension (lowercase letters/digits, no dot) or narrow to a subset
|
||||||
|
#: (owner permission 2026-08-31, phase 56); see
|
||||||
|
#: :py:attr:`Settings.import_extensions`.
|
||||||
|
_DEFAULT_IMPORT_EXTENSIONS: frozenset[str] = frozenset(
|
||||||
{
|
{
|
||||||
"md", "markdown", "txt", "yaml", "yml", "json", "py",
|
"md", "markdown", "txt", "yaml", "yml", "json", "py",
|
||||||
# A9 revised 2026-08-27 (owner permission): quadlet family + jinja.
|
# A9 revised 2026-08-27 (owner permission): quadlet family + jinja.
|
||||||
@@ -136,13 +140,18 @@ class Settings(BaseSettings):
|
|||||||
session_max_age: int = 43_200
|
session_max_age: int = 43_200
|
||||||
session_cookie: str = "bor_session"
|
session_cookie: str = "bor_session"
|
||||||
|
|
||||||
# --- Import scope (A9, revised 2026-08-21 and 2026-08-27) ---
|
# --- Import scope (A9 default; any extension allowed — phase 56) ---
|
||||||
# Comma-separated list of lowercased file extensions (no dot) imported
|
# Comma-separated list of lowercased file extensions (no dot) imported
|
||||||
# by ``scripts/import_docs.py``. Hidden (dot) path components are always
|
# by ``scripts/import_docs.py``. **Any** well-formed extension is
|
||||||
|
# allowed (lowercase letters/digits, 1-16 chars — the shape guard
|
||||||
|
# doubles as the typo guard); the value below is the built-in default
|
||||||
|
# (the A9 family, incl. the quadlet family + ``j2``) and the documented
|
||||||
|
# example in ``.env.example``. Hidden (dot) path components are always
|
||||||
# skipped, plus the importer's exclusion list.
|
# skipped, plus the importer's exclusion list.
|
||||||
# Stored as a raw CSV string (env-native — no JSON) and parsed on demand
|
# Stored as a raw CSV string (env-native — no JSON) and parsed on demand
|
||||||
# via :py:meth:`import_extension_set`. ``mode="after"`` validation runs
|
# via :py:meth:`import_extension_set`. The validator rejects an empty
|
||||||
# against the raw string so a typo fails loudly at startup.
|
# list and malformed tokens so a typo fails loudly at startup (it can
|
||||||
|
# no longer reject a novel extension).
|
||||||
import_extensions: str = (
|
import_extensions: str = (
|
||||||
"md,markdown,txt,yaml,yml,json,py,"
|
"md,markdown,txt,yaml,yml,json,py,"
|
||||||
"container,network,volume,image,pod,kube,swap,os,endpoint,j2"
|
"container,network,volume,image,pod,kube,swap,os,endpoint,j2"
|
||||||
@@ -172,16 +181,21 @@ class Settings(BaseSettings):
|
|||||||
@field_validator("import_extensions")
|
@field_validator("import_extensions")
|
||||||
@classmethod
|
@classmethod
|
||||||
def _import_extensions_known(cls, v: str) -> str:
|
def _import_extensions_known(cls, v: str) -> str:
|
||||||
"""Reject unknown/empty formats loudly instead of silently importing
|
"""Reject an empty list or malformed tokens loudly instead of
|
||||||
nothing (a typo like ``md,jsonn`` would otherwise walk zero files)."""
|
silently importing nothing (a typo like ``md,jsonn`` would
|
||||||
|
otherwise walk zero files). Any well-formed extension is accepted —
|
||||||
|
the A9 family is the default, not a ceiling (owner permission
|
||||||
|
2026-08-31, phase 56)."""
|
||||||
exts = {part.strip().lstrip(".").lower() for part in v.split(",") if part.strip()}
|
exts = {part.strip().lstrip(".").lower() for part in v.split(",") if part.strip()}
|
||||||
if not exts:
|
if not exts:
|
||||||
raise ValueError("import_extensions must name at least one format")
|
raise ValueError("import_extensions must name at least one format")
|
||||||
unknown = exts - _ALLOWED_IMPORT_EXTENSIONS
|
malformed = sorted(
|
||||||
if unknown:
|
ext for ext in exts if re.fullmatch(r"[a-z0-9]{1,16}", ext) is None
|
||||||
|
)
|
||||||
|
if malformed:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"unknown import extension(s): {', '.join(sorted(unknown))} — "
|
f"import_extensions contains malformed token(s): {', '.join(malformed)} — "
|
||||||
f"allowed: {', '.join(sorted(_ALLOWED_IMPORT_EXTENSIONS))}"
|
"each extension must be lowercase letters/digits only, 1-16 chars, no dot"
|
||||||
)
|
)
|
||||||
return v
|
return v
|
||||||
|
|
||||||
|
|||||||
+3
-2
@@ -1,7 +1,8 @@
|
|||||||
"""Knowledge-base importer (PLAN §5 / §9 / §11).
|
"""Knowledge-base importer (PLAN §5 / §9 / §11).
|
||||||
|
|
||||||
Walks the A9-format files (``md, markdown, txt, yaml, yml, json, py`` by
|
Walks the in-scope files (the A9 family by default — the original seven
|
||||||
default — ``BOR_IMPORT_EXTENSIONS``; case-insensitive), diffs by sha256
|
plus the quadlet family and ``j2`` — ``BOR_IMPORT_EXTENSIONS``, which may
|
||||||
|
name any well-formed extension; case-insensitive), diffs by sha256
|
||||||
against ``documents.content_hash`` and, for every new or changed file, runs
|
against ``documents.content_hash`` and, for every new or changed file, runs
|
||||||
the two-phase upsert:
|
the two-phase upsert:
|
||||||
|
|
||||||
|
|||||||
@@ -26,8 +26,11 @@ precedence order:
|
|||||||
``~/Deployments``), kept for backwards compatibility (reached only
|
``~/Deployments``), kept for backwards compatibility (reached only
|
||||||
while both the table and ``BOR_GIT_SOURCES`` are empty).
|
while both the table and ``BOR_GIT_SOURCES`` are empty).
|
||||||
|
|
||||||
Imported formats (PLAN anchor A9, revised): ``md, markdown, txt, yaml,
|
Imported formats (PLAN anchor A9, revised; phase 56): the A9 family by
|
||||||
yml, json, py`` (case-insensitive; narrow with ``BOR_IMPORT_EXTENSIONS``).
|
default — ``md, markdown, txt, yaml, yml, json, py`` plus the quadlet
|
||||||
|
family and ``j2`` (case-insensitive). ``BOR_IMPORT_EXTENSIONS`` may add
|
||||||
|
ANY well-formed extension or narrow the list (the A9 family is the
|
||||||
|
default, not a ceiling — owner permission 2026-08-31).
|
||||||
Any path with a dot-prefixed component (hidden files/dirs — vendored
|
Any path with a dot-prefixed component (hidden files/dirs — vendored
|
||||||
caches) is skipped, along with non-content dirs (``.venv``,
|
caches) is skipped, along with non-content dirs (``.venv``,
|
||||||
``node_modules``, ``.git``, ``__pycache``, ``.pytest_cache``, ``dist``,
|
``node_modules``, ``.git``, ``__pycache``, ``.pytest_cache``, ``dist``,
|
||||||
|
|||||||
@@ -0,0 +1,163 @@
|
|||||||
|
"""Phase 56 E2E (Playwright): a NOVEL extension (``.sh``) flows config →
|
||||||
|
import → chunks → mock summary → Sources page.
|
||||||
|
|
||||||
|
TODO.md L6: "Allow the user to specify extensions to be read in .env,
|
||||||
|
don't hard-code working extensions." The subject is the env-driven
|
||||||
|
extension scope (``import_extensions="md,sh"``); the story-dedicated
|
||||||
|
fixture (``tests/fixtures/extension_kb/``) is seeded in-process against
|
||||||
|
the deterministic mock LLM — the phase-02 seeding-thread pattern, the
|
||||||
|
fixture, not the subject of the tests.
|
||||||
|
|
||||||
|
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||||
|
|
||||||
|
uv run pytest tests/e2e/test_import_extensions_env.py -v --no-cov
|
||||||
|
|
||||||
|
DB isolation: the fixture's source name (``extension_kb``) is
|
||||||
|
distinctive — the suite never asserts on absolute row counts and
|
||||||
|
deletes the rows it creates in a ``finally`` (other suites' documents
|
||||||
|
stay untouched in the shared E2E database).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from pathlib import Path
|
||||||
|
from threading import Thread
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from playwright.sync_api import Page, expect
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.config import Settings
|
||||||
|
from app.db import SessionLocal
|
||||||
|
from app.models import Document
|
||||||
|
from app.rag.importer import ImportSummary, import_sources
|
||||||
|
from app.rag.llm import LLMClient
|
||||||
|
from e2e.auth_helpers import login
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
FIXTURES = REPO / "tests" / "fixtures" / "extension_kb"
|
||||||
|
SOURCE = FIXTURES.name # "extension_kb" — distinctive, never asserted by count
|
||||||
|
SH_REL = "homelab/scripts/uptime.sh"
|
||||||
|
MD_REL = "homelab/notes/note.md"
|
||||||
|
SENTINEL = "UPTIME-PROBE-SENTINEL-9c2f"
|
||||||
|
|
||||||
|
|
||||||
|
async def _import_fixtures(mock_port: int, extensions: str) -> ImportSummary:
|
||||||
|
kwargs: dict[str, Any] = {
|
||||||
|
"_env_file": None,
|
||||||
|
"llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
|
||||||
|
"import_extensions": extensions,
|
||||||
|
}
|
||||||
|
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||||
|
return await import_sources([FIXTURES], LLMClient(settings))
|
||||||
|
|
||||||
|
|
||||||
|
def _run_in_thread(coro: Any) -> Any:
|
||||||
|
"""Run a coroutine on a worker thread.
|
||||||
|
|
||||||
|
Playwright's sync API keeps an asyncio loop running on the test
|
||||||
|
thread, so ``asyncio.run`` cannot be called directly from a test
|
||||||
|
body.
|
||||||
|
"""
|
||||||
|
box: dict[str, Any] = {}
|
||||||
|
|
||||||
|
def runner() -> None:
|
||||||
|
try:
|
||||||
|
box["value"] = asyncio.run(coro)
|
||||||
|
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||||
|
box["error"] = e
|
||||||
|
|
||||||
|
t = Thread(target=runner)
|
||||||
|
t.start()
|
||||||
|
t.join()
|
||||||
|
if "error" in box:
|
||||||
|
raise box["error"]
|
||||||
|
return box["value"]
|
||||||
|
|
||||||
|
|
||||||
|
def _delete_source_rows() -> None:
|
||||||
|
"""Delete every row of this suite's distinctive source (chunks
|
||||||
|
cascade with the document rows)."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
for doc in db.scalars(select(Document).where(Document.source == SOURCE)).all():
|
||||||
|
db.delete(doc)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def extension_kb(mock_llm: int, db_ready: None) -> Iterator[ImportSummary]:
|
||||||
|
"""Seed the fixture with the NOVEL scope (``md,sh``) for one test
|
||||||
|
and delete every row it creates afterwards (DB isolation — see the
|
||||||
|
module docstring)."""
|
||||||
|
_delete_source_rows() # idempotent: leftovers from a crashed run
|
||||||
|
summary = _run_in_thread(_import_fixtures(mock_llm, "md,sh"))
|
||||||
|
try:
|
||||||
|
yield summary
|
||||||
|
finally:
|
||||||
|
_delete_source_rows()
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_sources_lists_the_novel_extension(
|
||||||
|
page: Page, app_url: str, extension_kb: ImportSummary
|
||||||
|
) -> None:
|
||||||
|
# The seed saw exactly the two fixture files in their formats — the
|
||||||
|
# novel .sh extension walked, chunked, and summarized.
|
||||||
|
assert extension_kb.formats == {"sh": 1, "md": 1}
|
||||||
|
assert (extension_kb.added, extension_kb.errors) == (2, 0)
|
||||||
|
|
||||||
|
login(page, app_url) # phase 16: the catalog is admin-only
|
||||||
|
# The novel .sh document is listed; the path cell carries the full
|
||||||
|
# path (the column is ellipsized — the title attribute is the pin).
|
||||||
|
row = page.locator("#docs-tbody tr", has_text=SH_REL)
|
||||||
|
expect(row).to_have_count(1)
|
||||||
|
link = row.locator("td:nth-child(2) a.doc-link")
|
||||||
|
expect(link).to_have_count(1)
|
||||||
|
expect(link).to_have_attribute("title", SH_REL)
|
||||||
|
# The markdown control doc is listed too (never asserted by count —
|
||||||
|
# other suites' documents may share the shared E2E database).
|
||||||
|
expect(page.locator("#docs-tbody tr", has_text=MD_REL)).to_have_count(1)
|
||||||
|
|
||||||
|
# Format badge: the row's path link opens the same-page modal and
|
||||||
|
# its meta row shows the .sh format (house assertion style —
|
||||||
|
# test_document_viewer.py asserts the same locator for yaml/md).
|
||||||
|
before = len(page.context.pages)
|
||||||
|
link.click()
|
||||||
|
assert len(page.context.pages) == before, "clicking a row link must not open a new tab"
|
||||||
|
expect(page.locator("#doc-modal-meta .doc-source-badge")).to_have_text(SOURCE)
|
||||||
|
expect(page.locator("#doc-modal-meta .format-badge")).to_have_text("sh")
|
||||||
|
# Non-markdown content renders as escaped monospace text in a pre —
|
||||||
|
# the sentinel proves it is THIS document's content.
|
||||||
|
pre = page.locator("#doc-modal-content pre.doc-raw")
|
||||||
|
expect(pre).to_have_count(1)
|
||||||
|
expect(pre).to_contain_text(SENTINEL)
|
||||||
|
# Still on the Sources page: no navigation happened.
|
||||||
|
assert page.url == app_url + "/sources.html", f"navigated away: {page.url}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_anonymous_sources_gate_and_no_api_docs(
|
||||||
|
page: Page, app_url: str, extension_kb: ImportSummary
|
||||||
|
) -> None:
|
||||||
|
"""A fresh anonymous context (function-scoped ``page`` = new
|
||||||
|
browser context, no cookies): the sign-in gate renders and the page
|
||||||
|
never calls ``/api/docs`` — the phase-16 pin, regression-checked
|
||||||
|
with the novel-extension KB seeded."""
|
||||||
|
api_docs_calls: list[str] = []
|
||||||
|
page.on(
|
||||||
|
"request",
|
||||||
|
lambda r: api_docs_calls.append(r.url) if "/api/docs" in r.url else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
page.goto(f"{app_url}/sources.html")
|
||||||
|
# The gate, with its sign-in link — not a redirect.
|
||||||
|
gate = page.locator("#sources-gate")
|
||||||
|
expect(gate).to_be_visible()
|
||||||
|
expect(gate).to_contain_text("Sign in to view the full catalog")
|
||||||
|
expect(gate.locator("a[href='/login.html?next=/sources.html']")).to_have_count(1)
|
||||||
|
# Stat cards + table hidden…
|
||||||
|
expect(page.locator("#stat-cards")).to_be_hidden()
|
||||||
|
expect(page.locator("#docs-table")).to_be_hidden()
|
||||||
|
expect(page.locator("#sources-empty")).to_be_hidden()
|
||||||
|
# …and NO /api/docs call was ever made.
|
||||||
|
assert api_docs_calls == [], f"anonymous sources page called /api/docs: {api_docs_calls}"
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Extension fixture note
|
||||||
|
|
||||||
|
A small markdown control document for the phase 56 extension-kb fixture.
|
||||||
|
It exists so the `md` scope and the novel `sh` scope are told apart when
|
||||||
|
the importer walks `tests/fixtures/extension_kb/` — with
|
||||||
|
`import_extensions="md"` only this file should land in the index.
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# UPTIME-PROBE-SENTINEL-9c2f — phase 56 fixture marker: this token exists
|
||||||
|
# nowhere else, so the extension_kb rows are unambiguous in the shared DB.
|
||||||
|
#
|
||||||
|
# uptime.sh — homelab service probe: polls the core services and posts a
|
||||||
|
# ntfy alert on the first failure. A novel (.sh) file on purpose — it
|
||||||
|
# only imports when BOR_IMPORT_EXTENSIONS names the sh extension.
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ALERT_TOPIC="homelab-alerts"
|
||||||
|
NTFY_URL="https://ntfy.reeseapps.com"
|
||||||
|
CHECKS=(
|
||||||
|
"k3s|https://10.0.1.10:6443/healthz"
|
||||||
|
"gitlab|https://gitlab.reeseapps.com/-/health_check"
|
||||||
|
"ntfy|https://ntfy.reeseapps.com/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
probe() {
|
||||||
|
local name="$1" url="$2"
|
||||||
|
curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
local line name code
|
||||||
|
for line in "${CHECKS[@]}"; do
|
||||||
|
name="${line%%|*}"
|
||||||
|
code="$(probe "$name" "${line#*|}")"
|
||||||
|
if [[ "$code" != "200" ]]; then
|
||||||
|
echo "uptime: $name answered $code (expected 200)" >&2
|
||||||
|
curl -s -X POST "$NTFY_URL/$ALERT_TOPIC" \
|
||||||
|
-H "Title: homelab check failed" \
|
||||||
|
-d "$name is down (HTTP $code)"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "uptime: round complete"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
"""Integration test: phase 56 — ``BOR_IMPORT_EXTENSIONS`` is user-extensible.
|
||||||
|
|
||||||
|
Proves a NOVEL (non-A9) extension flows through the import machinery
|
||||||
|
(config → walk → delta → chunk → mock ``SUMMARY_MODE`` digest) against
|
||||||
|
the story-dedicated fixture directory ``tests/fixtures/extension_kb/``
|
||||||
|
(the shared ``docs`` / ``summary_kb`` fixtures stay pinned by their own
|
||||||
|
suites). The LLM is the deterministic mock server (``tests/e2e/mock_llm.py``)
|
||||||
|
on a scratch port — the integration analogue of the e2e ``mock_llm``
|
||||||
|
fixture — so the summary is the byte-stable ``SUMMARY_MODE`` digest and
|
||||||
|
the vectors are genuine token-overlap embeddings. Runs against the local
|
||||||
|
compose Postgres (the ``db`` fixture from ``tests/conftest.py``); only
|
||||||
|
the distinctive ``extension_kb`` source rows are created and deleted, so
|
||||||
|
the rest of the shared KB is untouched.
|
||||||
|
|
||||||
|
Runs (DB must be up: ``podman compose up -d db``):
|
||||||
|
|
||||||
|
uv run pytest tests/integration/test_import_extensions_env.py -v
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.config import Settings
|
||||||
|
from app.models import Document
|
||||||
|
from app.rag.importer import import_sources
|
||||||
|
from app.rag.llm import LLMClient
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
FIXTURES = REPO / "tests" / "fixtures" / "extension_kb"
|
||||||
|
SOURCE = FIXTURES.name # "extension_kb" — distinctive, never asserted by count
|
||||||
|
SH_REL = "homelab/scripts/uptime.sh"
|
||||||
|
MD_REL = "homelab/notes/note.md"
|
||||||
|
SENTINEL = "UPTIME-PROBE-SENTINEL-9c2f"
|
||||||
|
#: The mock's SUMMARY_MODE tokenizes the document (``[a-z0-9]+``) — the
|
||||||
|
#: hyphenated sentinel lands in the digest in its tokenized form.
|
||||||
|
SENTINEL_TOKENS = "uptime probe sentinel 9c2f"
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_http(url: str, timeout: float = 30.0) -> None:
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
last_err = "unknown"
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
httpx.get(url, timeout=2.0)
|
||||||
|
return
|
||||||
|
except Exception as e: # noqa: BLE001 — retry until deadline
|
||||||
|
last_err = str(e)
|
||||||
|
time.sleep(0.2)
|
||||||
|
raise RuntimeError(f"mock LLM at {url} did not come up: {last_err}")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def mock_llm_port() -> Iterator[int]:
|
||||||
|
"""The deterministic mock LLM (``tests/e2e/mock_llm.py``) on a free
|
||||||
|
scratch port — same server as the e2e ``mock_llm`` fixture, but
|
||||||
|
private to this file (integration tests otherwise run network-free)."""
|
||||||
|
sock = socket.socket()
|
||||||
|
sock.bind(("127.0.0.1", 0))
|
||||||
|
port = sock.getsockname()[1]
|
||||||
|
sock.close()
|
||||||
|
env = dict(os.environ)
|
||||||
|
env.pop("DEBUGPY", None)
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
[sys.executable, "-m", "uvicorn", "tests.e2e.mock_llm:app",
|
||||||
|
"--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
|
||||||
|
cwd=REPO,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
_wait_http(f"http://127.0.0.1:{port}/v1/models")
|
||||||
|
yield port
|
||||||
|
finally:
|
||||||
|
proc.terminate()
|
||||||
|
try:
|
||||||
|
proc.wait(timeout=10)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
proc.kill()
|
||||||
|
|
||||||
|
|
||||||
|
def _settings(mock_port: int, extensions: str) -> Settings:
|
||||||
|
kwargs: dict[str, Any] = {
|
||||||
|
"_env_file": None,
|
||||||
|
"llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
|
||||||
|
"import_extensions": extensions,
|
||||||
|
}
|
||||||
|
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_source(db: Session, source: str) -> None:
|
||||||
|
for doc in db.scalars(select(Document).where(Document.source == source)).all():
|
||||||
|
db.delete(doc)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def test_novel_extension_imports_end_to_end(mock_llm_port: int, db: Session) -> None:
|
||||||
|
"""``md,sh`` (a novel extension) imports the ``.sh`` file end to
|
||||||
|
end: row + plain-text chunks + the mock ``SUMMARY_MODE`` digest,
|
||||||
|
with the markdown control doc imported as well."""
|
||||||
|
settings = _settings(mock_llm_port, "md,sh")
|
||||||
|
assert settings.import_extension_set == {".md", ".sh"}
|
||||||
|
summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
|
||||||
|
try:
|
||||||
|
assert (
|
||||||
|
summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
|
||||||
|
) == (2, 2, 0, 0, 0)
|
||||||
|
assert summary.formats == {"sh": 1, "md": 1}
|
||||||
|
# The .sh file is non-markdown → exactly one lite summary (phase 30).
|
||||||
|
assert (summary.summaries, summary.summary_errors) == (1, 0)
|
||||||
|
|
||||||
|
sh = db.scalar(
|
||||||
|
select(Document).where(
|
||||||
|
Document.source == SOURCE, Document.path == SH_REL
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert sh is not None, "the novel .sh extension was not imported"
|
||||||
|
# Non-markdown: the title comes from the file stem (a ``#`` line
|
||||||
|
# is a comment, not a heading).
|
||||||
|
assert sh.title == "uptime"
|
||||||
|
# Plain-text chunking: the content (incl. the sentinel) is
|
||||||
|
# chunked and every content chunk is embedded at the 768-dim
|
||||||
|
# contract.
|
||||||
|
content = [c for c in sh.chunks if not c.is_summary]
|
||||||
|
assert content, "the .sh file has no content chunks"
|
||||||
|
assert all(
|
||||||
|
c.embedding is not None and len(c.embedding) == 768 for c in content
|
||||||
|
)
|
||||||
|
assert any(SENTINEL in c.content for c in content)
|
||||||
|
# Mock SUMMARY_MODE digest: byte-stable, the tokenized sentinel
|
||||||
|
# inside it, plus the code-appended pointer line.
|
||||||
|
assert sh.summary is not None
|
||||||
|
assert sh.summary.startswith("This document covers")
|
||||||
|
assert SENTINEL_TOKENS in sh.summary
|
||||||
|
assert f"Source: {SOURCE}/{SH_REL}" in sh.summary
|
||||||
|
schunks = [c for c in sh.chunks if c.is_summary]
|
||||||
|
assert len(schunks) == 1 and schunks[0].position == -1
|
||||||
|
assert schunks[0].embedding is not None
|
||||||
|
|
||||||
|
# The markdown control doc imported too — but markdown never
|
||||||
|
# gets a summary (phase 30).
|
||||||
|
note = db.scalar(
|
||||||
|
select(Document).where(
|
||||||
|
Document.source == SOURCE, Document.path == MD_REL
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert note is not None
|
||||||
|
assert note.summary is None
|
||||||
|
assert [c for c in note.chunks if not c.is_summary]
|
||||||
|
finally:
|
||||||
|
_cleanup_source(db, SOURCE)
|
||||||
|
|
||||||
|
|
||||||
|
def test_narrowing_to_md_still_excludes_the_novel_extension(
|
||||||
|
mock_llm_port: int, db: Session
|
||||||
|
) -> None:
|
||||||
|
"""``md`` (the A9-era narrowing, preserved as a special case): the
|
||||||
|
``.sh`` file is out of scope, only the control note imports."""
|
||||||
|
settings = _settings(mock_llm_port, "md")
|
||||||
|
assert settings.import_extension_set == {".md"}
|
||||||
|
summary = asyncio.run(import_sources([FIXTURES], LLMClient(settings), session=db))
|
||||||
|
try:
|
||||||
|
assert (
|
||||||
|
summary.files, summary.added, summary.unchanged, summary.updated, summary.errors
|
||||||
|
) == (1, 1, 0, 0, 0)
|
||||||
|
assert summary.formats == {"md": 1}
|
||||||
|
assert summary.summaries == 0
|
||||||
|
assert db.scalar(
|
||||||
|
select(Document).where(
|
||||||
|
Document.source == SOURCE, Document.path == SH_REL
|
||||||
|
)
|
||||||
|
) is None
|
||||||
|
assert db.scalar(
|
||||||
|
select(Document).where(
|
||||||
|
Document.source == SOURCE, Document.path == MD_REL
|
||||||
|
)
|
||||||
|
) is not None
|
||||||
|
finally:
|
||||||
|
_cleanup_source(db, SOURCE)
|
||||||
+42
-25
@@ -8,7 +8,7 @@ import pytest
|
|||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from pydantic_settings import SettingsError
|
from pydantic_settings import SettingsError
|
||||||
|
|
||||||
from app.config import _ALLOWED_IMPORT_EXTENSIONS, Settings # pyright: ignore[reportPrivateUsage]
|
from app.config import _DEFAULT_IMPORT_EXTENSIONS, Settings # pyright: ignore[reportPrivateUsage]
|
||||||
|
|
||||||
|
|
||||||
def _settings(**kwargs: Any) -> Settings:
|
def _settings(**kwargs: Any) -> Settings:
|
||||||
@@ -57,15 +57,15 @@ NEW_A9_FORMATS = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_allowed_import_extensions_contains_all_seventeen_formats() -> None:
|
def test_default_import_extensions_is_the_full_a9_family() -> None:
|
||||||
"""The validator's base set is the full A9 set: the original seven
|
"""Phase 56: the built-in default is the full A9 set — the original
|
||||||
plus the ten added 2026-08-27 (quadlet family + ``j2``). The
|
seven plus the ten added 2026-08-27 (quadlet family + ``j2``). It is
|
||||||
never-widen contract bounds :py:data:`import_extensions` against
|
the default and the ``.env.example`` example, NOT a ceiling: the
|
||||||
exactly this set."""
|
validator accepts any well-formed extension beyond it."""
|
||||||
assert {
|
assert {
|
||||||
"md", "markdown", "txt", "yaml", "yml", "json", "py",
|
"md", "markdown", "txt", "yaml", "yml", "json", "py",
|
||||||
*NEW_A9_FORMATS,
|
*NEW_A9_FORMATS,
|
||||||
} == _ALLOWED_IMPORT_EXTENSIONS
|
} == _DEFAULT_IMPORT_EXTENSIONS
|
||||||
|
|
||||||
|
|
||||||
def test_default_import_extensions_include_the_ten_new_formats() -> None:
|
def test_default_import_extensions_include_the_ten_new_formats() -> None:
|
||||||
@@ -151,34 +151,51 @@ def test_import_extensions_env_override_is_a_csv_list(monkeypatch) -> None:
|
|||||||
assert s.import_extension_set == {".md", ".yml"}
|
assert s.import_extension_set == {".md", ".yml"}
|
||||||
|
|
||||||
|
|
||||||
def test_import_extensions_rejects_unknown_format(monkeypatch) -> None:
|
def test_import_extensions_accepts_novel_extension(monkeypatch) -> None:
|
||||||
"""A typo in the CSV fails at startup (loudly), not by silently
|
"""Phase 56 (owner permission 2026-08-31): the A9 family is the
|
||||||
walking zero files."""
|
default, not the ceiling — a novel well-formed extension (``sh``) is
|
||||||
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,docx")
|
accepted and simply becomes importable."""
|
||||||
with pytest.raises(ValidationError, match="docx"):
|
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,sh")
|
||||||
_settings()
|
s = _settings()
|
||||||
|
assert s.import_extension_set == {".md", ".sh"}
|
||||||
|
|
||||||
|
|
||||||
def test_import_extensions_rejects_empty(monkeypatch) -> None:
|
def test_import_extensions_normalizes_case_and_leading_dot(monkeypatch) -> None:
|
||||||
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", " ")
|
"""Case and a leading dot are both tolerated (unchanged tolerance)."""
|
||||||
with pytest.raises(ValidationError):
|
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "MD,.Py")
|
||||||
_settings()
|
s = _settings()
|
||||||
|
assert s.import_extension_set == {".md", ".py"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_import_extensions_rejects_empty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""A blank list would silently import nothing — fail loudly at
|
||||||
|
startup, naming the field (empty, whitespace-only, and comma-only
|
||||||
|
all parse to zero formats)."""
|
||||||
|
for value in ("", " ", ",,"):
|
||||||
|
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", value)
|
||||||
|
with pytest.raises(ValidationError, match="import_extensions"):
|
||||||
|
_settings()
|
||||||
|
|
||||||
|
|
||||||
def test_import_extensions_validator_accepts_new_a9_formats(monkeypatch) -> None:
|
def test_import_extensions_validator_accepts_new_a9_formats(monkeypatch) -> None:
|
||||||
"""A9 revised 2026-08-27: the new names are first-class — the
|
"""A9 revised 2026-08-27: quadlet/jinja names are first-class default
|
||||||
never-widen contract now holds against the widened base set, so a
|
formats — a CSV using them (a narrowing of the default family) is
|
||||||
narrowing CSV with quadlet/jinja names is accepted."""
|
accepted."""
|
||||||
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,container,j2")
|
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,container,j2")
|
||||||
s = _settings()
|
s = _settings()
|
||||||
assert s.import_extension_set == {".md", ".container", ".j2"}
|
assert s.import_extension_set == {".md", ".container", ".j2"}
|
||||||
|
|
||||||
|
|
||||||
def test_import_extensions_validator_still_rejects_unknown(monkeypatch) -> None:
|
def test_import_extensions_rejects_malformed_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
"""Truly unknown extensions still fail loudly at startup (the
|
"""The shape guard (``^[a-z0-9]{1,16}$``) is the typo guard — it
|
||||||
validator is intact — only the allowed base set widened)."""
|
keeps punctuation and path-ish values out of the set, naming the
|
||||||
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,xyz")
|
offending token(s), while any extension a file could actually be
|
||||||
with pytest.raises(ValidationError, match="xyz"):
|
suffixed with still goes through."""
|
||||||
|
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,sh!")
|
||||||
|
with pytest.raises(ValidationError, match="sh!"):
|
||||||
|
_settings()
|
||||||
|
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,../x")
|
||||||
|
with pytest.raises(ValidationError, match=r"/x"):
|
||||||
_settings()
|
_settings()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user