ducoterra
99c48cbe06
Phase 02 (story: import documents):
- fence-aware markdown chunker (heading sections, 200-char overlap,
heading anchor on every chunk, 1200-char hard cap, fence blocks
kept atomic and split under the cap)
- LLMClient over aipi (LiteLLM) reusing the openai client's httpx
transport to send a clean {model, input} payload — the openai SDK
injects encoding_format, which aipi's openai_like group rejects;
token-budget batching + halving retry for the endpoint's
~1024-token per-request input cap
- two-phase per-file upsert importer: sha256 delta (unchanged skip),
atomic commit, A9 exclusion walk, per-source prune, per-file error
tolerance (rollback + log + continue, non-zero CLI exit), adaptive
re-chunk at half target for URL-dense files the endpoint rejects
- scripts/import_docs CLI (repeatable --source, --prune, --limit,
defaults ~/Homelab + ~/Deployments)
- GET /api/docs with per-doc chunk counts; Sources page wired to the
real endpoint (stat cards, full-width a11y table, designed empty
state, DOM-built rows — no innerHTML)
- tests: 63 passed (chunker/llm/importer units, docs API + importer
integration), story E2E 3/3 (real endpoints, in-thread import);
app/ coverage 98%
- real KB imported: 672 docs / 8969 chunks in ~3m, idempotent
re-run (672 unchanged, 0 batches)
- harness: .agent/validate.sh now gates through uv (pytest +
coverage >90% + ruff + pyright) instead of system python3
66 lines
2.4 KiB
Python
66 lines
2.4 KiB
Python
"""Integration test: importer end-to-end against ``tests/fixtures/docs/``.
|
|
|
|
Runs the real import pipeline (walk → chunk → embed → upsert) into the
|
|
local compose Postgres, then checks the DB state *and* the API shape a
|
|
browser would consume. Embeddings come from a deterministic in-process
|
|
fake, so no network is needed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import func, select, text
|
|
|
|
from app.models import Chunk, Document
|
|
from app.rag.importer import import_sources
|
|
from tests.fakes import FakeEmbedder
|
|
|
|
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs"
|
|
|
|
EXPECTED_DOCS = {
|
|
("docs", "homelab/kubernetes.md"),
|
|
("docs", "homelab/backups.md"),
|
|
("docs", "deployments/new-service.md"),
|
|
}
|
|
|
|
|
|
def test_import_fixtures_end_to_end(client, db) -> None:
|
|
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
|
db.commit()
|
|
llm = FakeEmbedder()
|
|
|
|
summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
|
|
assert (summary.files, summary.added, summary.unchanged) == (3, 3, 0)
|
|
assert summary.chunks >= 3
|
|
|
|
docs = db.scalars(select(Document)).all()
|
|
assert {(d.source, d.path) for d in docs} == EXPECTED_DOCS
|
|
titles = {d.path: d.title for d in docs}
|
|
assert titles["homelab/kubernetes.md"] == "Kubernetes Homelab Cluster"
|
|
assert titles["deployments/new-service.md"] == "Deploying a New Service"
|
|
# Full content is stored — that is what the RAG context will be.
|
|
k8s = next(d for d in docs if d.path == "homelab/kubernetes.md")
|
|
assert "Talos Linux" in k8s.content and k8s.content_hash
|
|
|
|
n_chunks = db.scalar(select(func.count()).select_from(Chunk))
|
|
assert n_chunks == summary.chunks
|
|
for c in db.scalars(select(Chunk)).all():
|
|
assert c.embedding is not None and len(c.embedding) == 768
|
|
|
|
# The Sources page consumes exactly this shape.
|
|
r = client.get("/api/docs")
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert len(body["documents"]) == 3
|
|
assert all(d["chunks"] >= 1 for d in body["documents"])
|
|
|
|
# Idempotent re-run: nothing re-embedded.
|
|
calls_before = len(llm.calls)
|
|
s2 = asyncio.run(import_sources([FIXTURES], llm, session=db))
|
|
assert s2.unchanged == 3 and s2.added == 0
|
|
assert len(llm.calls) == calls_before # unchanged → no embedding requests
|
|
|
|
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
|
db.commit()
|