From 7e8d14702e2ac5b09314a87edea1a5a17e667065 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sat, 22 Aug 2026 01:27:02 -0400 Subject: [PATCH] =?UTF-8?q?feat(rag):=20hybrid=20FTS+vector=20retrieval=20?= =?UTF-8?q?and=20multi-format=20ingestion=20=E2=80=94=20name-your-tool=20q?= =?UTF-8?q?uestions=20find=20the=20right=20document?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../complete/09_story_retrieval_quality.md | 164 +++++++++++++ .../phases/todo/08_story_dark_tech_theme.md | 136 ----------- .env.example | 11 +- README.md | 109 ++++++--- alembic/versions/0002_hybrid_retrieval.py | 40 +++ app/api/chat.py | 65 +++-- app/config.py | 64 ++++- app/models.py | 5 +- app/rag/chunker.py | 189 +++++++++++++-- app/rag/importer.py | 70 ++++-- app/rag/retriever.py | 217 +++++++++++++++-- scripts/eval_retrieval.py | 129 ++++++++++ scripts/import_docs.py | 20 +- tests/conftest.py | 12 +- tests/e2e/conftest.py | 6 + tests/e2e/test_chat_rag.py | 2 +- tests/e2e/test_dark_tech_theme.py | 2 +- tests/e2e/test_honest_deflection.py | 2 +- tests/e2e/test_import_documents.py | 14 +- tests/e2e/test_loading_feedback.py | 2 +- tests/e2e/test_retrieval_quality.py | 197 +++++++++++++++ tests/e2e/test_suggestion_chips.py | 2 +- tests/fixtures/docs/.hidden/junk.md | 5 + .../container_gitlab/gitlab-compose.yaml | 24 ++ .../docs/homelab/container_gitlab/gitlab.md | 27 +++ .../docs/homelab/networking/static-dns.json | 14 ++ .../docs/homelab/scripts/uptime_probe.py | 60 +++++ .../fixtures/docs/homelab/ssh/ssh_aliases.txt | 14 ++ tests/integration/test_chat_api.py | 47 +++- tests/integration/test_importer_e2e.py | 26 +- tests/integration/test_migration_0002.py | 72 ++++++ tests/unit/test_chat_gate.py | 79 +++++- tests/unit/test_chunker.py | 228 +++++++++++++++++- tests/unit/test_config.py | 39 ++- tests/unit/test_importer.py | 114 ++++++++- tests/unit/test_retriever.py | 101 ++++++++ 36 files changed, 2018 insertions(+), 290 deletions(-) create mode 100644 .agent/phases/complete/09_story_retrieval_quality.md delete mode 100644 .agent/phases/todo/08_story_dark_tech_theme.md create mode 100644 alembic/versions/0002_hybrid_retrieval.py create mode 100644 scripts/eval_retrieval.py create mode 100644 tests/e2e/test_retrieval_quality.py create mode 100644 tests/fixtures/docs/.hidden/junk.md create mode 100644 tests/fixtures/docs/homelab/container_gitlab/gitlab-compose.yaml create mode 100644 tests/fixtures/docs/homelab/container_gitlab/gitlab.md create mode 100644 tests/fixtures/docs/homelab/networking/static-dns.json create mode 100644 tests/fixtures/docs/homelab/scripts/uptime_probe.py create mode 100644 tests/fixtures/docs/homelab/ssh/ssh_aliases.txt create mode 100644 tests/integration/test_migration_0002.py diff --git a/.agent/phases/complete/09_story_retrieval_quality.md b/.agent/phases/complete/09_story_retrieval_quality.md new file mode 100644 index 0000000..1631ada --- /dev/null +++ b/.agent/phases/complete/09_story_retrieval_quality.md @@ -0,0 +1,164 @@ +# Phase 09 — Story: Retrieval Quality — Multi-Format Ingestion + Hybrid Search + +**Story:** `.agent/user_stories/retrieval-quality.md` +**Context:** `.agent/PLAN.md` §3 (data flow), §5 (data model), §6 (retrieval), §11 (import) + +## Goal +Fix "RAG retrieval is terrible": ingest the full text-format set (not +just `.md`), purge vendored-cache junk from the index, and replace +pure-cosine top-4 with hybrid (vector + Postgres FTS, RRF-fused) +retrieval so name-your-tool questions find the right document. + +## Owner permission (recorded per phase protocol) +> "I'm giving you explicit permission to update the locked decisions and +> proceed with writing all 3 of these phases" — Reese, 2026-08-21. + +This phase revises anchors **A9** (content scope: `*.md` only → +`md, markdown, txt, yaml, yml, json, py` + hidden-dir skip), **A7** +(pure-cosine top-4 → hybrid RRF retrieval; the whole-document context +contract is preserved), **A8** (gate: LOW only when best cosine < +threshold **and** zero FTS hits; threshold re-tuned 0.30 → 0.62 default). +`PLAN.md` anchors were updated 2026-08-21 under this permission. + +## Evidence (measured 2026-08-21 against the live KB + `embed` model) +- "How did I install gitlab?": the best `gitlab.md` chunk ranks **7th** + (cosine 0.804) — outside the top-4 window. Ranks 1–6: a vendored-cache + README (`.esphome/.espressif/…/esp-tflite-micro/README.md`, 0.838) and + generic templates (`project_readme_template.md`, `templates/…/foobar. + md`, 0.81–0.82). The LLM therefore answered from junk docs and honestly + reported "no notes on gitlab". +- Corpus cosine range: **0.41–0.84** — the old 0.30 gate never + discriminated. +- FTS: `plainto_tsquery('english','gitlab')` matches **exactly** + gitlab.md's 4 chunks and nothing else. +- ~470 of 672 indexed docs live under dot-prefixed path components + (vendored caches) that A9's exclusion list doesn't cover. + +## Dependencies +Phases 01–07 (02 importer, 03 retriever, 04 gate especially). +Independent of 08 (backend + E2E only). Phase 10 builds on the new +multi-format corpus. + +## Implementation steps +1. **Config** (`app/config.py`): `import_extensions` (csv, default + `md,markdown,txt,yaml,yml,json,py`; env `BOR_IMPORT_EXTENSIONS`), + `hybrid_vector_candidates` (30, `BOR_HYBRID_VECTOR_CANDIDATES`), + `hybrid_lexical_candidates` (30, `BOR_HYBRID_LEXICAL_CANDIDATES`), + `rrf_k` (60, `BOR_RRF_K`), `relevance_threshold` default **0.62** + (re-tuned; env override stays). In `tests/e2e/conftest.py`'s + `app_server` fixture set `BOR_RELEVANCE_THRESHOLD=0.30` — the mock's + token-overlap embeddings need their own calibration; this keeps + stories 02–07's E2E suites green. +2. **Chunker** (`app/rag/chunker.py`): add a `chunk_document(content, + path)` dispatcher by lowercased suffix + per-format functions — + **stdlib only, no new dependencies**: + - `yaml`/`yml`: split on `---` document separators and top-level keys + (indent-0 `key:` lines); every chunk keeps its key line as anchor. + - `json`: `json.dumps(obj, indent=2)` then split at top-level keys + (track brace depth); unparseable JSON → paragraph packing. + - `py`: stdlib `ast` top-level node line ranges → split at + defs/classes; oversized functions fall back to line packing. + - `txt`: paragraph packing (reuse `_paragraph_blocks`/`_pack_blocks`). + - All formats honor `HARD_MAX_CHARS` (1200 — the aipi ~1024-token + request cap) and the target/overlap settings; the `md` path stays + byte-for-byte unchanged (existing chunker tests must stay green). +3. **Importer** (`app/rag/importer.py`, `scripts/import_docs.py`): + extension filter (case-insensitive, config-driven); **skip any path + containing a dot-prefixed component** (hidden dirs); chunker dispatch + by suffix; `--prune` now also drops docs whose files **no longer + match the filter** (this is how the ~470 junk docs leave the index); + summary log gains per-format counts + (`formats=md:203,yaml:267,…`). +4. **Migration `0002_hybrid_retrieval.py`** (alembic): + - `chunks.tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', + content)) STORED` + `CREATE INDEX … USING gin (chunks.tsv)`. + - `query_log.fts_hits INT` (nullable; pre-existing rows stay NULL). +5. **Retriever** (`app/rag/retriever.py`) — hybrid path: + - `retrieve(db, question, question_embedding)`: vector top-N (cosine, + as today) ∪ lexical top-N — `to_tsquery('english', )` (skip pure-stopword/no-token + questions → empty lexical list), ordered by `ts_rank` — fused with + RRF: `score = Σ 1/(k + rank)` over the lists a chunk appears in + (single-list chunks get one term; k from config). + - `RetrievedChunk` gains `cosine` (for the gate) and `fts_hit: bool` + alongside `score` (now the fused score, used for ranking); + `select_documents` / `weak_hit_titles` keep working off `score`. + - Deterministic tie-break: `(−fused, −cosine, document.path, + chunk.position)`. +6. **Chat flow + gate** (`app/api/chat.py`): pass the raw question into + `retrieve`; **LOW only when `best_cosine < threshold and fts_hits == + 0`** (`fts_hits` = count of lexical candidates matched); per-turn log + line gains `fts_hits=…` (PLAN §9); `query_log` row stores `fts_hits`. +7. **Eval script** `scripts/eval_retrieval.py`: + `uv run python -m scripts.eval_retrieval "q1" "q2" …` (or + `--from-file questions.txt`) — embeds via aipi, runs the hybrid + search, prints top-5 docs per question with cosine/fts/fused scores + + the gate verdict. Requires `AIPI_KEY` in the environment (same + convention as `llm_probe.py`). +8. **Re-import the live KB** (one-time; expect ~15–40 min of embedding + batches — the importer logs per file): + `uv run python -m scripts.import_docs --prune`. Expect ~470 + hidden-dir docs pruned and ~500 docs indexed (md + new formats). Then + verify with the eval script: + - "How did I install gitlab?" → top doc `active/container_gitlab/ + gitlab.md` (the compose yaml should land in the top-2). + - "How is my Kubernetes cluster set up?" → kubernetes docs. + - "sourdough starter" → LOW (deflect). + If the gitlab case isn't #1, iterate the **fusion** (k, candidate + counts, token handling) — not the threshold — until it is, and record + the final numbers in the phase report. +9. **E2E fixtures** (`tests/fixtures/docs/`): add + `homelab/container_gitlab/gitlab.md` (H1 "Gitlab", docker install + steps, "gitlab" repeated), `homelab/container_gitlab/ + gitlab-compose.yaml` (`services: gitlab: …`), a `.py` note, a `.json` + note, a `.txt` note, and `.hidden/junk.md` (must never be imported). + Follow the existing in-process seeding pattern from + `tests/e2e/test_import_documents.py`. +10. **README**: import workflow section — supported formats, hidden-dir + skip, `scripts/eval_retrieval.py`, threshold tuning; note that the + Sources count drops after the prune (intended cleanup). + +## Testing & Quality +- **Unit:** chunker per format (yaml top-level + `---` split, json + top-level keys + pretty-print + unparseable fallback, py ast split + + oversized-func fallback, txt paragraphs, dispatch, 1200-cap) with md + output unchanged; importer (hidden-dir skip, extension filter, + prune-when-filtered-out, per-format summary); retriever (RRF math: + both-lists / one-list / tie-break; OR tsquery construction incl. + no-token and stopword-only questions; gate: `cosine ≥ T` → HIGH; + `cosine < T` + `fts>0` → HIGH; `cosine < T` + `fts=0` → LOW; boundary + exactly `T` → HIGH). +- **Integration:** `/api/chat` hybrid against a seeded temp schema — + keyword question grounded + `fts_hits` in `query_log`; off-topic + deflected with `fts_hits=0`; migration up clean. +- **Coverage:** `uv run pytest --cov=app --cov-report=term-missing` — + **>90%** on `app/`. +- **No regressions:** existing story E2E suites (02–07) green in + isolation after the change (the conftest threshold override is what + keeps them green — verify each one). + +## Playwright Execution Phase +Run ONLY this story's suite: + +```bash +uv run pytest tests/e2e/test_retrieval_quality.py -v --no-cov +``` + +Implements the story mapping: multi-format fixture import (hidden doc +excluded, `/api/docs` counts); "How did I install gitlab?" → grounded, +not deflected, gitlab chip, `query_log` row; keyword-only question beats +vector ranking (FTS-OR gate end to end); "sourdough" → deflected bubble + +≥2 chips. + +## Success criteria +- [ ] live eval: "How did I install gitlab?" → `gitlab.md` is the top doc +- [ ] zero dot-prefixed path components in `documents` after re-import +- [ ] off-topic still deflects; on-topic still grounds (new + existing E2E) +- [ ] unit + integration green, coverage >90%, ruff + pyright green +- [ ] README documents formats / hidden-dir skip / eval / tuning +- [ ] committed + +## Commit +```bash +git add -A && git commit --no-gpg-sign -m "feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document" +``` diff --git a/.agent/phases/todo/08_story_dark_tech_theme.md b/.agent/phases/todo/08_story_dark_tech_theme.md deleted file mode 100644 index 6960ca9..0000000 --- a/.agent/phases/todo/08_story_dark_tech_theme.md +++ /dev/null @@ -1,136 +0,0 @@ -# Phase 08 — Story: Dark Tech Theme - -**Story:** `.agent/user_stories/dark-tech-theme.md` -**Context:** `.agent/PLAN.md` §7 (UI/UX strategy), §10 (testing) - -## Goal -Re-skin the whole UI to a dark, techy, emoji-free look with a subtly -animated pure-CSS background — zero behavior or layout changes, WCAG 2.1 -AA re-proven on the new palette. - -## Dependencies -Phases 01–07 (all complete). Independent of 09/10; phase 10's viewer page -inherits this theme, so 08 must land first. - -## Locked decisions -No anchors changed. Replaces the §7.2 light contrast pairs with the dark -palette below (PLAN §7.2 already updated 2026-08-21 with owner -permission). No new technology — pure CSS/HTML/inline SVG (A11). - -## Implementation steps -1. **Palette swap** — `frontend/assets/styles.css` `:root` tokens (all - pairs computed, ≥4.5:1): - - | token | dark value | computed pair | - |---|---|---| - | `--bg` | `#0a0e17` | ink on bg 16.2:1 | - | `--surface` | `#121a2e` | ink on surface 14.5:1 | - | `--ink` | `#e8ebf4` | — | - | `--ink-soft` | `#9aa4bd` | ink-soft on surface 6.9:1 | - | `--line` | `#26304a` | decorative | - | `--brand` | `#6d78f2` | **dark ink `--bg` on brand 5.2:1** | - | `--brand-soft` | `#232b52` | brand-ink on brand-soft 6.9:1 | - | `--brand-ink` | `#a5b4fc` | brand-ink on surface 8.7:1 | - | `--accent-bg` | `#2b2110` | accent-ink on accent-bg 9.5:1 | - | `--accent-ink` | `#fbbf24` | — | - | `--accent-line` | `#f59e0b` | unchanged (8.9:1 on bg) | - | `--err-bg` / `--err-ink` | `#2d1318` / `#fca5a5` | 9.1:1 | - | `--err-line` | `#ef4444` | 4.6:1 on err-bg (UI boundary) | - | `--ok-bg` / `--ok-ink` | `#10241b` / `#6ee7a8` | 10.6:1 | - - Button text is `--bg` (dark) on `--brand` — **never white on brand** - (3.7:1, fails). Busy button: keep the `#a5b4fc` background (the - `tests/unit/test_frontend_feedback.py` assertion greps this token) - with a **dark** arc (`--bg`, 9.7:1). Update derived light-mode values: - shadows (black-based, lower alpha), selection, typing dots, chip - hover. -2. **Emoji purge** — replace every emoji in chrome with inline SVG - (`aria-hidden` kept, ~16–20px, `currentColor` where sensible): - - `frontend/assets/app.js` (~L116, ~L132): avatars 🧠/🧑 → SVG - circuit-node glyph (brain) / minimal silhouette (user) as JS string - constants. - - `frontend/index.html`: favicon 🧠 data-URI → SVG tech mark (hex + - node, brand color on dark, <1 KB), still a `data:` URI; - `.brand-mark` 🧠 → same mark; ⚠️ banner icon → SVG triangle; 👋 - empty state → SVG glyph. - - `frontend/sources.html`: favicon, `.brand-mark`, 📂 empty state → - SVG marks. -3. **Tech details** — mono wordmark with letter-spacing; stat values - mono; radii `10px`/`6px`; 1px `--line` borders on cards/bubbles/table; - 2px gradient hairline (brand→cyan, low alpha) under the sticky header. -4. **Animated background (pure CSS, zero JS)** — working recipe: - `html { background: var(--bg) }`, `body { background: transparent; - position: relative }` (body must not create a stacking context): - - `body::before` — fine grid: two `linear-gradient`s (1px lines, - `--line` at ~35% alpha), `background-size: 44px 44px`, masked with a - radial fade (visible center-top, fading to the edges), animated - `background-position` `0 0 → 44px 44px`, 60s linear infinite - (seamless loop — the delta equals one cell). - - `body::after` — two large soft radial glows: indigo - `rgba(109,120,242,0.14)` top-left, cyan `rgba(34,211,238,0.10)` - bottom-right; 14s ease-in-out infinite alternate breathing - (opacity/scale). No `filter: blur` (perf). - - Both: `position: fixed; inset: 0; pointer-events: none; z-index: - -1`. Keep glow alpha low — subtle, never competing with text. -5. **Reduced motion** — `@media (prefers-reduced-motion: reduce)`: - `body::before, body::after { animation: none }` (static grid + glows - remain). Existing typing/spinner reduced-motion handling stays. -6. **Test updates (behavior unchanged):** - - `tests/e2e/test_honest_deflection.py` (~L104): deflection bubble - `backgroundColor` assertion `rgb(255, 247, 232)` → `rgb(43, 33, - 16)`; the border assertion `rgb(245, 158, 11)` is unchanged. - - New integration test - `tests/integration/test_api.py::test_ui_chrome_has_no_emoji`: GET - `/`, `/sources.html`, `/assets/app.js`, `/assets/styles.css` — assert - no characters in the emoji code-point set (U+1F300–U+1FAFF, - U+2600–U+27BF, U+2B00–U+2BFF, U+FE0F, U+200D, plus the specific - glyphs previously used: 🧠 🧑 👋 📂 ⚠️). -7. **PLAN.md §7.2** — dark contrast table already applied (2026-08-21, - owner permission); no further plan edits in this phase. - -## UI Verification -Manual screenshot pass (1280px + 375px, both pages): grid is faint -(barely-there), glows soft, no banding; brand button legible (dark -text); deflection bubble distinct from normal answers; avatars crisp at -16px; reduced-motion preview (DevTools emulation) shows the static -background. - -## Testing & Quality -- Integration: no-CDN check still green on both pages (it covers - `/sources.html` since Phase 07); new emoji-guard test above. -- **Existing E2E regression check:** after the reskin, run the existing - story suites in isolation and confirm they stay green — at minimum - `test_chat_rag.py`, `test_honest_deflection.py`, `test_responsive_ - polish.py` (the contrast helper computes from live styles and must pass - on the new palette). -- Coverage: `uv run pytest --cov=app --cov-report=term-missing` — **>90%** - on `app/`. -- `uv run ruff check . && uv run pyright` green. - -## Playwright Execution Phase -Run ONLY this story's suite: - -```bash -uv run pytest tests/e2e/test_dark_tech_theme.py -v --no-cov -``` - -Implements the story mapping: dark bg + computed contrast pairs ≥4.5:1; -no emoji in innerText/outerHTML on both pages; `body::before`/`::after` -animate; reduced-motion context → `animation-name: none`; on-topic smoke -(stream + chip + button recovery) unchanged; all assets local. - -## Success criteria -- [ ] both pages dark; every text pair ≥4.5:1 (computed in E2E) -- [ ] zero emoji in chrome (E2E + new integration guard) -- [ ] animated background subtle, pure CSS, reduced-motion honored -- [ ] layout metrics + chat behavior unchanged (smoke E2E) -- [ ] existing story E2E suites still green in isolation -- [ ] unit + integration green, coverage >90%, ruff + pyright green -- [ ] committed (force-add `.agent/PLAN.md` + this phase record — rule 8) - -## Commit -```bash -git add -A -git add -f .agent/PLAN.md .agent/phases/todo/08_story_dark_tech_theme.md -git commit --no-gpg-sign -m "feat(ui): dark tech theme — emoji-free chrome, subtle animated CSS background, WCAG AA dark palette" -``` diff --git a/.env.example b/.env.example index ff855c0..f0d2370 100644 --- a/.env.example +++ b/.env.example @@ -18,13 +18,20 @@ BOR_LLM_EMBED_MODEL=embed BOR_EMBEDDING_DIM=768 # verified 2026-08 via scripts/llm_probe.py # --- RAG tuning --- -BOR_TOP_K_CHUNKS=4 BOR_TOP_N_DOCS=2 -BOR_RELEVANCE_THRESHOLD=0.30 # max cosine similarity required to answer (else honest deflection) +BOR_RELEVANCE_THRESHOLD=0.62 # answer when best cosine >= this OR an FTS hit; else honest deflection BOR_MAX_CONTEXT_CHARS=24000 # cap on total document text sent to the LLM BOR_CHUNK_TARGET_CHARS=2000 BOR_CHUNK_OVERLAP_CHARS=200 BOR_EMBED_BATCH_SIZE=16 + +# --- Hybrid retrieval (vector + Postgres FTS, RRF-fused) --- +BOR_HYBRID_VECTOR_CANDIDATES=100 # cosine list width for the fusion +BOR_HYBRID_LEXICAL_CANDIDATES=30 # FTS list width for the fusion +BOR_RRF_K=60 # Reciprocal Rank Fusion damping constant + +# --- Import scope (A9 formats; may only narrow, never widen) --- +# BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py # BOR_SUGGESTIONS=["How is my Kubernetes cluster set up?"] # JSON list of onboarding chips # --- Debugging (0/1 — 1 enables attach-on-demand debugpy on port 5678) --- diff --git a/README.md b/README.md index ef75ddf..dff92b8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # 🧠 Brain of Reese A chippy, honest **RAG chatbot** over the `~/Homelab` and `~/Deployments` -projects. Point it at your markdown docs, ask it anything — it retrieves -the relevant notes with **Postgres 17 + pgvector** cosine search, feeds the -**whole relevant document** to a **self-hosted LLM** (`turbo` via -`https://aipi.reeseapps.com/v1`), and streams a grounded answer back. +projects. Point it at your notes — markdown, YAML, JSON, Python, plain +text — ask it anything, and it retrieves the relevant chunks with +**hybrid search** (pgvector cosine ∪ Postgres full-text search, fused with +Reciprocal Rank Fusion), feeds the **whole relevant document** to a +**self-hosted LLM** (`turbo` via `https://aipi.reeseapps.com/v1`), and +streams a grounded answer back. If it doesn't have notes for your question, it admits it: *"I haven't done anything like that"* — plus suggestions for what it **does** know. @@ -77,9 +79,9 @@ uv run uvicorn app.main:app --reload file), so a refresh after a normal editing session takes seconds: ```bash -# After editing/adding/removing markdown in your projects: +# After editing/adding/removing notes in your projects: uv run python -m scripts.import_docs # re-index what changed -uv run python -m scripts.import_docs --prune # also drop deleted files +uv run python -m scripts.import_docs --prune # also drop deleted/out-of-scope files # Point it at extra directories (repeatable): uv run python -m scripts.import_docs --source ~/SomeOtherDocs @@ -92,16 +94,61 @@ embedded. - The import prints one line per file (`import: added|updated|unchanged| pruned …`) and ends with a greppable summary (`import: summary files=… - added=… updated=… unchanged=… pruned=… chunks=… embed_batches=…`), so it - is safe to run from a cron job or after every commit. -- Only **`*.md`** files are indexed. Directories like `.venv`, - `node_modules`, `.git`, `__pycache__`, `.pytest_cache`, `dist`, `build` - are skipped (see `.agent/PLAN.md` anchor A9). + added=… updated=… unchanged=… pruned=… chunks=… embed_batches=… + formats=md:203,yaml:267,…`), so it is safe to run from a cron job or + after every commit. +- Indexed formats (A9): **`md, markdown, txt, yaml, yml, json, py`** + (case-insensitive; narrow with `BOR_IMPORT_EXTENSIONS`). Any path with a + **dot-prefixed component** — hidden files or vendored caches like + `.esphome/.espressif/**` — is skipped, along with `.venv`, + `node_modules`, `.git`, `__pycache__`, `.pytest_cache`, `dist`, `build`. + `--prune` also drops documents whose files no longer match the filter — + that's how previously imported junk leaves the index. +- Non-markdown files get format-aware chunking (YAML top-level keys / + `---` docs, JSON top-level keys, Python top-level defs/classes via + stdlib `ast`) and their title comes from the file stem. - Unchanged files are **not re-embedded** — only new/changed ones, so refreshes are cheap. - To sanity-check the LLM backend (models + embedding dimension) after any aipi change: `uv run python -m scripts.llm_probe`. +## Checking retrieval quality + +Ask the *real* pipeline (live aipi embeddings + the current KB) whether a +question lands on the right document, with the gate verdict and per-document +cosine / FTS / fused scores: + +```bash +uv run python -m scripts.eval_retrieval "How did I install gitlab?" +uv run python -m scripts.eval_retrieval --from-file questions.txt --top 8 +``` + +Requires `AIPI_KEY` in the environment (same convention as +`scripts/llm_probe.py`) and an imported knowledge base. + +## How retrieval works (hybrid) + +Every question is embedded and also lexically tokenized (OR-joined, English +stemming) and searched **twice** against Postgres: + +1. **Vector** — pgvector cosine top-N (default `BOR_HYBRID_VECTOR_CANDIDATES=100`) +2. **Lexical** — a stored `tsvector` (GIN-indexed) matched with `to_tsquery`, + top-N by `ts_rank` (default `BOR_HYBRID_LEXICAL_CANDIDATES=30`) + +The two ranked lists are fused with **Reciprocal Rank Fusion** +(`score = Σ 1/(k + rank)`, `BOR_RRF_K=60`) — a chunk in both lists scores +nearly double, which is what lets a name-your-tool question ("gitlab") find +its own document even when the question embeds close to generic templates. + +The **honesty gate** (A8) then answers (HIGH) when the best cosine is ≥ +`BOR_RELEVANCE_THRESHOLD` (default `0.62`) **or** at least one chunk matched +lexically (`fts_hits > 0`) — it deflects (LOW) only when *both* signals are +absent. The top `BOR_TOP_N_DOCS` full documents are still what the LLM sees. + +`query_log` records every turn (`top_score` = best cosine, `fts_hits`, +`chunk_hits`, `deflected`, `sources`, `latency_ms`) — the raw material for +tuning: `psql … -c 'SELECT question, top_score, fts_hits, deflected FROM query_log ORDER BY created_at DESC LIMIT 20'`. + ## Debugging `debugpy` is **off by default** and *never imported* unless you opt in — @@ -203,9 +250,12 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`. | `BOR_LLM_CHAT_MODEL` | `turbo` | chat model | | `BOR_LLM_EMBED_MODEL` | `embed` | embedding model | | `BOR_EMBEDDING_DIM` | `768` | vector dimension (fixed at table creation) | -| `BOR_TOP_K_CHUNKS` | `4` | chunks retrieved per question | | `BOR_TOP_N_DOCS` | `2` | full documents fed to the LLM | -| `BOR_RELEVANCE_THRESHOLD` | `0.30` | best cosine similarity required to answer; below ⇒ honest deflection | +| `BOR_RELEVANCE_THRESHOLD` | `0.62` | answer when best cosine ≥ this **or** an FTS hit; below + no FTS ⇒ honest deflection | +| `BOR_HYBRID_VECTOR_CANDIDATES` | `100` | cosine list width for the RRF fusion | +| `BOR_HYBRID_LEXICAL_CANDIDATES` | `30` | FTS list width for the RRF fusion | +| `BOR_RRF_K` | `60` | RRF damping constant (`1/(k + rank)`) | +| `BOR_IMPORT_EXTENSIONS` | `md,markdown,txt,yaml,yml,json,py` | csv of importable formats (may only narrow the A9 set) | | `BOR_MAX_CONTEXT_CHARS` | `24000` | cap on total document text sent to the LLM | | `BOR_SUGGESTIONS` | built-in list | JSON list of onboarding chips | | `DEBUGPY` | `0` | `1` ⇒ attach-on-demand debugpy on `DEBUGPY_PORT` (default 5678) | @@ -227,23 +277,26 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`. drop + recreate the chunks table (new migration or manual `TRUNCATE chunks, documents`). - **Honest deflection (the amber “I haven't done anything like that” - bubble)** — every question passes the honesty gate: when the best - cosine similarity is below `BOR_RELEVANCE_THRESHOLD` (default `0.30`), - Brain switches to deflection mode instead of guessing. The LLM prompt - then carries weak-hit *titles only* (no document content), the reply - opens with “I haven't done anything like that”, the bubble renders - amber with “Maybe try” chips derived from the closest indexed titles, - the SSE `done` event carries `deflected: true` + `suggestions[]`, and - the `query_log` row records `deflected=true` + the weak `top_score`. - This is a feature, not a bug — the KB simply has no notes that close; - the chips always point at topics Brain really covers. + bubble)** — every question passes the honesty gate: deflection happens + only when the best cosine similarity is below + `BOR_RELEVANCE_THRESHOLD` (default `0.62`) **and** no chunk matched the + question lexically (`fts_hits = 0`). A weak cosine with a lexical hit + (name-your-tool questions) still gets a grounded answer. When it does + deflect, the LLM prompt carries weak-hit *titles only* (no document + content), the reply opens with “I haven't done anything like that”, the + bubble renders amber with “Maybe try” chips derived from the closest + indexed titles, the SSE `done` event carries `deflected: true` + + `suggestions[]`, and the `query_log` row records `deflected=true` + the + weak `top_score` + `fts_hits`. This is a feature, not a bug — the KB + simply has no notes that close; the chips always point at topics Brain + really covers. - **Answers deflect too often / too rarely** — tune `BOR_RELEVANCE_THRESHOLD` (lower = answers more, higher = more honest - deflection): `0.0` ⇒ every question gets answered, even unknown topics - (expect confident-sounding guesses); `1.0` ⇒ everything deflects - (nothing but a perfect 1.0 score counts as relevant). After changing - it, check the real scores: - `psql … -c 'SELECT question, top_score, deflected FROM query_log ORDER BY created_at DESC LIMIT 20'` + deflection): `0.0` ⇒ the gate leans entirely on FTS hits; `1.0` ⇒ + everything deflects unless a chunk matches lexically. The `embed` model's + cosines cluster in a ~0.6–0.85 band on the live KB, so the default is + `0.62`; after changing it, check the real scores: + `psql … -c 'SELECT question, top_score, fts_hits, deflected FROM query_log ORDER BY created_at DESC LIMIT 20'` - **KB offline banner in the chat** — Postgres isn't running: `podman compose up -d db`. - **Stuck "Thinking…"** — the LLM is slow or down; a 120s client timeout diff --git a/alembic/versions/0002_hybrid_retrieval.py b/alembic/versions/0002_hybrid_retrieval.py new file mode 100644 index 0000000..23f75d3 --- /dev/null +++ b/alembic/versions/0002_hybrid_retrieval.py @@ -0,0 +1,40 @@ +"""hybrid retrieval: generated FTS column on chunks + query_log.fts_hits + +Revision ID: 0002 +Revises: 0001 +Create Date: 2026-08-21 + +A7/A8 (revised 2026-08-21, owner permission): retrieval becomes hybrid +(cosine top-N + Postgres full-text top-N, RRF-fused). This adds: + +* ``chunks.tsv`` — generated ``TSVECTOR`` (``to_tsvector('english', + content) STORED``) + GIN index for the lexical candidate list. +* ``query_log.fts_hits`` — INT, nullable (pre-existing rows stay NULL: + the column only carries meaning from hybrid retrieval onward). +""" +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +revision = "0002" +down_revision = "0001" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + "ALTER TABLE chunks " + "ADD COLUMN tsv tsvector " + "GENERATED ALWAYS AS (to_tsvector('english', content)) STORED" + ) + op.execute("CREATE INDEX ix_chunks_tsv ON chunks USING gin (tsv)") + op.add_column("query_log", sa.Column("fts_hits", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("query_log", "fts_hits") + op.execute("DROP INDEX IF EXISTS ix_chunks_tsv") + op.execute("ALTER TABLE chunks DROP COLUMN IF EXISTS tsv") diff --git a/app/api/chat.py b/app/api/chat.py index 9300ee9..36b8238 100644 --- a/app/api/chat.py +++ b/app/api/chat.py @@ -1,18 +1,21 @@ """POST /api/chat — a RAG chat turn streamed over SSE (PLAN §3/§4). -Flow (LOCKED A7/A15): embed the question → pgvector cosine top-K chunks → -the **honesty gate** (A8: best score < ``BOR_RELEVANCE_THRESHOLD`` ⇒ -deflection) → locked persona prompt (PLAN §6) → ``turbo`` streamed as -``delta`` events → final ``done`` event (``deflected``, ``sources``, -``suggestions``) + ``query_log`` row + the per-turn log line (PLAN §9). +Flow (LOCKED A7/A15): embed the question → hybrid retrieval (cosine +top-N ∪ Postgres FTS top-N, RRF-fused) → the **honesty gate** → locked +persona prompt (PLAN §6) → ``turbo`` streamed as ``delta`` events → final +``done`` event (``deflected``, ``sources``, ``suggestions``) + +``query_log`` row + the per-turn log line (PLAN §9). Mid-stream failures become a structured ``error`` event; a pre-stream DB outage is a plain 503 JSON. -Honesty gate: a weak retrieval (score strictly below the threshold — or -an empty KB) flips the turn to deflection mode: the LOW prompt carries -weak-hit *titles only* (never document content) plus deterministic -"Maybe try" chips, and the ``done`` event / ``query_log`` row record -``deflected=true`` with the weak score. +Honesty gate (A8, revised 2026-08-21): LOW — deflection — only when the +best cosine is strictly below ``BOR_RELEVANCE_THRESHOLD`` **and** no +candidate chunk FTS-matches the question (``fts_hits == 0``). A +name-your-tool question with weak vector overlap but a lexical hit still +gets a grounded answer. Deflection mode carries weak-hit *titles only* +(never document content) plus deterministic "Maybe try" chips, and the +``done`` event / ``query_log`` row record ``deflected=true``, the weak +score and the ``fts_hits`` count. """ from __future__ import annotations @@ -62,7 +65,8 @@ def sse_event(payload: dict[str, Any]) -> str: class TurnPlan: """What one chat turn sends to the LLM and reports on ``done``.""" - top_score: float + top_score: float # best cosine across candidates (query_log.top_score) + fts_hits: int # lexical (OR-tsquery) candidates matched deflected: bool system_prompt: str docs: list[Document] # cited sources (weak hits when deflected) @@ -70,25 +74,32 @@ class TurnPlan: def plan_turn(chunks: Sequence[RetrievedChunk], settings: Settings) -> TurnPlan: - """Apply the honesty gate (A8) and assemble prompt + context for a turn. + """Apply the honesty gate (A8, revised) and assemble prompt + context. - * ``top_score >= threshold`` → grounded: HIGH prompt with the full - top-N documents, no suggestions. A score exactly at the threshold - is an answer — the gate is strict (``score < threshold``). - * ``top_score < threshold`` (or no hits at all) → deflected: LOW - prompt (``DEFLECT_MODE``) with weak-hit titles only — never document - content — plus deterministic alternative-question chips derived - from those titles. + * **HIGH (grounded)** when ``best_cosine >= threshold`` **or** + ``fts_hits > 0``: HIGH prompt with the full top-N documents, no + suggestions. A cosine exactly at the threshold is an answer — the + gate is strict (``< threshold``). + * **LOW (deflected)** only when ``best_cosine < threshold`` **and** + ``fts_hits == 0`` (or no hits at all): LOW prompt (``DEFLECT_MODE``) + with weak-hit titles only — never document content — plus + deterministic alternative-question chips derived from those titles. + + ``top_score`` (stored in ``query_log``) is the best cosine, so the + gate input is always a pure vector-similarity number; the lexical + signal is recorded separately as ``fts_hits``. """ - top_score = chunks[0].score if chunks else 0.0 - if top_score >= settings.relevance_threshold: + best_cosine = max((c.cosine for c in chunks), default=0.0) + fts_hits = sum(1 for c in chunks if c.fts_hit) + if best_cosine >= settings.relevance_threshold or fts_hits > 0: docs = select_documents( chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars ) - return TurnPlan(top_score, False, build_high_prompt(docs), docs, []) + return TurnPlan(best_cosine, fts_hits, False, build_high_prompt(docs), docs, []) titles = weak_hit_titles(chunks) return TurnPlan( - top_score, + best_cosine, + fts_hits, True, build_deflect_prompt(titles), select_documents(chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars), @@ -143,7 +154,7 @@ async def chat( # HIGH (grounded) or LOW (deflected) prompt + context. settings = get_settings() try: - chunks = retrieve(db, question_vec) + chunks = retrieve(db, request.message, question_vec) plan = plan_turn(chunks, settings) except Exception: # noqa: BLE001 — DB failure mid-turn logger.exception( @@ -188,6 +199,7 @@ async def chat( QueryLog( question=request.message, top_score=plan.top_score, + fts_hits=plan.fts_hits, chunk_hits=len(chunks), deflected=plan.deflected, sources=", ".join(source_paths), @@ -199,11 +211,12 @@ async def chat( logger.exception("chat: failed to write query_log question=%r", request.message) logger.info( - "question=%r embed_ms=%d top_score=%.3f threshold=%.2f deflected=%s " - "sources=%r total_ms=%d", + "question=%r embed_ms=%d top_score=%.3f fts_hits=%d threshold=%.2f " + "deflected=%s sources=%r total_ms=%d", request.message, embed_ms, plan.top_score, + plan.fts_hits, settings.relevance_threshold, plan.deflected, source_paths, diff --git a/app/config.py b/app/config.py index 9fa1c5d..bb2a2cc 100644 --- a/app/config.py +++ b/app/config.py @@ -8,8 +8,15 @@ from __future__ import annotations import os from functools import lru_cache +from pydantic import field_validator from pydantic_settings import BaseSettings, SettingsConfigDict +#: The A9 import formats (PLAN anchor A9, revised 2026-08-21). +#: ``BOR_IMPORT_EXTENSIONS`` may narrow — but never widen — this set. +_ALLOWED_IMPORT_EXTENSIONS: frozenset[str] = frozenset( + {"md", "markdown", "txt", "yaml", "yml", "json", "py"} +) + class Settings(BaseSettings): model_config = SettingsConfigDict( @@ -37,14 +44,58 @@ class Settings(BaseSettings): # --- RAG tuning --- embedding_dim: int = 768 # verified against aipi /v1 (embed model) - top_k_chunks: int = 4 top_n_docs: int = 2 - relevance_threshold: float = 0.30 + # Honesty gate (A8, re-tuned 2026-08-21): the ``embed`` model's cosine + # scores compress into 0.41–0.84 on the real corpus, so the old 0.30 + # default never discriminated. LOW only fires when best cosine < this + # AND no candidate chunk matches the question lexically (see A8). + relevance_threshold: float = 0.62 max_context_chars: int = 24_000 chunk_target_chars: int = 2_000 chunk_overlap_chars: int = 200 embed_batch_size: int = 16 + # --- Hybrid retrieval (A7, revised 2026-08-21) --- + # cosine top-N ∪ Postgres FTS top-N, fused with Reciprocal Rank Fusion + # (score = Σ 1/(rrf_k + rank) over the lists a chunk appears in). + # + # The vector window is deliberately wider than the lexical one: a + # name-your-tool question's best *lexical* chunk (e.g. the "Install" + # section of gitlab.md) can sit far down the vector ranking because the + # question embeds close to generic templates. A 100-wide window is what + # lets such chunks double-hit (one RRF term per list) and outrank a + # template that owns vector rank 1 — measured 2026-08-22 against the + # live 2774-chunk KB for "How did I install gitlab?" (gitlab.md:1 at + # vrank 100 / lrank 3 → fused 0.0221 vs the template's 0.0164). + hybrid_vector_candidates: int = 100 + hybrid_lexical_candidates: int = 30 + rrf_k: int = 60 + + # --- Import scope (A9, revised 2026-08-21) --- + # Comma-separated list of lowercased file extensions (no dot) imported + # by ``scripts/import_docs.py``. Hidden (dot) path components are always + # skipped, plus the importer's exclusion list. + # Stored as a raw CSV string (env-native — no JSON) and parsed on demand + # via :py:meth:`import_extension_set`. ``mode="after"`` validation runs + # against the raw string so a typo fails loudly at startup. + import_extensions: str = "md,markdown,txt,yaml,yml,json,py" + + @field_validator("import_extensions") + @classmethod + def _import_extensions_known(cls, v: str) -> str: + """Reject unknown/empty formats loudly instead of silently importing + nothing (a typo like ``md,jsonn`` would otherwise walk zero files).""" + exts = {part.strip().lstrip(".").lower() for part in v.split(",") if part.strip()} + if not exts: + raise ValueError("import_extensions must name at least one format") + unknown = exts - _ALLOWED_IMPORT_EXTENSIONS + if unknown: + raise ValueError( + f"unknown import extension(s): {', '.join(sorted(unknown))} — " + f"allowed: {', '.join(sorted(_ALLOWED_IMPORT_EXTENSIONS))}" + ) + return v + # Suggested questions (onboarding + empty state). suggestions: list[str] = [ "How is my Kubernetes cluster set up?", @@ -53,6 +104,15 @@ class Settings(BaseSettings): "What's currently running in the homelab?", ] + @property + def import_extension_set(self) -> frozenset[str]: + """Lowercased, dotted extension set (``.md``) for path filtering.""" + return frozenset( + f".{part.strip().lstrip('.').lower()}" + for part in self.import_extensions.split(",") + if part.strip() + ) + @property def effective_api_key(self) -> str: """API key for aipi: explicit setting, then $AIPI_KEY, then a placeholder.""" diff --git a/app/models.py b/app/models.py index 1d9ac22..94450f1 100644 --- a/app/models.py +++ b/app/models.py @@ -2,7 +2,7 @@ Data model — see ``.agent/PLAN.md`` §Data Model: -* ``documents`` — one row per ``*.md`` file (full content, path, sha256 hash). +* ``documents`` — one row per imported A9 file (full content, path, sha256 hash). * ``chunks`` — retrieval units; each chunk points at its parent document via ``document_id``. This is how an embedding maps back to a document path (the "feed the whole document" requirement). @@ -74,6 +74,9 @@ class QueryLog(Base): id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) question: Mapped[str] = mapped_column(Text) top_score: Mapped[float] = mapped_column(Float, default=0.0) # best cosine similarity + #: Lexical (FTS) candidates matched — the OR-tsquery hit count (A8). NULL + #: for pre-hybrid rows (migration 0002). + fts_hits: Mapped[int | None] = mapped_column(Integer) chunk_hits: Mapped[int] = mapped_column(Integer, default=0) deflected: Mapped[bool] = mapped_column(Boolean, default=False) # True = honest "no idea" sources: Mapped[str] = mapped_column(Text, default="") # comma-joined source paths diff --git a/app/rag/chunker.py b/app/rag/chunker.py index 95ee71a..af6bca7 100644 --- a/app/rag/chunker.py +++ b/app/rag/chunker.py @@ -1,25 +1,36 @@ -"""Markdown-aware chunker (PLAN §5 chunking policy). +"""Format-aware chunker (PLAN §5 chunking policy). -Pure functions, no I/O — fully unit-testable. +Pure functions, no I/O — fully unit-testable. Stdlib only. -Policy ------- -* **Sections** are split on ATX headings of level ≥ 2 (``## ``/``### ``/…). -* A section that fits in ``target_chars`` becomes a single chunk. -* A longer section is sub-split at paragraph boundaries (blank lines outside - code fences); each chunk after the first starts with the trailing - ``overlap_chars`` of the previous chunk so context survives the cut. -* Every chunk keeps its nearest preceding heading line (the section anchor), - so a retrieval hit is always readable in context. -* **Code fences** (``` / ~~~) are atomic: a chunk boundary never falls - inside one, and lines inside a fence are never mistaken for headings or - paragraph breaks. One exception: a fence *larger than* :data:`HARD_MAX_CHARS` - is split by line, because aipi's local embedding model rejects requests - over ~1024 input tokens and a single 5000-char code block would blow - past that on its own. +:func:`chunk_document` dispatches on the file's lowercased suffix; +per-format policies: + +* **md / markdown** — sections are split on ATX headings of level ≥ 2 + (``## ``/``### ``/…); a section that fits in ``target_chars`` becomes a + single chunk, a longer one is sub-split at paragraph boundaries (blank + lines outside code fences) with ``overlap_chars`` carry-over, and every + chunk keeps its nearest preceding heading line (the section anchor). Code + fences are atomic (a boundary never falls inside one) except a fence + larger than :data:`HARD_MAX_CHARS`, which is split by line. +* **yaml / yml** — blocks start at ``---`` document separators and at + top-level (indent-0) ``key:`` lines; every chunk keeps its key lines as + anchors, so a hit is always readable in context. +* **json** — pretty-printed (``json.dumps(obj, indent=2)``) and split on + top-level keys (one ``{key: value}`` block per key); unparseable input + falls back to paragraph packing. +* **py** — split at top-level defs/classes via the stdlib ``ast`` (the + module preamble — imports, constants — is its own block); an oversized + definition falls back to line packing. +* **txt** (and any unknown suffix) — paragraph packing. + +Every format honors :data:`HARD_MAX_CHARS` (1200 — the aipi ~1024-token +request cap) and the target/overlap settings; oversized blocks are split +by line so no chunk can exceed the cap. """ from __future__ import annotations +import ast +import json import re from collections.abc import Sequence @@ -203,3 +214,147 @@ def chunk_markdown( for start, end in _section_ranges(lines, flags): chunks.extend(_chunk_section(lines[start:end], flags[start:end], target, overlap)) return [c for c in chunks if c.strip()] + + +# --------------------------------------------------------------------------- +# Non-markdown formats (A9, revised 2026-08-21): yaml/yml, json, py, txt +# --------------------------------------------------------------------------- + +#: YAML document separator (column 0). +_YAML_DOC_SEP_RE = re.compile(r"^-{3,}\s*$") +#: Top-level YAML key (column 0, no leading whitespace) — the block anchor. +_YAML_KEY_RE = re.compile(r"^[A-Za-z0-9_.\-]+\s*:") + + +def _yaml_blocks(lines: Sequence[str]) -> list[str]: + """Group YAML lines into blocks: ``---`` separators and indent-0 + ``key:`` lines each start a new block (the key line stays the anchor).""" + blocks: list[str] = [] + cur: list[str] = [] + for line in lines: + if cur and (_YAML_DOC_SEP_RE.match(line) or _YAML_KEY_RE.match(line)): + blocks.append("\n".join(cur)) + cur = [] + cur.append(line) + if cur: + blocks.append("\n".join(cur)) + return [b for b in blocks if b.strip()] + + +def chunk_yaml(content: str, target_chars: int = 2000, overlap_chars: int = 200) -> list[str]: + """Split YAML on document separators + top-level keys (see module docstring).""" + target, overlap = _normalize_target_overlap(target_chars, overlap_chars) + return _pack_blocks(_yaml_blocks(content.splitlines()), target, overlap) + + +def _json_blocks(content: str) -> list[str] | None: + """Pretty-printed per-top-level-key blocks, or ``None`` if unparseable.""" + try: + obj = json.loads(content) + except (ValueError, TypeError): + return None + if isinstance(obj, dict): + return [json.dumps({k: v}, indent=2) for k, v in obj.items()] + # Top-level list/scalar: nothing to key on — one pretty-printed block. + return [json.dumps(obj, indent=2)] + + +def chunk_json(content: str, target_chars: int = 2000, overlap_chars: int = 200) -> list[str]: + """Pretty-print JSON and split on top-level keys (unparseable → paragraphs).""" + target, overlap = _normalize_target_overlap(target_chars, overlap_chars) + blocks = _json_blocks(content) + if blocks is None: + return chunk_text(content, target, overlap) + return _pack_blocks(blocks, target, overlap) + + +def _python_blocks(content: str) -> list[str] | None: + """Line blocks: module preamble, then one per top-level def/class. + + Returns ``None`` when the source does not parse (→ line/paragraph + packing fallback). + """ + try: + tree = ast.parse(content) + except (SyntaxError, ValueError): + return None + lines = content.splitlines() + tops = [ + node + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + ] + tops.sort(key=lambda n: n.lineno) + ranges: list[tuple[int, int]] = [] + for node in tops: + start = node.lineno - 1 + for dec in node.decorator_list: + start = min(start, dec.lineno - 1) + end = node.end_lineno or node.lineno # end_lineno is None on odd parses + ranges.append((start, end)) # 0-based start, 1-based end + blocks: list[str] = [] + cursor = 0 + for start, end in ranges: + if start > cursor: + blocks.append("\n".join(lines[cursor:start])) + blocks.append("\n".join(lines[start:end])) + cursor = end + if cursor < len(lines): + blocks.append("\n".join(lines[cursor:])) + return [b for b in blocks if b.strip()] + + +def chunk_python(content: str, target_chars: int = 2000, overlap_chars: int = 200) -> list[str]: + """Split Python on top-level defs/classes (stdlib ``ast``; see module doc).""" + target, overlap = _normalize_target_overlap(target_chars, overlap_chars) + blocks = _python_blocks(content) + if blocks is None: + return chunk_text(content, target, overlap) + return _pack_blocks(blocks, target, overlap) + + +def chunk_text(content: str, target_chars: int = 2000, overlap_chars: int = 200) -> list[str]: + """Plain-text paragraph packing (blank lines separate paragraphs).""" + target, overlap = _normalize_target_overlap(target_chars, overlap_chars) + lines = content.splitlines() + return _pack_blocks(_paragraph_blocks(lines, [False] * len(lines)), target, overlap) + + +def _normalize_target_overlap(target_chars: int, overlap_chars: int) -> tuple[int, int]: + """Validate + clamp the size policy (shared by every format).""" + if target_chars <= 0: + raise ValueError("target_chars must be > 0") + if overlap_chars < 0: + raise ValueError("overlap_chars must be >= 0") + # The endpoint's token cap is absolute — a larger target is unsafe. + target = min(target_chars, HARD_MAX_CHARS) + return target, min(overlap_chars, target - 1) + + +#: suffix → chunker (A9, revised: md, markdown, txt, yaml, yml, json, py). +_FORMAT_CHUNKERS = { + ".md": chunk_markdown, + ".markdown": chunk_markdown, + ".txt": chunk_text, + ".yaml": chunk_yaml, + ".yml": chunk_yaml, + ".json": chunk_json, + ".py": chunk_python, +} + + +def chunk_document( + content: str, + path: str, + target_chars: int = 2000, + overlap_chars: int = 200, +) -> list[str]: + """Chunk *content* according to *path*'s lowercased suffix. + + Unknown suffixes fall back to plain-text paragraph packing (the + importer only passes A9-format files, so this is belt-and-braces). + """ + name = path.rsplit("/", 1)[-1] + suffix = "." + name.rsplit(".", 1)[-1].lower() if "." in name else "" + chunker = _FORMAT_CHUNKERS.get(suffix, chunk_text) + return chunker(content, target_chars, overlap_chars) diff --git a/app/rag/importer.py b/app/rag/importer.py index 6928755..1baeb82 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -1,23 +1,30 @@ """Knowledge-base importer (PLAN §5 / §9 / §11). -Walks ``*.md`` files (A9 exclusion list), diffs by sha256 against -``documents.content_hash`` and, for every new or changed file, runs the -two-phase upsert: +Walks the A9-format files (``md, markdown, txt, yaml, yml, json, py`` by +default — ``BOR_IMPORT_EXTENSIONS``; case-insensitive), diffs by sha256 +against ``documents.content_hash`` and, for every new or changed file, runs +the two-phase upsert: 1. upsert the document row and replace its chunk rows (embeddings NULL) 2. embed the new chunks in batches and attach the vectors 3. commit — one transaction per file, so a failed embedding leaves the database untouched and the file is simply retried on the next run +Scope (A9, revised 2026-08-21): any path containing a dot-prefixed +component (hidden dirs — vendored caches like ``.esphome/.espressif/**`` — +or hidden files) is skipped, plus the well-known exclusion list. + ``prune=True`` deletes documents (of the imported sources only) whose files -no longer exist. Per-file logging uses the verbs -``added | updated | unchanged | pruned`` plus a summary line (PLAN §9). +no longer exist **or no longer match the format filter** — this is how +previously-imported junk (e.g. dot-dir READMEs) leaves the index. Per-file +logging uses the verbs ``added | updated | unchanged | pruned`` plus a +summary line with per-format counts (PLAN §9). """ from __future__ import annotations import hashlib import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path from typing import Protocol @@ -28,7 +35,7 @@ from sqlalchemy.orm import Session from app.config import Settings from app.db import SessionLocal from app.models import Chunk, Document -from app.rag.chunker import chunk_markdown, extract_title +from app.rag.chunker import chunk_document, extract_title from app.rag.llm import EmbeddingError logger = logging.getLogger("app.importer") @@ -60,11 +67,20 @@ class ImportSummary: errors: int = 0 chunks: int = 0 embed_batches: int = 0 + #: Files walked, keyed by lowercased extension (``md``, ``yaml``, …). + formats: dict[str, int] = field(default_factory=dict) + + def format_counts(self) -> str: + """``md:203,yaml:267,py:14`` — highest count first (PLAN §9).""" + if not self.formats: + return "none" + ordered = sorted(self.formats.items(), key=lambda kv: (-kv[1], kv[0])) + return ",".join(f"{ext}:{count}" for ext, count in ordered) def log(self) -> None: logger.info( "import: summary files=%d added=%d updated=%d unchanged=%d pruned=%d " - "errors=%d chunks=%d embed_batches=%d", + "errors=%d chunks=%d embed_batches=%d formats=%s", self.files, self.added, self.updated, @@ -73,17 +89,32 @@ class ImportSummary: self.errors, self.chunks, self.embed_batches, + self.format_counts(), ) -def iter_markdown_files(root: Path, excluded: frozenset[str] = EXCLUDED_DIRS) -> list[Path]: - """All ``*.md`` files under *root* (sorted), skipping excluded dirs (A9).""" +def iter_importable_files( + root: Path, + extensions: frozenset[str], + excluded: frozenset[str] = EXCLUDED_DIRS, +) -> list[Path]: + """All importable files under *root* (sorted), per the A9 scope rules. + + *extensions* is a set of lowercased dotted suffixes (``{'.md', '.py'}``). + Skips: any path with a dot-prefixed component (hidden dirs/files — + vendored caches like ``.esphome/.espressif/**``) and the well-known + non-content directories in *excluded*. + """ if not root.is_dir(): return [] files: list[Path] = [] - for path in sorted(root.rglob("*.md")): + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue rel = path.relative_to(root) - if any(part in excluded for part in rel.parts[:-1]): + if any(part.startswith(".") or part in excluded for part in rel.parts): + continue + if path.suffix.lower() not in extensions: continue files.append(path) return files @@ -97,7 +128,7 @@ async def import_sources( limit: int | None = None, session: Session | None = None, ) -> ImportSummary: - """Import every ``*.md`` under *sources* (see module docstring). + """Import every A9-format file under *sources* (see module docstring). ``session`` may be supplied (tests); a private one is opened and closed otherwise. ``limit`` caps the number of files processed (debug only) and @@ -120,12 +151,14 @@ async def import_sources( break source = root.name source_names.add(source) - for path in iter_markdown_files(root): + for path in iter_importable_files(root, llm.settings.import_extension_set): if limit is not None and summary.files >= limit: break rel = path.relative_to(root).as_posix() seen.add((source, rel)) summary.files += 1 + ext = path.suffix.lower().lstrip(".") or "unknown" + summary.formats[ext] = summary.formats.get(ext, 0) + 1 try: await _index_file( session, source=source, rel=rel, full_path=path, llm=llm, @@ -172,7 +205,12 @@ async def _index_file( return verb = "updated" if doc is not None else "added" - title = extract_title(content, fallback=full_path.stem) + # A ``#`` line is a real heading in markdown but a comment in every + # other format — titles for those come from the file stem. + if full_path.suffix.lower() in (".md", ".markdown"): + title = extract_title(content, fallback=full_path.stem) + else: + title = full_path.stem if doc is None: doc = Document( source=source, @@ -200,7 +238,7 @@ async def _index_file( # policy stays intact for the rest of the KB. target = max(400, settings.chunk_target_chars) while True: - chunks_text = chunk_markdown(content, target, settings.chunk_overlap_chars) + chunks_text = chunk_document(content, rel, target, settings.chunk_overlap_chars) doc.chunks = [ Chunk(document_id=doc.id, position=i, content=c) for i, c in enumerate(chunks_text) ] diff --git a/app/rag/retriever.py b/app/rag/retriever.py index 071ada7..ad32711 100644 --- a/app/rag/retriever.py +++ b/app/rag/retriever.py @@ -1,19 +1,31 @@ -"""pgvector cosine retrieval → parent-document mapping (PLAN §3/§6, A7). +"""Hybrid retrieval: pgvector cosine ∪ Postgres FTS, RRF-fused (PLAN §6, A7). -Retrieval returns the *chunks* closest to the question embedding (top-K by -cosine distance). The product requirement is that the LLM receives the -**entire relevant document**, not just the chunk (LOCKED A7) — so -:meth:`select_documents` maps chunk hits back to their parent documents -(``chunks.document_id → documents``), dedupes, ranks by best chunk score, -and caps the combined context at ``BOR_MAX_CONTEXT_CHARS``. +* **Vector list** — top-N chunks by cosine distance (``embedding <=> $1``), + each carrying its cosine ``1 − distance`` (the honesty-gate input). +* **Lexical list** — top-N chunks matching an OR-``tsquery`` over the + question's tokens, ordered by ``ts_rank``. This is what finds + name-your-tool questions ("gitlab") that vector similarity buries. +* **Fusion** — Reciprocal Rank Fusion (``score = Σ 1/(k + rank)`` over the + lists a chunk appears in; chunks hit by both lists get both terms). The + fused score ranks; :meth:`select_documents` and :func:`weak_hit_titles` + keep working off ``score``. + +The product requirement is unchanged (LOCKED A7): the LLM receives the +**entire relevant document**, not just the chunk — chunk hits map back to +their parents, dedupe, rank by best fused score, and the combined context +is capped at ``BOR_MAX_CONTEXT_CHARS``. + +Deterministic tie-break for equal fused scores: +``(−fused, −cosine, document.path, chunk.position)``. """ from __future__ import annotations +import re import uuid from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace -from sqlalchemy import select +from sqlalchemy import select, text from sqlalchemy.orm import Session from app.config import get_settings @@ -22,50 +34,209 @@ from app.models import Chunk, Document #: Marker appended when the context budget is exceeded (PLAN §6). TRUNCATION_MARKER = "[…truncated…]" +#: Alphanumeric tokens of a question (``to_tsquery`` input, OR-joined). +_TOKEN_RE = re.compile(r"[a-z0-9]+") + +#: One row of the lexical candidate query (all fields needed to build a +#: detached :class:`Document` plus the chunk fields and ``ts_rank``). +_LEXICAL_SQL = text( + """ + SELECT c.id AS chunk_id, + c.position AS position, + c.content AS content, + d.id AS doc_id, + d.source AS source, + d.path AS path, + d.full_path AS full_path, + d.title AS title, + d.content AS doc_content, + d.content_hash AS content_hash, + d.indexed_at AS indexed_at, + ts_rank(c.tsv, to_tsquery('english', :tsquery)) AS rank + FROM chunks c + JOIN documents d ON d.id = c.document_id + WHERE c.tsv @@ to_tsquery('english', :tsquery) + ORDER BY rank DESC, d.path ASC, c.position ASC + LIMIT :limit + """ +) + @dataclass class RetrievedChunk: - """One chunk hit: its cosine score plus the parent document row.""" + """One retrieval candidate: fused rank score + parent document row. + + * ``score`` — RRF fused score (the ranking key for document selection + and weak-hit titles). + * ``cosine`` — vector similarity ``1 − distance`` (the honesty-gate + input; ``0.0`` for lexical-only hits that have no vector rank). + * ``fts_hit`` — the chunk matched the question's OR-tsquery. + """ chunk_id: uuid.UUID position: int content: str - score: float # 1 − cosine_distance (higher is more similar) + score: float document: Document + cosine: float = 0.0 + fts_hit: bool = False -def retrieve( - db: Session, question_embedding: list[float], top_k: int | None = None -) -> list[RetrievedChunk]: - """Top-*top_k* chunks by pgvector cosine distance (``<=>``). +def lexical_tsquery(question: str) -> str | None: + """OR-joined token string for ``to_tsquery('english', …)``, or ``None``. - ``score = 1 − distance``. Results are ordered by ascending distance, so - index 0 is the best hit. Chunks whose embedding is still NULL (two-phase - import in progress) are skipped. + Tokens are lowercased ``[a-z0-9]+`` runs, de-duplicated in order of + first appearance. Postgres does the lexing/stemming; a question whose + tokens are all stopwords lexes to an *empty* tsquery (which matches + nothing), so no special-casing is needed there. Pure-symbol questions + ("???", "🔧") yield no tokens → ``None`` → no lexical query at all. + """ + seen: set[str] = set() + tokens: list[str] = [] + for tok in _TOKEN_RE.findall(question.lower()): + if tok not in seen: + seen.add(tok) + tokens.append(tok) + return " | ".join(tokens) if tokens else None + + +def fuse( + vector: Sequence[RetrievedChunk], + lexical: Sequence[RetrievedChunk], + k: int, +) -> list[RetrievedChunk]: + """Reciprocal Rank Fusion over the two ranked candidate lists. + + ``score(chunk) = Σ 1/(k + rank)`` — one term per list the chunk appears + in (ranks are 1-based; a chunk in both lists gets both terms). Returns + the union ordered by ``(-score, -cosine, document.path, position)``. + + Lexical-only hits (no vector rank) enter with ``cosine=0.0`` and + ``fts_hit=True``; vector chunks matched by the lexical list get + ``fts_hit=True`` in place (the input objects are mutated — callers + should not reuse them afterwards). + """ + if k <= 0: + raise ValueError("rrf k must be > 0") + by_id: dict[uuid.UUID, RetrievedChunk] = {} + fused: dict[uuid.UUID, float] = {} + for rank, rc in enumerate(vector, start=1): + by_id[rc.chunk_id] = rc + fused[rc.chunk_id] = fused.get(rc.chunk_id, 0.0) + 1.0 / (k + rank) + for rank, rc in enumerate(lexical, start=1): + term = 1.0 / (k + rank) + if rc.chunk_id in by_id: + existing = by_id[rc.chunk_id] + by_id[rc.chunk_id] = replace(existing, fts_hit=True) + fused[rc.chunk_id] += term + else: + rc = replace(rc, fts_hit=True) + by_id[rc.chunk_id] = rc + fused[rc.chunk_id] = fused.get(rc.chunk_id, 0.0) + term + out = [replace(rc, score=fused[rc.chunk_id]) for rc in by_id.values()] + out.sort(key=lambda rc: (-rc.score, -rc.cosine, rc.document.path, rc.position)) + return out + + +def _vector_candidates( + db: Session, question_embedding: list[float], limit: int +) -> list[RetrievedChunk]: + """Top-*limit* chunks by pgvector cosine distance (``<=>``). + + ``cosine = 1 − distance``. Chunks whose embedding is still NULL + (two-phase import in progress) are skipped. """ - k = top_k if top_k is not None else get_settings().top_k_chunks distance = Chunk.embedding.cosine_distance(question_embedding) rows = db.execute( select(Chunk, distance.label("distance"), Document) .join(Document, Chunk.document_id == Document.id) .where(Chunk.embedding.is_not(None)) .order_by(distance) - .limit(k) + .limit(limit) ).all() return [ RetrievedChunk( chunk_id=chunk.id, position=chunk.position, content=chunk.content, - score=round(1.0 - float(dist), 6), + score=0.0, # fused score is filled in by :func:`fuse` document=doc, + cosine=round(1.0 - float(dist), 6), ) for chunk, dist, doc in rows ] +def _lexical_candidates(db: Session, question: str, limit: int) -> list[RetrievedChunk]: + """Top-*limit* chunks matching the question's OR-tsquery (A7). + + Ordered by ``ts_rank`` (with ``d.path, c.position`` as the + deterministic tie-break); an empty tsquery (stopword-only question) + simply matches nothing. + """ + tsquery = lexical_tsquery(question) + if tsquery is None: + return [] + rows = db.execute( + _LEXICAL_SQL, {"tsquery": tsquery, "limit": limit} + ).all() + out: list[RetrievedChunk] = [] + for row in rows: + doc = Document( + id=row.doc_id, + source=row.source, + path=row.path, + full_path=row.full_path, + title=row.title, + content=row.doc_content, + content_hash=row.content_hash, + indexed_at=row.indexed_at, + ) + out.append( + RetrievedChunk( + chunk_id=row.chunk_id, + position=row.position, + content=row.content, + score=0.0, # filled in by :func:`fuse` + document=doc, + cosine=0.0, # no vector rank — lexical-only hit + fts_hit=True, + ) + ) + return out + + +def retrieve( + db: Session, + question: str, + question_embedding: list[float], + vector_candidates: int | None = None, + lexical_candidates: int | None = None, +) -> list[RetrievedChunk]: + """Hybrid retrieval (A7): vector top-N ∪ FTS top-N, RRF-fused. + + Returns the fused candidate list in rank order (best first). Each + :class:`RetrievedChunk` carries the fused ``score`` (ranking), the + ``cosine`` similarity (honesty gate) and the ``fts_hit`` flag. + """ + settings = get_settings() + v_n = ( + settings.hybrid_vector_candidates if vector_candidates is None else vector_candidates + ) + l_n = ( + settings.hybrid_lexical_candidates if lexical_candidates is None else lexical_candidates + ) + if v_n <= 0: + raise ValueError("vector_candidates must be >= 1") + if l_n <= 0: + raise ValueError("lexical_candidates must be >= 1") + vector = _vector_candidates(db, question_embedding, v_n) + lexical = _lexical_candidates(db, question, l_n) + return fuse(vector, lexical, settings.rrf_k) + + def weak_hit_titles(chunks: Sequence[RetrievedChunk]) -> list[str]: - """Distinct parent-document titles of *chunks*, best chunk score first. + """Distinct parent-document titles of *chunks*, best fused score first. Deflection mode (PLAN §6, A8) is built from these *titles only* — the LOW prompt and the "Maybe try" chips never see document content. @@ -85,7 +256,7 @@ def select_documents( n: int | None = None, max_chars: int | None = None, ) -> list[Document]: - """Map chunk hits to distinct parent documents, ranked by best chunk score. + """Map chunk hits to distinct parent documents, ranked by best fused score. At most *n* documents are returned (default ``BOR_TOP_N_DOCS``). The returned rows carry the full document content; if the combined content diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py new file mode 100644 index 0000000..152f5ab --- /dev/null +++ b/scripts/eval_retrieval.py @@ -0,0 +1,129 @@ +"""Evaluate hybrid retrieval against the live knowledge base (phase 09). + +Embeds each question via aipi, runs the same hybrid search the chat API +uses (cosine top-N + FTS top-N, RRF-fused), and prints the top-5 documents +with their cosine / fts / fused scores plus the honesty-gate verdict: + + uv run python -m scripts.eval_retrieval "How did I install gitlab?" + uv run python -m scripts.eval_retrieval --from-file questions.txt + +Requires ``AIPI_KEY`` in the environment (same convention as +``scripts/llm_probe.py``) and an imported knowledge base +(``python -m scripts.import_docs``). Exit code 0 when all questions were +scored (deflections are a normal result — the verdict column shows them). +""" +from __future__ import annotations + +import argparse +import asyncio +import os +import sys + +from dotenv import load_dotenv + +load_dotenv() + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="python -m scripts.eval_retrieval", + description="Rank hybrid retrieval results for one or more questions.", + ) + p.add_argument( + "questions", + nargs="*", + metavar="QUESTION", + help="one or more questions to evaluate", + ) + p.add_argument( + "--from-file", + type=str, + default=None, + metavar="PATH", + help="read questions from a file (one per line, blanks/# skipped)", + ) + p.add_argument( + "--top", + type=int, + default=5, + help="documents to print per question (default: 5)", + ) + return p + + +def read_questions(args: argparse.Namespace) -> list[str]: + questions = list(args.questions) + if args.from_file: + with open(args.from_file, encoding="utf-8") as f: + questions.extend( + line.strip() for line in f if line.strip() and not line.lstrip().startswith("#") + ) + return questions + + +async def _embed_all(llm, questions: list[str]) -> list[list[float]]: + return await llm.embed(questions) + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + api_key = os.environ.get("BOR_LLM_API_KEY") or os.environ.get("AIPI_KEY", "") + if not api_key or api_key == "not-needed": + print( + "eval_retrieval: AIPI_KEY is required in the environment " + "(same convention as scripts/llm_probe.py).", + file=sys.stderr, + ) + return 1 + questions = read_questions(args) + if not questions: + print("eval_retrieval: no questions given (positional or --from-file).", file=sys.stderr) + return 1 + + from app.config import get_settings + from app.db import SessionLocal, db_available + from app.rag.llm import LLMClient + from app.rag.retriever import RetrievedChunk, retrieve + + settings = get_settings() + if not db_available(): + print("eval_retrieval: Postgres is down — run `podman compose up -d db`.", file=sys.stderr) + return 1 + + llm = LLMClient(settings) + vectors = asyncio.run(_embed_all(llm, questions)) + + print( + f"eval: threshold={settings.relevance_threshold} " + f"vector_candidates={settings.hybrid_vector_candidates} " + f"lexical_candidates={settings.hybrid_lexical_candidates} rrf_k={settings.rrf_k}" + ) + with SessionLocal() as db: + for question, vec in zip(questions, vectors, strict=True): + chunks = retrieve(db, question, vec) + best_cosine = max((c.cosine for c in chunks), default=0.0) + fts_hits = sum(1 for c in chunks if c.fts_hit) + verdict = ( + "LOW (deflect)" + if best_cosine < settings.relevance_threshold and fts_hits == 0 + else "HIGH (answer)" + ) + print(f"\nquestion: {question!r}") + print(f" gate: best_cosine={best_cosine:.4f} fts_hits={fts_hits} -> {verdict}") + # Best chunk per document, in fused rank order. + best_by_doc: dict[str, RetrievedChunk] = {} + for c in chunks: + key = f"{c.document.source}/{c.document.path}" + if key not in best_by_doc: + best_by_doc[key] = c + for i, c in enumerate(list(best_by_doc.values())[: args.top], start=1): + print( + f" {i}. {c.document.source}/{c.document.path} " + f"cosine={c.cosine:.4f} fts={int(c.fts_hit)} fused={c.score:.5f} " + f"({c.document.title})" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import_docs.py b/scripts/import_docs.py index 678fc1c..7f076c8 100644 --- a/scripts/import_docs.py +++ b/scripts/import_docs.py @@ -1,16 +1,20 @@ -"""Import markdown directories into the Brain of Reese knowledge base. +"""Import A9-format directories into the Brain of Reese knowledge base. Examples:: uv run python -m scripts.import_docs # ~/Homelab + ~/Deployments uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable) - uv run python -m scripts.import_docs --prune # also drop deleted files + uv run python -m scripts.import_docs --prune # also drop deleted/out-of-scope files uv run python -m scripts.import_docs --limit 5 # debug: first 5 files only -Only ``*.md`` files are imported; non-content dirs (``.venv``, +Imported formats (PLAN anchor A9, revised): ``md, markdown, txt, yaml, +yml, json, py`` (case-insensitive; narrow with ``BOR_IMPORT_EXTENSIONS``). +Any path with a dot-prefixed component (hidden files/dirs — vendored +caches) is skipped, along with non-content dirs (``.venv``, ``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``, -``build``) are skipped (PLAN anchor A9). Re-runs are cheap: files are -diffed by sha256 and unchanged ones are not re-embedded. +``build``). Re-runs are cheap: files are diffed by sha256 and unchanged +ones are not re-embedded; ``--prune`` also drops documents whose files no +longer match the format filter. """ from __future__ import annotations @@ -31,7 +35,7 @@ DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")] def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog="python -m scripts.import_docs", - description="Import *.md files into the Brain of Reese knowledge base.", + description="Import A9-format files (md/txt/yaml/json/py) into the knowledge base.", ) p.add_argument( "--source", @@ -43,7 +47,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument( "--prune", action="store_true", - help="also delete documents whose files no longer exist", + help="also delete documents whose files no longer exist or match the format filter", ) p.add_argument( "--limit", @@ -74,7 +78,7 @@ def main(argv: list[str] | None = None) -> int: f"import_docs: files={summary.files} added={summary.added} " f"updated={summary.updated} unchanged={summary.unchanged} " f"pruned={summary.pruned} errors={summary.errors} chunks={summary.chunks} " - f"embed_batches={summary.embed_batches}" + f"embed_batches={summary.embed_batches} formats={summary.format_counts()}" ) # Non-zero if any file failed, so cron/CI notice — the rest of the KB # was imported and the failed files are retried on the next run. diff --git a/tests/conftest.py b/tests/conftest.py index 5dbb203..b2644d9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,22 @@ """Shared fixtures for unit + integration tests.""" from __future__ import annotations +import os from collections.abc import Iterator import pytest from fastapi.testclient import TestClient from sqlalchemy.orm import Session -from app.db import SessionLocal, db_available -from app.main import app as fastapi_app +# In-process integration/E2E tests drive the app with *mock* embeddings +# (bag-of-words, cosine ~0.1–0.8), not the live aipi model — so the honesty +# gate is calibrated to the mock's distribution, mirroring tests/e2e/ +# conftest.py. Must be set before ``app.main`` (below) caches settings. +# The production default stays 0.62 (app/config.py, A8 revised). +os.environ.setdefault("BOR_RELEVANCE_THRESHOLD", "0.30") + +from app.db import SessionLocal, db_available # noqa: E402 +from app.main import app as fastapi_app # noqa: E402 @pytest.fixture() diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 800775a..9463256 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -82,6 +82,12 @@ def app_server(mock_llm: int) -> Iterator[str]: if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1" ) + # The E2E mock's token-overlap embeddings have their own score + # distribution (phase 09) — the app under test gets the mock-calibrated + # threshold so every story suite keeps its deterministic gate behavior. + # The production default stays 0.62 (re-tuned against the real + # `embed` model's 0.41–0.84 cosine range, PLAN A8). + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" env.setdefault("BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese") proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "app.main:app", diff --git a/tests/e2e/test_chat_rag.py b/tests/e2e/test_chat_rag.py index 3172486..e4cc328 100644 --- a/tests/e2e/test_chat_rag.py +++ b/tests/e2e/test_chat_rag.py @@ -76,7 +76,7 @@ def test_on_topic_question_streams_grounded_answer( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: summary = _reset_db(mock_llm, seed=True) - assert summary is not None and summary.added == 3 + assert summary is not None and summary.added == 8 # A9 formats page.set_default_timeout(30_000) page.goto(app_url) diff --git a/tests/e2e/test_dark_tech_theme.py b/tests/e2e/test_dark_tech_theme.py index 1cf0776..1aa8cc3 100644 --- a/tests/e2e/test_dark_tech_theme.py +++ b/tests/e2e/test_dark_tech_theme.py @@ -111,7 +111,7 @@ def _seed_kb(mock_port: int) -> ImportSummary: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() summary = _run_in_thread(_import_fixtures(mock_port)) - assert summary is not None and summary.added == 3 + assert summary is not None and summary.added == 8 # A9 formats return summary diff --git a/tests/e2e/test_honest_deflection.py b/tests/e2e/test_honest_deflection.py index 4bf14c8..24bf478 100644 --- a/tests/e2e/test_honest_deflection.py +++ b/tests/e2e/test_honest_deflection.py @@ -84,7 +84,7 @@ def test_off_topic_question_deflects_honestly( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: summary = _reset_db(mock_llm, seed=True) - assert summary is not None and summary.added == 3 + assert summary is not None and summary.added == 8 # A9 formats page.set_default_timeout(30_000) page.goto(app_url) expect(page.locator("#kb-banner")).to_be_hidden() diff --git a/tests/e2e/test_import_documents.py b/tests/e2e/test_import_documents.py index d3abaec..291129b 100644 --- a/tests/e2e/test_import_documents.py +++ b/tests/e2e/test_import_documents.py @@ -31,6 +31,11 @@ EXPECTED_ROWS = ( "homelab/kubernetes.md", "homelab/backups.md", "deployments/new-service.md", + "homelab/container_gitlab/gitlab.md", + "homelab/container_gitlab/gitlab-compose.yaml", + "homelab/networking/static-dns.json", + "homelab/scripts/uptime_probe.py", + "homelab/ssh/ssh_aliases.txt", ) @@ -76,16 +81,21 @@ def test_sources_page_lists_indexed_docs( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: summary = _reset_db(mock_llm, seed=True) - assert summary is not None and summary.added == 3 + # Eight A9-format files are imported; .hidden/junk.md is out of scope + # (A9 revised — hidden path components are never walked). + assert summary is not None and summary.added == 8 + assert summary.formats == {"md": 4, "yaml": 1, "json": 1, "py": 1, "txt": 1} page.goto(f"{app_url}/sources.html") - expect(page.locator("#stat-docs")).to_have_text("3") + expect(page.locator("#stat-docs")).to_have_text("8") expect(page.locator("#stat-chunks")).to_have_text(str(summary.chunks)) expect(page.locator("#stat-last")).not_to_have_text("–") expect(page.locator("#sources-empty")).to_be_hidden() for row_path in EXPECTED_ROWS: expect(page.locator("#docs-tbody tr", has_text=row_path)).to_have_count(1) + # The hidden junk was never indexed (A9 scope). + expect(page.locator("#docs-tbody tr", has_text=".hidden")).to_have_count(0) # The path column carries the full path for hover (ellipsis is visual only). expect(page.locator("#docs-tbody tr", has_text="homelab/kubernetes.md") .get_by_role("cell").nth(1)).to_have_attribute("title", "homelab/kubernetes.md") diff --git a/tests/e2e/test_loading_feedback.py b/tests/e2e/test_loading_feedback.py index b1422ab..5c7022c 100644 --- a/tests/e2e/test_loading_feedback.py +++ b/tests/e2e/test_loading_feedback.py @@ -176,7 +176,7 @@ def test_typing_indicator_during_slow_think( """AC1/AC5: the 3s mock warm-up must show the typing indicator for >=2s before any text appears, then it is gone once the answer lands.""" summary = _reset_db(mock_llm, seed=True) - assert summary is not None and summary.added == 3 + assert summary is not None and summary.added == 8 # A9 formats page.set_default_timeout(30_000) page.goto(app_url) diff --git a/tests/e2e/test_retrieval_quality.py b/tests/e2e/test_retrieval_quality.py new file mode 100644 index 0000000..d3437b2 --- /dev/null +++ b/tests/e2e/test_retrieval_quality.py @@ -0,0 +1,197 @@ +"""Phase 09 E2E (Playwright): retrieval quality — hybrid search end to end. + +Story: ``.agent/user_stories/retrieval-quality.md`` +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_retrieval_quality.py -v --no-cov + +Seeding reuses the real importer against ``tests/fixtures/docs/`` with the +deterministic mock embeddings (same pattern as the earlier story suites). +The four tests map the story's acceptance criteria: + +1. multi-format fixture import — hidden doc excluded, ``/api/docs`` counts +2. "How did I install gitlab?" — grounded (not deflected), gitlab chip, + ``query_log`` row with the gitlab doc in ``sources`` +3. keyword-only question ("kafkabridge") beats the vector ranking — the + FTS-OR gate grounds it end to end despite weak cosine +4. "sourdough" — deflected bubble + ≥2 "Maybe try" chips +""" +from __future__ import annotations + +import asyncio +from pathlib import Path +from threading import Thread +from typing import Any + +import httpx +from playwright.sync_api import Page, expect +from sqlalchemy import select, text + +from app.config import Settings, get_settings +from app.db import SessionLocal +from app.models import QueryLog +from app.rag.importer import ImportSummary, import_sources +from app.rag.llm import LLMClient + +REPO = Path(__file__).resolve().parents[2] +FIXTURES = REPO / "tests" / "fixtures" / "docs" +GITLAB_QUESTION = "How did I install gitlab?" +KEYWORD_QUESTION = "How does kafkabridge work?" +OFF_TOPIC = "sourdough starter" +MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" + + +async def _import_fixtures(mock_port: int) -> ImportSummary: + kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} + settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] + return await import_sources([FIXTURES], LLMClient(settings)) + + +def _run_in_thread(coro: Any) -> Any: + """Run a coroutine on a worker thread (Playwright owns the test loop).""" + box: dict[str, Any] = {} + + def runner() -> None: + try: + box["value"] = asyncio.run(coro) + except BaseException as e: # noqa: BLE001 — re-raised on the test thread + box["error"] = e + + t = Thread(target=runner) + t.start() + t.join() + if "error" in box: + raise box["error"] + return box["value"] + + +def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: + """Truncate the KB (and query log), then optionally re-import fixtures.""" + with SessionLocal() as db: + db.execute(text("TRUNCATE chunks, documents, query_log")) + db.commit() + if not seed: + return None + return _run_in_thread(_import_fixtures(mock_port)) + + +def _ask(page: Page, message: str) -> None: + page.fill("#message-input", message) + page.click("#send-btn") + + +def test_multi_format_import_hidden_doc_excluded( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + """A9 (revised): all seven formats import; hidden (dot) paths never do.""" + summary = _reset_db(mock_llm, seed=True) + assert summary is not None + # Eight A9-format fixture files; .hidden/junk.md must never be walked. + assert summary.added == 8 + assert summary.formats == {"md": 4, "yaml": 1, "json": 1, "py": 1, "txt": 1} + + r = httpx.get(f"{app_url}/api/docs", timeout=10) + assert r.status_code == 200 + docs = r.json()["documents"] + assert len(docs) == 8 + assert all(".hidden" not in d["path"] for d in docs) + assert {d["path"] for d in docs} >= { + "homelab/container_gitlab/gitlab.md", + "homelab/container_gitlab/gitlab-compose.yaml", + "homelab/networking/static-dns.json", + "homelab/scripts/uptime_probe.py", + "homelab/ssh/ssh_aliases.txt", + } + + # The Sources page reflects the same set. + page.goto(f"{app_url}/sources.html") + expect(page.locator("#stat-docs")).to_have_text("8") + expect(page.locator("#docs-tbody tr", has_text=".hidden")).to_have_count(0) + + +def test_gitlab_question_is_grounded_with_gitlab_chip( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + """The ranking problem that motivated this phase: a tool-name question + must land on the tool's own document — not a generic template.""" + _reset_db(mock_llm, seed=True) + page.set_default_timeout(30_000) + page.goto(app_url) + + _ask(page, GITLAB_QUESTION) + expect(page.locator(".msg.user .bubble")).to_contain_text(GITLAB_QUESTION) + + bubble = page.locator(".msg.brain .bubble").first + bubble.wait_for(state="visible", timeout=30_000) + expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) + # Grounded: no deflected bubble at all. + expect(page.locator(".msg.brain.is-deflected")).to_have_count(0) + + # The gitlab document is cited (a chip carrying its path). + chip = page.locator(".msg.brain .source-chip", has_text="container_gitlab/gitlab.md") + expect(chip).to_have_count(1, timeout=30_000) + + # Durable record: not deflected, and the gitlab doc is in sources. + with SessionLocal() as db: + row = db.scalars(select(QueryLog)).one() + assert row.question == GITLAB_QUESTION + assert row.deflected is False + assert "container_gitlab/gitlab.md" in row.sources + + +def test_keyword_only_question_beats_vector_ranking( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + """The FTS-OR gate end to end: "kafkabridge" appears in exactly one + fixture doc (static-dns.json) and the question's cosine overlap is + weak — the lexical branch is what grounds the answer.""" + _reset_db(mock_llm, seed=True) + page.set_default_timeout(30_000) + page.goto(app_url) + + _ask(page, KEYWORD_QUESTION) + bubble = page.locator(".msg.brain .bubble").first + bubble.wait_for(state="visible", timeout=30_000) + expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) + expect(page.locator(".msg.brain.is-deflected")).to_have_count(0) + + # The FTS-matched doc is the TOP source chip (it beats the vector rank). + first_chip = page.locator(".msg.brain .source-chip").first + first_chip.wait_for(state="visible", timeout=30_000) + expect(first_chip).to_contain_text("static-dns.json") + + with SessionLocal() as db: + row = db.scalars(select(QueryLog)).one() + # Weak vector score… + assert row.top_score < get_settings().relevance_threshold + # …but a lexical hit grounded it (the FTS-OR branch). + assert (row.fts_hits or 0) >= 1 + assert row.deflected is False + assert "homelab/networking/static-dns.json" in row.sources + + +def test_off_topic_still_deflects_with_chips( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + """A8 (revised): deflection requires weak cosine AND zero FTS hits. + "sourdough" matches nothing in the KB lexically → honest deflection.""" + _reset_db(mock_llm, seed=True) + page.set_default_timeout(30_000) + page.goto(app_url) + + _ask(page, OFF_TOPIC) + bubble = page.locator(".msg.brain.is-deflected .bubble").first + bubble.wait_for(state="visible", timeout=30_000) + expect(page.locator(".msg.brain.is-deflected")).to_have_count(1) + + chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") + expect(chips.first).to_be_visible(timeout=30_000) + assert chips.count() >= 2, "deflection must offer 2-3 alternative chips" + assert all(c.strip() for c in chips.all_inner_texts()) + + with SessionLocal() as db: + row = db.scalars(select(QueryLog)).one() + assert row.question == OFF_TOPIC + assert row.deflected is True + assert 0.0 < row.top_score < get_settings().relevance_threshold + assert row.fts_hits == 0 # deflection is only reached with zero hits diff --git a/tests/e2e/test_suggestion_chips.py b/tests/e2e/test_suggestion_chips.py index 226b5e8..259e775 100644 --- a/tests/e2e/test_suggestion_chips.py +++ b/tests/e2e/test_suggestion_chips.py @@ -63,7 +63,7 @@ def _seed_kb(mock_port: int) -> ImportSummary: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() summary = _run_in_thread(_import_fixtures(mock_port)) - assert summary is not None and summary.added == 3 + assert summary is not None and summary.added == 8 # A9 formats return summary diff --git a/tests/fixtures/docs/.hidden/junk.md b/tests/fixtures/docs/.hidden/junk.md new file mode 100644 index 0000000..d0f1d4f --- /dev/null +++ b/tests/fixtures/docs/.hidden/junk.md @@ -0,0 +1,5 @@ +# Vendored Junk + +This file lives under a dot-prefixed directory and must **never** be +imported into the knowledge base (A9 hidden-dir skip). It exists so the +retrieval-quality E2E can prove the filter works. diff --git a/tests/fixtures/docs/homelab/container_gitlab/gitlab-compose.yaml b/tests/fixtures/docs/homelab/container_gitlab/gitlab-compose.yaml new file mode 100644 index 0000000..414f6a8 --- /dev/null +++ b/tests/fixtures/docs/homelab/container_gitlab/gitlab-compose.yaml @@ -0,0 +1,24 @@ +# gitlab stack — single container + gitlab-data volume +services: + gitlab: + image: gitlab/gitlab-ce:17.2.1-ce.0 + container_name: gitlab + restart: unless-stopped + hostname: "gitlab.reeseapps.com" + environment: + GITLAB_OMNIBUS_CONFIG: | + external_url 'https://gitlab.reeseapps.com' + gitlab_rails['gitlab_shell_ssh_port'] = 2222 + ports: + - "8929:80" + - "2222:22" + volumes: + - gitlab-data:/var/opt/gitlab + shm_size: "256m" + deploy: + resources: + limits: + memory: 4G + +volumes: + gitlab-data: diff --git a/tests/fixtures/docs/homelab/container_gitlab/gitlab.md b/tests/fixtures/docs/homelab/container_gitlab/gitlab.md new file mode 100644 index 0000000..0a9c462 --- /dev/null +++ b/tests/fixtures/docs/homelab/container_gitlab/gitlab.md @@ -0,0 +1,27 @@ +# Gitlab + +Gitlab CE runs as a single Docker container on the `gitlab` host +(`10.0.1.14`), managed by Ansible (`deployments/gitlab/`). + +## Install + +1. Install Docker and the compose plugin on the host. +2. Create the `gitlab-data` volume: `docker volume create gitlab-data`. +3. Run the stack from `gitlab-compose.yaml`: + `docker compose -f gitlab-compose.yaml up -d` +4. Wait ~2 minutes for the initial gitlab migration to finish. + +## Access + +- Web UI: https://gitlab.reeseapps.com (Traefik routes it to port 8929). +- Root password: `gitlab-root-password` file in the repo (rotated yearly). +- Backup: nightly `gitlab-backup create` at 03:30, copy to BorgBase. + +## Operations + +- Upgrade gitlab: bump the image tag in the compose file, + `docker compose up -d gitlab`, watch the logs for the version banner. +- Logs: `docker logs -f gitlab` or the gitlab admin area → Admin + area → Logs. +- If the container is OOM-killed, raise the memory limit in the compose + file (it needs 4GB free). diff --git a/tests/fixtures/docs/homelab/networking/static-dns.json b/tests/fixtures/docs/homelab/networking/static-dns.json new file mode 100644 index 0000000..29a2619 --- /dev/null +++ b/tests/fixtures/docs/homelab/networking/static-dns.json @@ -0,0 +1,14 @@ +{ + "comment": "Static DNS overrides for the homelab Pi-hole (applied by the ddns updater).", + "hosts": { + "kafkabridge": "10.0.3.7", + "k3s-control": "10.0.1.10", + "gitea": "10.0.2.21", + "ntfy": "10.0.2.30" + }, + "domains": [ + "reeseapps.com", + "homelab.lan" + ], + "expiry_days": 365 +} diff --git a/tests/fixtures/docs/homelab/scripts/uptime_probe.py b/tests/fixtures/docs/homelab/scripts/uptime_probe.py new file mode 100644 index 0000000..0f5c303 --- /dev/null +++ b/tests/fixtures/docs/homelab/scripts/uptime_probe.py @@ -0,0 +1,60 @@ +"""Uptime probe — the homelab healthcheck runner. + +Polls every service listed in ``CHECKS`` every 5 minutes and posts a +failure to the ntfy topic ``homelab-alerts``. +""" +from __future__ import annotations + +import subprocess + +#: (name, health URL) for every long-running service. +CHECKS: list[tuple[str, str]] = [ + ("k3s", "https://10.0.1.10:6443/healthz"), + ("gitea", "https://gitea.reeseapps.com/api/healthz"), + ("ntfy", "https://ntfy.reeseapps.com/health"), + ("gitlab", "https://gitlab.reeseapps.com/-/health_check"), +] + + +def probe(name: str, url: str) -> bool: + """One HTTP check; returns True when the service answered 200.""" + result = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "10", url], + capture_output=True, + text=True, + ) + return result.stdout.strip() == "200" + + +def notify_failure(name: str) -> None: + """Push an alert to ntfy (best effort — alerting must not crash the probe).""" + subprocess.run( + [ + "curl", + "-s", + "-X", + "POST", + "https://ntfy.reeseapps.com/homelab-alerts", + "-H", + "Title: homelab check failed", + "-d", + f"{name} is down", + ], + capture_output=True, + ) + + +def run_round() -> int: + """Probe everything once; returns the number of failing services.""" + failed = 0 + for name, url in CHECKS: + if not probe(name, url): + failed += 1 + notify_failure(name) + return failed + + +if __name__ == "__main__": + import sys + + sys.exit(run_round()) diff --git a/tests/fixtures/docs/homelab/ssh/ssh_aliases.txt b/tests/fixtures/docs/homelab/ssh/ssh_aliases.txt new file mode 100644 index 0000000..c2c7abc --- /dev/null +++ b/tests/fixtures/docs/homelab/ssh/ssh_aliases.txt @@ -0,0 +1,14 @@ +SSH notes for the homelab jump host. + +All admin hosts are reachable through the jump box at 10.0.1.2 +(`ssh reese@jump`). The `~/.ssh/config` aliases: + +k3s — the kubernetes control plane node (10.0.1.10, user talos) +gitlab — the gitlab container host (10.0.1.14) +nuc — the low-power media box (10.0.1.20) + +Keys: ed25519 per host, no passwords. The old RSA key was retired in +2025 and its line removed from authorized_keys on every host. + +Forwarding X11 stays off everywhere; use `ssh -L` port forwards for the +occasional GUI tool instead. diff --git a/tests/integration/test_chat_api.py b/tests/integration/test_chat_api.py index ac83a86..7f31cd2 100644 --- a/tests/integration/test_chat_api.py +++ b/tests/integration/test_chat_api.py @@ -94,7 +94,7 @@ def seeded_kb(db) -> Iterator[FakeRagLLM]: db.commit() llm = FakeRagLLM() summary = asyncio.run(import_sources([FIXTURES], llm, session=db)) - assert summary.added == 3 + assert summary.added == 8 # A9 formats; .hidden/ skipped yield llm db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() @@ -163,12 +163,19 @@ def test_chat_writes_query_log_row(client, db, seeded_kb: FakeRagLLM) -> None: assert row.question == QUESTION assert row.deflected is False total_chunks = db.scalar(select(func.count()).select_from(Chunk)) - assert row.chunk_hits == min(get_settings().top_k_chunks, total_chunks) + # chunk_hits is the fused candidate set (cosine top-N ∪ FTS top-N). + assert 1 <= row.chunk_hits <= total_chunks assert row.top_score > 0.0 # genuine token-overlap cosine, best hit - assert row.top_score >= get_settings().relevance_threshold # why the gate answered assert row.top_score <= 1.0 assert "docs/homelab/kubernetes.md" in row.sources assert row.latency_ms >= 0 + # Why the gate answered (A8 revised): cosine over the threshold OR a + # lexical hit. The mock-calibrated threshold (0.30, see tests/conftest.py) + # makes the cosine branch true here; the FTS branch is covered too — + # "kubernetes" / "cluster" match the doc's tsvector. + thr = get_settings().relevance_threshold + assert row.top_score >= thr or (row.fts_hits or 0) > 0 + assert (row.fts_hits or 0) >= 1 # the lexical branch really fired def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM) -> None: @@ -202,14 +209,46 @@ def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM) assert "Talos Linux" not in system["content"] # full doc content never sent assert "" not in system["content"] - # Durable record: deflected=true + the weak top_score. + # Durable record: deflected=true + the weak top_score. Deflection is + # only reached when the cosine is under the threshold AND no chunk + # FTS-matches the question — so fts_hits must be zero here. row = db.scalars(select(QueryLog)).one() assert row.question == OFF_TOPIC assert row.deflected is True assert 0.0 < row.top_score < get_settings().relevance_threshold + assert row.fts_hits == 0 assert row.chunk_hits >= 1 +def test_keyword_question_grounded_by_lexical_hit_despite_weak_cosine( + client, db, seeded_kb: FakeRagLLM +) -> None: + """Phase 09: a name-your-tool question the vector model barely ranks + ("kafkabridge" only appears in static-dns.json) must still be grounded + via the FTS branch — LOW only fires at weak cosine AND zero hits.""" + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb + try: + _, _, frames = _stream_chat(client, "How does kafkabridge work?") + finally: + fastapi_app.dependency_overrides.clear() + + done = frames[-1] + assert done["type"] == "done" + assert done["deflected"] is False # weak cosine, but a lexical hit + assert done["suggestions"] == [] + sources = done["sources"] + assert sources and sources[0]["path"] == "homelab/networking/static-dns.json" + + (system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1] + assert "HIGH" in system["content"] # grounded prompt + + row = db.scalars(select(QueryLog)).one() + assert row.deflected is False + assert row.top_score < get_settings().relevance_threshold # weak vector score + assert (row.fts_hits or 0) >= 1 # …and it is the FTS hit that grounds it + assert "docs/homelab/networking/static-dns.json" in row.sources + + def test_chat_empty_kb_streams_empty_sources(client, db) -> None: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() diff --git a/tests/integration/test_importer_e2e.py b/tests/integration/test_importer_e2e.py index 46115c7..984eb9b 100644 --- a/tests/integration/test_importer_e2e.py +++ b/tests/integration/test_importer_e2e.py @@ -22,6 +22,11 @@ EXPECTED_DOCS = { ("docs", "homelab/kubernetes.md"), ("docs", "homelab/backups.md"), ("docs", "deployments/new-service.md"), + ("docs", "homelab/container_gitlab/gitlab.md"), + ("docs", "homelab/container_gitlab/gitlab-compose.yaml"), + ("docs", "homelab/networking/static-dns.json"), + ("docs", "homelab/scripts/uptime_probe.py"), + ("docs", "homelab/ssh/ssh_aliases.txt"), } @@ -31,14 +36,27 @@ def test_import_fixtures_end_to_end(client, db) -> None: llm = FakeEmbedder() summary = asyncio.run(import_sources([FIXTURES], llm, session=db)) - assert (summary.files, summary.added, summary.unchanged) == (3, 3, 0) - assert summary.chunks >= 3 + # Eight A9-format files; .hidden/junk.md is out of scope (A9 revised). + assert (summary.files, summary.added, summary.unchanged) == (8, 8, 0) + assert summary.chunks >= 8 + assert summary.formats == {"md": 4, "yaml": 1, "json": 1, "py": 1, "txt": 1} + # PLAN §9 per-format summary line: highest count first, then alpha. + assert summary.format_counts() == "md:4,json:1,py:1,txt:1,yaml:1" docs = db.scalars(select(Document)).all() assert {(d.source, d.path) for d in docs} == EXPECTED_DOCS titles = {d.path: d.title for d in docs} assert titles["homelab/kubernetes.md"] == "Kubernetes Homelab Cluster" assert titles["deployments/new-service.md"] == "Deploying a New Service" + assert titles["homelab/container_gitlab/gitlab.md"] == "Gitlab" + # Non-markdown titles come from the file stem (a leading ``#`` or docstring + # line is a comment there, not a heading). + assert titles["homelab/container_gitlab/gitlab-compose.yaml"] == "gitlab-compose" + assert titles["homelab/scripts/uptime_probe.py"] == "uptime_probe" + assert titles["homelab/networking/static-dns.json"] == "static-dns" + assert titles["homelab/ssh/ssh_aliases.txt"] == "ssh_aliases" + # Hidden junk was never imported. + assert not any(".hidden" in d.path for d in docs) # Full content is stored — that is what the RAG context will be. k8s = next(d for d in docs if d.path == "homelab/kubernetes.md") assert "Talos Linux" in k8s.content and k8s.content_hash @@ -52,13 +70,13 @@ def test_import_fixtures_end_to_end(client, db) -> None: r = client.get("/api/docs") assert r.status_code == 200 body = r.json() - assert len(body["documents"]) == 3 + assert len(body["documents"]) == 8 assert all(d["chunks"] >= 1 for d in body["documents"]) # Idempotent re-run: nothing re-embedded. calls_before = len(llm.calls) s2 = asyncio.run(import_sources([FIXTURES], llm, session=db)) - assert s2.unchanged == 3 and s2.added == 0 + assert s2.unchanged == 8 and s2.added == 0 assert len(llm.calls) == calls_before # unchanged → no embedding requests db.execute(text("TRUNCATE chunks, documents, query_log")) diff --git a/tests/integration/test_migration_0002.py b/tests/integration/test_migration_0002.py new file mode 100644 index 0000000..899d585 --- /dev/null +++ b/tests/integration/test_migration_0002.py @@ -0,0 +1,72 @@ +"""Integration: migration 0002 (hybrid retrieval) schema contract. + +Asserts the state the migration must leave on the live schema: +``chunks.tsv`` as a stored generated tsvector, its GIN index, and the +nullable ``query_log.fts_hits`` column (pre-0002 rows stay NULL, so it +must accept NULL and an int). Requires ``podman compose up -d db``. +""" +from __future__ import annotations + +from sqlalchemy import text + + +def test_migration_0002_schema_contract(db) -> None: + tsv_col = db.execute( + text( + "SELECT count(*) FROM information_schema.columns" + " WHERE table_name = 'chunks' AND column_name = 'tsv'" + ) + ).scalar() + assert tsv_col == 1, "chunks.tsv (stored tsvector) is missing" + + gin = db.execute( + text( + "SELECT count(*) FROM pg_indexes" + " WHERE tablename = 'chunks' AND indexdef ILIKE '%USING gin%'" + " AND indexdef ILIKE '%tsv%'" + ) + ).scalar() + assert gin == 1, "GIN index on chunks.tsv is missing" + + fts = db.execute( + text( + "SELECT is_nullable = 'YES' FROM information_schema.columns" + " WHERE table_name = 'query_log' AND column_name = 'fts_hits'" + ) + ).scalar() + assert fts is True, "query_log.fts_hits must exist and be nullable (pre-0002 rows)" + + +def test_tsv_is_generated_and_lexically_queryable(db) -> None: + """The tsvector is generated from ``content`` (not maintained by app + code) and answers a tsquery — the retrieval path's lexical branch.""" + doc_id = db.execute(text("SELECT gen_random_uuid()")).scalar() + try: + db.execute( + text( + "INSERT INTO documents (id, source, path, full_path, title, content," + " content_hash, indexed_at) VALUES" + " (:id, 'mig_test', 't.md', '/t.md', 'T', 'kafkabridge routes here'," + " repeat('0', 64), now())" + ), + {"id": doc_id}, + ) + db.execute( + text( + "INSERT INTO chunks (id, document_id, position, content) VALUES" + " (gen_random_uuid(), :id, 0, 'kafkabridge routes here')" + ), + {"id": doc_id}, + ) + db.commit() + hit = db.execute( + text( + "SELECT count(*) FROM chunks" + " WHERE tsv @@ to_tsquery('english', 'kafkabridge')" + ) + ).scalar() + assert hit == 1 + finally: + db.execute(text("DELETE FROM chunks WHERE document_id = :id"), {"id": doc_id}) + db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id}) + db.commit() diff --git a/tests/unit/test_chat_gate.py b/tests/unit/test_chat_gate.py index 8e7aa69..64e244a 100644 --- a/tests/unit/test_chat_gate.py +++ b/tests/unit/test_chat_gate.py @@ -45,13 +45,19 @@ def _doc(title: str, content: str) -> Document: ) -def _chunk(doc: Document, score: float) -> RetrievedChunk: +def _chunk( + doc: Document, score: float, cosine: float | None = None, fts_hit: bool = False +) -> RetrievedChunk: + """Fake candidate: *score* is the fused rank score; *cosine* (defaults to + *score*) is the vector-similarity gate input.""" return RetrievedChunk( chunk_id=uuid.uuid4(), position=0, content=doc.content[:32], score=score, document=doc, + cosine=score if cosine is None else cosine, + fts_hit=fts_hit, ) @@ -89,6 +95,68 @@ def test_gate_is_env_tunable_via_settings() -> None: assert chat_api.plan_turn(hits, _settings(threshold=0.25)).deflected is False +# ---------- hybrid gate matrix (A8, revised: cosine AND fts) ---------- + + +def test_gate_weak_cosine_with_fts_hit_still_answers() -> None: + """cosine < threshold but a lexical hit ⇒ HIGH — the FTS-OR branch. + This is the name-your-tool case: "kafkabridge" grounds despite weak + vector overlap.""" + doc = _doc("Static DNS", "DNS_DOC_CONTENT") + plan = chat_api.plan_turn( + [_chunk(doc, 0.02, cosine=0.10, fts_hit=True)], _settings(threshold=0.30) + ) + assert plan.deflected is False + assert plan.top_score == pytest.approx(0.10) # gate input is the cosine + assert plan.fts_hits == 1 + assert "DNS_DOC_CONTENT" in plan.system_prompt + assert plan.suggestions == [] + + +def test_gate_weak_cosine_zero_fts_deflects() -> None: + doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT") + plan = chat_api.plan_turn([_chunk(doc, 0.02, cosine=0.10)], _settings(threshold=0.30)) + assert plan.deflected is True + assert plan.top_score == pytest.approx(0.10) + assert plan.fts_hits == 0 + + +def test_gate_strong_cosine_without_fts_answers() -> None: + doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT") + plan = chat_api.plan_turn([_chunk(doc, 0.90, cosine=0.90)], _settings(threshold=0.30)) + assert plan.deflected is False + assert plan.fts_hits == 0 + + +def test_gate_fts_hits_counts_all_lexical_candidates() -> None: + a = _doc("Alpha", "ALPHA_CONTENT") + b = _doc("Beta", "BETA_CONTENT") + chunks = [ + _chunk(a, 0.03, cosine=0.05, fts_hit=True), + _chunk(a, 0.02, cosine=0.04, fts_hit=True), # same doc, second chunk + _chunk(b, 0.01, cosine=0.03), + ] + plan = chat_api.plan_turn(chunks, _settings(threshold=0.30)) + assert plan.deflected is False + assert plan.fts_hits == 2 # per chunk, not per doc + + +def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None: + """top_score stays the best *vector* cosine even when a lexical-only + chunk (cosine 0.0 by construction) carries the highest fused score.""" + a = _doc("Alpha", "ALPHA_CONTENT") + b = _doc("Beta", "BETA_CONTENT") + chunks = [ + _chunk(a, 0.50, cosine=0.55), # vector rank 1 + _chunk(b, 0.90, cosine=0.0, fts_hit=True), # lexical rank 1 wins the ranking + ] + plan = chat_api.plan_turn(chunks, _settings(threshold=0.30)) + assert plan.top_score == pytest.approx(0.55) + assert plan.deflected is False # 0.55 >= 0.30 anyway + # ranking follows the fused score: Beta's doc is the top source + assert plan.docs[0].title == "Beta" + + def test_gate_zero_chunks_deflects_with_fallback_chips() -> None: plan = chat_api.plan_turn([], _settings()) assert plan.deflected is True @@ -233,6 +301,13 @@ def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _C llm = _CannedLLM() monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_db, lambda: session) monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_llm, lambda: llm) + # These tests assert against a specific gate threshold; keep it stable + # regardless of the production default (0.62) or any .env. + monkeypatch.setattr( + chat_api, + "get_settings", + lambda: Settings(_env_file=None, relevance_threshold=0.30), # pyright: ignore[reportCallIssue] + ) yield session, llm fastapi_app.dependency_overrides.clear() @@ -254,7 +329,7 @@ def _ask(client: TestClient, message: str) -> list[dict[str, Any]]: def _fake_retriever(chunks: list[RetrievedChunk]) -> Any: - def retrieve(_db: Any, _vec: list[float]) -> list[RetrievedChunk]: + def retrieve(_db: Any, _question: str, _vec: list[float]) -> list[RetrievedChunk]: return chunks return retrieve diff --git a/tests/unit/test_chunker.py b/tests/unit/test_chunker.py index d38d621..82e347d 100644 --- a/tests/unit/test_chunker.py +++ b/tests/unit/test_chunker.py @@ -1,11 +1,25 @@ -"""Unit tests: markdown-aware chunker (PLAN §5 policy).""" +"""Unit tests: format-aware chunker (PLAN §5 policy, A9 formats). + +The markdown policy tests are the original contract (md output stays +unchanged); the per-format tests cover the phase-09 dispatcher +(yaml/yml, json, py, txt) and the 1200-char hard cap for every format. +""" from __future__ import annotations from itertools import pairwise import pytest -from app.rag.chunker import HARD_MAX_CHARS, chunk_markdown, extract_title +from app.rag.chunker import ( + HARD_MAX_CHARS, + chunk_document, + chunk_json, + chunk_markdown, + chunk_python, + chunk_text, + chunk_yaml, + extract_title, +) ANCHOR = "## Big" ANCHOR_PREFIX = f"{ANCHOR}\n\n" @@ -157,3 +171,213 @@ def test_extract_title_prefers_h1() -> None: assert extract_title("## not a title\n\nbody") == "" assert extract_title("## sub only", fallback="stem") == "stem" assert extract_title("", fallback="fallback") == "fallback" + + +# --------------------------------------------------------------------------- +# Format dispatcher (chunk_document) — A9 multi-format ingestion +# --------------------------------------------------------------------------- + + +def test_dispatch_by_lowercased_suffix() -> None: + md = "# T\n\n## A\n\nbody\n" + assert chunk_document(md, "notes/Doc.MD") == chunk_markdown(md) + assert chunk_document(md, "notes/doc.MARKDOWN") == chunk_markdown(md) + assert chunk_document("p1\n\np2\n", "x.TXT") == chunk_text("p1\n\np2\n") + assert chunk_document("a: 1\n", "x.YAML") == chunk_yaml("a: 1\n") + assert chunk_document("a: 1\n", "x.Yml") == chunk_yaml("a: 1\n") + assert chunk_document('{"a": 1}', "x.Json") == chunk_json('{"a": 1}') + assert chunk_document("def f(): pass\n", "x.PY") == chunk_python("def f(): pass\n") + + +def test_dispatch_unknown_suffix_falls_back_to_paragraphs() -> None: + assert chunk_document("hello\n\nworld", "data.csv") == ["hello\nworld"] + + +def test_dispatch_ignores_directory_part_of_path() -> None: + assert chunk_document("def f(): pass\n", "a/b/c/script.py") == chunk_python("def f(): pass\n") + + +# --------------------------------------------------------------------------- +# yaml / yml +# --------------------------------------------------------------------------- + + +def test_yaml_splits_on_top_level_keys_and_keeps_key_anchors() -> None: + doc = ( + "# leading comment\n" + "services:\n" + " gitlab:\n" + " image: gitlab/gitlab-ce\n" + " prometheus:\n" + " image: prom/prometheus\n" + "volumes:\n" + " gitlab-data:\n" + ) + chunks = chunk_yaml(doc) + joined = "\n".join(chunks) + for key in ("services:", "volumes:"): + assert key in joined + # Indented keys are NOT block starts — they stay inside their parent block. + assert not any(c.startswith(" gitlab:") for c in chunks) + # The leading comment stays with the first block (preamble). + assert chunks[0].startswith("# leading comment") + assert "gitlab/gitlab-ce" in joined and "prom/prometheus" in joined + + +def test_yaml_document_separators_start_new_blocks() -> None: + a = "site_a: " + "a" * 500 + "\n" + b = "site_b: " + "b" * 500 + "\n" + chunks = chunk_yaml(a + "---\n" + b, target_chars=600, overlap_chars=0) + # Each site is long enough to force its own chunk; the separator must not + # glue them into one over-budget chunk. + assert len(chunks) >= 2 + assert all(len(c) <= 600 for c in chunks) + assert not any("site_a" in c and "site_b" in c for c in chunks) + + +def test_yaml_oversized_key_block_is_split_under_hard_cap() -> None: + doc = "big_list:\n" + (" - " + "x" * 60 + "\n") * 60 # one ~3800-char block + chunks = chunk_yaml(doc) + assert len(chunks) >= 2 + assert all(len(c) <= HARD_MAX_CHARS for c in chunks) + # Overlap re-prints (≤50 chars per split), so only a little content is + # re-stated — the bulk of the block must survive. + assert sum(len(c) for c in chunks) >= len(doc) - 300 + + +def test_yaml_empty_content() -> None: + assert chunk_yaml("") == [] + assert chunk_yaml("\n\n \n") == [] + + +# --------------------------------------------------------------------------- +# json +# --------------------------------------------------------------------------- + + +def test_json_splits_on_top_level_keys_pretty_printed() -> None: + doc = '{"hosts": {"kafkabridge": "10.0.3.7"}, "count": 3}' + chunks = chunk_json(doc, target_chars=45, overlap_chars=0) # force 1 chunk/block + assert len(chunks) == 2 + first, second = chunks + assert '"hosts"' in first and "kafkabridge" in first + assert '"count"' in second + # Pretty-printed (indent=2), not the compact input form. + assert '"kafkabridge": "10.0.3.7"' in first + assert not any('{"hosts"' in c for c in chunks) + + +def test_json_each_key_block_is_self_contained() -> None: + doc = '{"a": "x", "b": "y"}' + chunks = chunk_json(doc, target_chars=13, overlap_chars=0) # force 1 chunk/block + assert [c for c in chunks if '"a"' in c] and [c for c in chunks if '"b"' in c] + assert not any('"a"' in c and '"b"' in c for c in chunks) + + +def test_json_oversized_value_falls_under_hard_cap() -> None: + doc = '{"blob": "' + "z" * 4000 + '"}' + chunks = chunk_json(doc) + assert len(chunks) >= 2 + assert all(len(c) <= HARD_MAX_CHARS for c in chunks) + assert "".join(chunks).count("z") >= 4000 + + +def test_json_top_level_list_is_one_pretty_block() -> None: + chunks = chunk_json("[1, 2, 3]") + assert chunks == ["[\n 1,\n 2,\n 3\n]"] + + +def test_json_unparseable_falls_back_to_paragraph_packing() -> None: + doc = "{broken json\n\nsecond paragraph here\n" + assert chunk_json(doc) == chunk_text(doc) + assert chunk_json("not json at all") == chunk_text("not json at all") + + +# --------------------------------------------------------------------------- +# python +# --------------------------------------------------------------------------- + + +def test_python_splits_on_top_level_defs_and_classes() -> None: + doc = ( + '"""Module doc."""\n' + "import asyncio\n" + "\n" + "CONST = 1\n" + "\n" + "def alpha():\n" + " return 1\n" + "\n" + "class Beta:\n" + " def run(self):\n" + " return 2\n" + ) + chunks = chunk_python(doc, target_chars=60, overlap_chars=0) # force 1 chunk/block + assert len(chunks) == 3 + assert chunks[0].startswith('"""Module doc."""') + assert "CONST = 1" in chunks[0] # preamble ends at the first def/class + assert chunks[1].startswith("def alpha") + assert chunks[2].startswith("class Beta") + assert "def run" in chunks[2] # nested def stays inside the class block + + +def test_python_decorators_stay_with_their_definition() -> None: + doc = "@app.get('/x')\ndef handler():\n return 'x'\n" + chunks = chunk_python(doc) + assert chunks[0].startswith("@app.get") + + +def test_python_oversized_function_falls_back_to_line_packing() -> None: + doc = "def big():\n" + "\n".join(f" val_{i:03d} = {i} # padding" for i in range(80)) + chunks = chunk_python(doc) + assert len(chunks) >= 2 + assert all(len(c) <= HARD_MAX_CHARS for c in chunks) + assert "val_000" in chunks[0] + assert "val_079" in chunks[-1] + assert sum(len(c) for c in chunks) >= len(doc) - 100 + + +def test_python_unparseable_source_falls_back_to_paragraphs() -> None: + src = "def broken(:\n\nstill text\n" + assert chunk_python(src) == chunk_text(src) + + +# --------------------------------------------------------------------------- +# txt +# --------------------------------------------------------------------------- + + +def test_txt_paragraph_packing() -> None: + doc = "alpha\n\nbeta\n\ngamma\n" + chunks = chunk_text(doc) + assert chunks == ["alpha\nbeta\ngamma"] # all three fit the target + + +def test_txt_long_doc_packs_with_overlap() -> None: + doc = "\n\n".join(f"para {i} " + "l" * 300 for i in range(6)) + chunks = chunk_text(doc, target_chars=800, overlap_chars=100) + assert len(chunks) >= 2 + assert all(len(c) <= 800 for c in chunks) + assert all(f"para {i}" in "\n".join(chunks) for i in range(6)) + + +# --------------------------------------------------------------------------- +# Hard cap across every format (aipi ~1024-token request cap) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("content", "path"), + [ + ("# T\n\n" + "word " * 1200, "big.md"), + ("key: " + "v" * 5000 + "\n", "big.yaml"), + ('{"blob": "' + "z" * 5000 + '"}', "big.json"), + ("def f():\n" + " x = 1\n" * 1000, "big.py"), + ("line of text\n\n" * 800, "big.txt"), + ], +) +def test_hard_cap_holds_for_every_format(content: str, path: str) -> None: + chunks = chunk_document(content, path) + assert chunks, "expected at least one chunk" + for c in chunks: + assert len(c) <= HARD_MAX_CHARS, f"{path}: {len(c)} chars" diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 3f052f1..9ca3093 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -5,6 +5,7 @@ import json from typing import Any import pytest +from pydantic import ValidationError from pydantic_settings import SettingsError from app.config import Settings @@ -16,16 +17,28 @@ def _settings(**kwargs: Any) -> Settings: return Settings(**kwargs) # pyright: ignore[reportCallIssue] (kwarg exists at runtime) -def test_defaults_match_locked_decisions() -> None: +def test_defaults_match_locked_decisions(monkeypatch: pytest.MonkeyPatch) -> None: + # The test process sets BOR_RELEVANCE_THRESHOLD=0.30 for the mock- + # calibrated in-process suites (see tests/conftest.py) — the *default* + # under test is the production one. + monkeypatch.delenv("BOR_RELEVANCE_THRESHOLD", raising=False) s = _settings() assert s.llm_chat_model == "turbo" assert s.llm_embed_model == "embed" assert s.embedding_dim == 768 assert s.llm_base_url.endswith("/v1") - assert 0 < s.relevance_threshold < 1 - assert s.top_k_chunks >= 1 + # A8 (revised): the honesty gate input is the best cosine, default 0.62. + assert s.relevance_threshold == 0.62 + # A7 (revised): hybrid retrieval — cosine top-N ∪ FTS top-N, RRF-fused. + assert s.hybrid_vector_candidates >= 1 + assert s.hybrid_lexical_candidates >= 1 + assert s.rrf_k >= 1 assert s.top_n_docs >= 1 assert len(s.suggestions) >= 3 + # A9 (revised): the import scope covers the seven A9 formats. + assert s.import_extension_set == { + ".md", ".markdown", ".txt", ".yaml", ".yml", ".json", ".py" + } def test_env_override(monkeypatch) -> None: @@ -36,6 +49,26 @@ def test_env_override(monkeypatch) -> None: assert s.llm_chat_model == "juggernaut" +def test_import_extensions_env_override_is_a_csv_list(monkeypatch) -> None: + monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,yml") + s = _settings() + assert s.import_extension_set == {".md", ".yml"} + + +def test_import_extensions_rejects_unknown_format(monkeypatch) -> None: + """A typo in the CSV fails at startup (loudly), not by silently + walking zero files.""" + monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,docx") + with pytest.raises(ValidationError, match="docx"): + _settings() + + +def test_import_extensions_rejects_empty(monkeypatch) -> None: + monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", " ") + with pytest.raises(ValidationError): + _settings() + + def test_suggestions_default_is_three_plus_real_questions() -> None: s = _settings() assert len(s.suggestions) >= 3 diff --git a/tests/unit/test_importer.py b/tests/unit/test_importer.py index d84ebc0..4a23cf4 100644 --- a/tests/unit/test_importer.py +++ b/tests/unit/test_importer.py @@ -16,11 +16,15 @@ from app.models import Chunk, Document from app.rag.importer import ( EXCLUDED_DIRS, import_sources, - iter_markdown_files, + iter_importable_files, ) from app.rag.llm import EmbeddingError from tests.fakes import FakeEmbedder +#: A9 default extension set as dotted suffixes (what the importer passes to +#: the walker when no override is configured). +DEFAULT_EXTS = frozenset({".md", ".markdown", ".txt", ".yaml", ".yml", ".json", ".py"}) + class _PoisonEmbedder(FakeEmbedder): """Fails (like a real endpoint) on any text containing 'poison'.""" @@ -50,7 +54,9 @@ def _cleanup_source(db, source: str) -> None: db.commit() -def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None: +def test_iter_importable_files_excludes_noncontent_dirs_and_hidden(tmp_path: Path) -> None: + """Well-known non-content dirs, hidden (dot-) dirs/files, and non-A9 + extensions are all skipped; the A9 formats pass.""" root = tmp_path / "proj" for d in ( "notes/sub", @@ -61,11 +67,20 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None: ".pytest_cache", "dist", "build", + ".esphome/.espressif", # vendored hidden cache — the real A9 case ): (root / d).mkdir(parents=True) files = { + # content that must be found: "README.md": "readme", "notes/sub/deep.md": "deep", + "compose.yaml": "services: {}", + "legacy.YML": "a: b", # case-insensitive suffix + "notes/sub/agent.py": "x = 1", + "config.json": "{}", + "README.txt": "plain", + "notes/sub/deep.markdown": "md2", + # must be skipped: ".venv/lib/junk.md": "junk", "node_modules/x/j.md": "j", ".git/c.md": "g", @@ -73,17 +88,40 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None: ".pytest_cache/c.md": "pc", "dist/d.md": "d", "build/b.md": "b", + ".esphome/.espressif/secret.md": "vendor", + ".secret.md": "hidden file", # dot-prefixed FILE, not just dir + "notes/sub/notes.csv": "a,b", # not an A9 format + "notes/sub/file.md.bak": "x", } for rel, text in files.items(): (root / rel).write_text(text) - (root / "notes" / "not-md.txt").write_text("skip me") - found = {p.relative_to(root).as_posix() for p in iter_markdown_files(root)} - assert found == {"README.md", "notes/sub/deep.md"} + found = {p.relative_to(root).as_posix() for p in iter_importable_files(root, DEFAULT_EXTS)} + assert found == { + "README.md", + "notes/sub/deep.md", + "compose.yaml", + "legacy.YML", + "notes/sub/agent.py", + "config.json", + "README.txt", + "notes/sub/deep.markdown", + } -def test_iter_markdown_files_missing_dir_yields_nothing(tmp_path: Path) -> None: - assert iter_markdown_files(tmp_path / "definitely-missing") == [] +def test_iter_importable_files_missing_dir_yields_nothing(tmp_path: Path) -> None: + assert iter_importable_files(tmp_path / "definitely-missing", DEFAULT_EXTS) == [] + + +def test_iter_importable_files_respects_custom_extension_filter(tmp_path: Path) -> None: + """A narrower filter (e.g. md only) excludes the other A9 formats.""" + root = tmp_path / "filtered" + root.mkdir() + (root / "a.md").write_text("a") + (root / "b.yaml").write_text("a: b") + (root / "c.py").write_text("x = 1") + found = {p.name for p in iter_importable_files(root, frozenset([".md"]))} + assert found == {"a.md"} def test_excluded_dirs_match_plan_anchor_a9() -> None: @@ -277,3 +315,65 @@ def test_chunk_positions_and_titles(db, tmp_path: Path) -> None: assert positions == list(range(len(doc.chunks))) and len(doc.chunks) >= 2 finally: _cleanup_source(db, root.name) + + +def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Path) -> None: + """A9 formats all import; the summary records per-format counts, and + non-markdown titles come from the file stem (a ``#`` line is a comment + there, not a heading).""" + root = tmp_path / "multi" + (root / "svc").mkdir(parents=True) + (root / "guide.md").write_text("# Real Heading\n\nbody\n") + (root / "svc" / "compose.yaml").write_text("# a comment\nservices:\n gitlab: {}\n") + (root / "svc" / "agent.py").write_text("# docstring-like comment\ndef ping():\n return 1\n") + (root / "inventory.json").write_text('{"hosts": []}\n') + (root / "notes.txt").write_text("plain text notes\n") + llm = FakeEmbedder() + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert summary.files == 5 + assert summary.added == 5 + assert summary.formats == {"md": 1, "yaml": 1, "py": 1, "json": 1, "txt": 1} + # PLAN §9 summary line: counts, highest first, ext:name pairs. + assert summary.format_counts() == "json:1,md:1,py:1,txt:1,yaml:1" + + titles = { + d.path: d.title + for d in db.scalars(select(Document).where(Document.source == root.name)).all() + } + assert titles["guide.md"] == "Real Heading" # markdown keeps the H1 + assert titles["svc/compose.yaml"] == "compose" # …comment is not a heading + assert titles["svc/agent.py"] == "agent" + assert titles["inventory.json"] == "inventory" + assert titles["notes.txt"] == "notes" + finally: + _cleanup_source(db, root.name) + + +def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None: + """Previously-imported junk leaves the index: a file that no longer + matches the A9 extension filter is pruned on the next ``prune=True`` run. + This is how dot-dir READMEs imported before the scope fix get cleaned up.""" + root = tmp_path / "cleanup" + root.mkdir() + (root / "keep.md").write_text("# Keep\n\nkept\n") + (root / "junk.md.bak").write_text("old junk that was once imported\n") + llm = FakeEmbedder() + try: + # Seed: import both files as if they were valid at the time. + (root / "junk.md").write_text("old junk that was once imported\n") + (root / "junk.md.bak").unlink() + asyncio.run(import_sources([root], llm, session=db)) + # Rename the junk out of the A9 formats, then prune. + (root / "junk.md").rename(root / "junk.md.bak") + summary = asyncio.run(import_sources([root], llm, session=db, prune=True)) + assert summary.pruned == 1 + assert summary.unchanged == 1 # keep.md survived + assert db.scalar( + select(Document).where(Document.source == root.name, Document.path == "junk.md") + ) is None + assert db.scalar( + select(Document).where(Document.source == root.name, Document.path == "keep.md") + ) is not None + finally: + _cleanup_source(db, root.name) diff --git a/tests/unit/test_retriever.py b/tests/unit/test_retriever.py index ea3503d..5d43fed 100644 --- a/tests/unit/test_retriever.py +++ b/tests/unit/test_retriever.py @@ -8,6 +8,8 @@ from __future__ import annotations import uuid +import pytest + from app.models import Document from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk, select_documents @@ -94,3 +96,102 @@ def test_under_budget_no_truncation() -> None: def test_empty_hits_yield_no_documents() -> None: assert select_documents([], n=2, max_chars=24_000) == [] + + +# --------------------------------------------------------------------------- +# Hybrid retrieval (A7): RRF fusion + lexical tsquery +# --------------------------------------------------------------------------- + +from app.rag.retriever import fuse, lexical_tsquery # noqa: E402 + + +def _rc( + doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0 +) -> RetrievedChunk: + return RetrievedChunk( + chunk_id=uuid.uuid4(), + position=position, + content="x" * 20, + score=0.0, + document=_doc(doc_path, "x" * 20), + cosine=cosine, + fts_hit=fts_hit, + ) + + +def test_lexical_tsquery_tokens_lowercased_deduped_in_order() -> None: + assert lexical_tsquery("How did I Install GITLAB gitlab?") == "how | did | i | install | gitlab" + + +def test_lexical_tsquery_punctuation_and_umlauts_ignored() -> None: + assert lexical_tsquery("c3-r00t? -- what's up!") == "c3 | r00t | what | s | up" + + +def test_lexical_tsquery_pure_symbols_return_none() -> None: + assert lexical_tsquery("??? ???") is None + assert lexical_tsquery("") is None + + +def test_lexical_tsquery_stopwords_left_to_postgres() -> None: + # lexical_tsquery passes raw tokens through; Postgres's to_tsquery + # lexing drops the stopwords (verified against real PG in + # test_retrieve_empty_kb / integration tests). + assert lexical_tsquery("how do i") == "how | do | i" + + +def test_fuse_combines_both_lists_for_double_hits() -> None: + v1 = _rc("a.md", cosine=0.9) + v2 = _rc("b.md", cosine=0.5) + l1 = _rc("a.md", cosine=0.1) # same chunk id -> matched in place + a_id = v1.chunk_id + l1.chunk_id = a_id + out = fuse([v1, v2], [l1], k=60) + by_id = {rc.chunk_id: rc for rc in out} + # a: 1/61 (vector rank 1) + 1/61 (lexical rank 1); b: 1/62 only. + assert by_id[a_id].score == pytest.approx(2 / 61) + assert by_id[a_id].fts_hit is True + assert by_id[v2.chunk_id].score == pytest.approx(1 / 62) + assert by_id[v2.chunk_id].fts_hit is False + assert [rc.chunk_id for rc in out] == [a_id, v2.chunk_id] + + +def test_fuse_lexical_only_chunks_enter_with_zero_cosine() -> None: + vector = [_rc("a.md", cosine=0.8)] + lexical = [_rc("b.md", cosine=0.0, fts_hit=True)] + out = fuse(vector, lexical, k=60) + assert len(out) == 2 + b = next(rc for rc in out if rc.document.path == "b.md") + assert b.cosine == 0.0 + assert b.fts_hit is True + # Still ranked by its (only) RRF term. + assert b.score == pytest.approx(1 / 61) + + +def test_fuse_orders_by_score_then_cosine_then_path() -> None: + # Two chunks share an RRF score (both rank 1 in different lists): + # the higher-cosine one must sort first. + hi = _rc("z.md", cosine=0.9) + lo = _rc("a.md", cosine=0.2) + out = fuse([hi], [lo], k=60) + assert [rc.document.path for rc in out] == ["z.md", "a.md"] + # Equal score AND cosine -> path order. + p1 = _rc("b.md", cosine=0.5) + p2 = _rc("a.md", cosine=0.5) + out = fuse([p1], [p2], k=60) + assert [rc.document.path for rc in out] == ["a.md", "b.md"] + # Equal score, cosine, path -> position order. + s1 = _rc("a.md", cosine=0.5, position=1) + s2 = _rc("a.md", cosine=0.5, position=0) + out = fuse([s1], [s2], k=60) + assert [rc.position for rc in out] == [0, 1] + + +def test_fuse_rejects_nonpositive_k() -> None: + with pytest.raises(ValueError): + fuse([], [], k=0) + with pytest.raises(ValueError): + fuse([], [], k=-1) + + +def test_fuse_empty_lists() -> None: + assert fuse([], [], k=60) == []