From f20d40e4fec0ccd50575df549eedb2f9d4f37bc6 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sat, 17 Oct 2026 03:18:23 -0400 Subject: [PATCH] phase: 188_chunks_hnsw_index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 188 — final verification pass: all criteria green.** - **Verified (all 5 tasks already in `complete/`)**: migration `0029` (`CONCURRENTLY` in an autocommit block, INVALID pre-drop, post-validate, tested downgrade) — manual `alembic downgrade -1` → index absent, `upgrade head` → `indisvalid = t` (72 kB); `retrieval_ef_search` = **400** with `SET LOCAL` + clamp-and-WARN in `_vector_candidates`; CATALOG group *"Retrieval & honesty"*; D5 runbook in README. - `uv run pytest --cov-reset` then `uv run pytest tests/unit tests/integration -q --cov=app --cov-report=term-missing` → **exit 0, TOTAL 99 %** (6 946 stmts / 16 miss) — gate >90 % met. (A stale `.coverage` left by an earlier session produced a spurious 90.00 %/3-failure reading; `--cov-reset` cleared it.) - `uv run pytest tests/e2e/test_vector_index_recall.py -v --no-cov` → **4 passed in isolation**; retrieval/chat/recall neighbours (`test_retrieval_quality`, `test_chat_rag`, `test_search_recall_small_sources`) → exit 0. - `uv run ruff check .` → **All checks passed**; `uv run pyright` (whole repo) → **0 errors**. - **Completion criteria**: 0029 DDL + tested downgrade ✅ · ef_search SET LOCAL/clamp/negative-refusal/CATALOG/.env.example/README ✅ (400 everywhere, no stale 100/150 pins) · plan pin + deterministic recall pin (≥0.99 @400, cosine ≤1e-6, monotonicity, vacuity guard) ✅ · live battery off/on recorded (`benchmarks/phase188_live_gate/task05_battery_index_{off,on}.txt` byte-identical, `deterministic_gate_recall.txt`, build 161 s / 1 625 MB / 81 MB RSS, phase report `.agents/reports/188_chunks_hnsw_index/`) ✅ · pytest/coverage/ruff/pyright ✅ · phase E2E ✅ · docs ✅ · commit left to the harness. - **Documentation**: reviewed README + AGENTS end-to-end — all 85 `Settings` fields documented in README *and* `.env.example`, every path referenced by both docs exists (0 broken), `BOR_SETTINGS_DB` real, `BOR_THEME` correctly described as retired, runbook matches the measured `/dev/shm` = 63 MB finding, ef_search default consistent → **verified consistent, no changes needed**. - **Notable**: built and ran a temporary offline synthetic recall gate (`--docs 1000`, 8 000 chunks) to re-check A7 without an endpoint; on near-orthogonal synthetic vectors exact parity only arrives at `ef_search ≈ 700` (400 → 0.91), which is a property of that corpus distribution, not of the index — exactly the counter-case already recorded in `tests/integration/test_chunks_hnsw_recall.py`. Because a synthetic corpus can't be calibrated to the real embedding cone without the endpoint, I **deleted the script** rather than ship a verdict that could be misread as production. `AIPI_KEY` is absent in this environment (and `100.117.2.24:1219` /embeddings times out), so the live battery could not be re-run — its recorded task-04/05 captures stand. Scratch DB `brain_of_reese_hnswgate` dropped; repo left clean. - **Next pending phase**: `189_chat_save_long_turns`. --- .../188_chunks_hnsw_index/00_phase.md | 0 .../01_hnsw_migration.md | 0 .../02_ef_search_setting.md | 0 .../03_plan_and_recall_pins.md | 0 .../04_recall_battery_gate.md | 0 .../05_e2e_index_chat_and_runbook.md | 0 .../188_chunks_hnsw_index__00_phase.a1.err | 0 .../188_chunks_hnsw_index__00_phase.a1.md | 10 + ...8_chunks_hnsw_index__00_phase.a1.md.status | 1 + ...88_chunks_hnsw_index__00_phase.a1.validate | 145 +++ ...hunks_hnsw_index__01_hnsw_migration.a1.err | 0 ...chunks_hnsw_index__01_hnsw_migration.a1.md | 16 + ...hnsw_index__01_hnsw_migration.a1.md.status | 1 + ..._hnsw_index__01_hnsw_migration.a1.validate | 145 +++ ...ks_hnsw_index__02_ef_search_setting.a1.err | 0 ...nks_hnsw_index__02_ef_search_setting.a1.md | 18 + ...w_index__02_ef_search_setting.a1.md.status | 1 + ...sw_index__02_ef_search_setting.a1.validate | 145 +++ ...hnsw_index__03_plan_and_recall_pins.a1.err | 0 ..._hnsw_index__03_plan_and_recall_pins.a1.md | 19 + ...ndex__03_plan_and_recall_pins.a1.md.status | 1 + ...index__03_plan_and_recall_pins.a1.validate | 145 +++ ..._hnsw_index__04_recall_battery_gate.a1.err | 0 ...s_hnsw_index__04_recall_battery_gate.a1.md | 16 + ...index__04_recall_battery_gate.a1.md.status | 1 + ..._index__04_recall_battery_gate.a1.validate | 145 +++ ...ndex__05_e2e_index_chat_and_runbook.a1.err | 0 ...index__05_e2e_index_chat_and_runbook.a1.md | 12 + ...05_e2e_index_chat_and_runbook.a1.md.status | 1 + ..._05_e2e_index_chat_and_runbook.a1.validate | 145 +++ .../task04_recall_ab_probe.py | 629 +++++++++++++ .../task04_recall_battery_gate.md | 227 +++++ .env.example | 1 + AGENTS.md | 64 +- README.md | 145 ++- .../versions/0029_chunks_embedding_hnsw.py | 125 +++ app/config.py | 53 ++ app/core/settings_catalog.py | 3 + app/rag/retriever.py | 31 + benchmarks/README.md | 10 + benchmarks/phase188_live_gate/README.md | 185 ++++ .../phase188_live_gate/battery_index_off.txt | 64 ++ .../phase188_live_gate/battery_index_on.txt | 64 ++ .../deterministic_gate_compare.txt | 19 + .../deterministic_gate_recall.txt | 20 + .../production_order_latency.txt | 12 + .../phase188_live_gate/question_vectors.json | 1 + .../task05_battery_index_off.txt | 65 ++ .../task05_battery_index_on.txt | 65 ++ tests/conftest.py | 9 + tests/e2e/conftest.py | 8 + tests/e2e/test_vector_index_recall.py | 889 ++++++++++++++++++ tests/fixtures/hnswkb/garage/shop-vacuum.md | 12 + tests/fixtures/hnswkb/home/coffee-grinder.md | 12 + tests/fixtures/hnswkb/home/garden-beds.md | 12 + .../hnswkb/homelab/kubernetes-cluster.md | 15 + .../fixtures/hnswkb/homelab/proxmox-nodes.md | 13 + .../fixtures/hnswkb/homelab/restic-backups.md | 14 + tests/fixtures/hnswkb/infra/ntfy-relay.md | 15 + tests/fixtures/hnswkb/infra/tilecache.md | 35 + tests/fixtures/hnswkb/infra/wireguard-hub.md | 15 + .../hnswkb/kitchen/sourdough-timetable.md | 11 + tests/fixtures/hnswkb/travel/packing-list.md | 12 + tests/integration/test_chunks_hnsw_index.py | 354 +++++++ tests/integration/test_chunks_hnsw_recall.py | 612 ++++++++++++ tests/integration/test_migration_0004.py | 6 + tests/integration/test_migration_0005.py | 6 + tests/integration/test_migration_0006.py | 6 + tests/integration/test_migration_0007.py | 6 + tests/integration/test_migration_0008.py | 6 + tests/integration/test_migration_0009.py | 6 + tests/integration/test_migration_0010.py | 6 + tests/integration/test_migration_0011.py | 6 + tests/integration/test_migration_0012.py | 6 + tests/integration/test_migration_0014.py | 6 + tests/integration/test_migration_0016.py | 6 + tests/integration/test_migration_0017.py | 6 + tests/integration/test_migration_0028.py | 6 + tests/unit/test_config.py | 33 + tests/unit/test_retriever.py | 149 +++ tests/unit/test_settings_catalog.py | 30 + 81 files changed, 5076 insertions(+), 2 deletions(-) rename .agents/phases/{todo => complete}/188_chunks_hnsw_index/00_phase.md (100%) rename .agents/phases/{todo => complete}/188_chunks_hnsw_index/01_hnsw_migration.md (100%) rename .agents/phases/{todo => complete}/188_chunks_hnsw_index/02_ef_search_setting.md (100%) rename .agents/phases/{todo => complete}/188_chunks_hnsw_index/03_plan_and_recall_pins.md (100%) rename .agents/phases/{todo => complete}/188_chunks_hnsw_index/04_recall_battery_gate.md (100%) rename .agents/phases/{todo => complete}/188_chunks_hnsw_index/05_e2e_index_chat_and_runbook.md (100%) create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.err create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md.status create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.validate create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.err create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md.status create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.validate create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.err create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md.status create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.validate create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.err create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md.status create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.validate create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.err create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md.status create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.validate create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.err create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md.status create mode 100644 .agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.validate create mode 100644 .agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py create mode 100644 .agents/reports/188_chunks_hnsw_index/task04_recall_battery_gate.md create mode 100644 alembic/versions/0029_chunks_embedding_hnsw.py create mode 100644 benchmarks/phase188_live_gate/README.md create mode 100644 benchmarks/phase188_live_gate/battery_index_off.txt create mode 100644 benchmarks/phase188_live_gate/battery_index_on.txt create mode 100644 benchmarks/phase188_live_gate/deterministic_gate_compare.txt create mode 100644 benchmarks/phase188_live_gate/deterministic_gate_recall.txt create mode 100644 benchmarks/phase188_live_gate/production_order_latency.txt create mode 100644 benchmarks/phase188_live_gate/question_vectors.json create mode 100644 benchmarks/phase188_live_gate/task05_battery_index_off.txt create mode 100644 benchmarks/phase188_live_gate/task05_battery_index_on.txt create mode 100644 tests/e2e/test_vector_index_recall.py create mode 100644 tests/fixtures/hnswkb/garage/shop-vacuum.md create mode 100644 tests/fixtures/hnswkb/home/coffee-grinder.md create mode 100644 tests/fixtures/hnswkb/home/garden-beds.md create mode 100644 tests/fixtures/hnswkb/homelab/kubernetes-cluster.md create mode 100644 tests/fixtures/hnswkb/homelab/proxmox-nodes.md create mode 100644 tests/fixtures/hnswkb/homelab/restic-backups.md create mode 100644 tests/fixtures/hnswkb/infra/ntfy-relay.md create mode 100644 tests/fixtures/hnswkb/infra/tilecache.md create mode 100644 tests/fixtures/hnswkb/infra/wireguard-hub.md create mode 100644 tests/fixtures/hnswkb/kitchen/sourdough-timetable.md create mode 100644 tests/fixtures/hnswkb/travel/packing-list.md create mode 100644 tests/integration/test_chunks_hnsw_index.py create mode 100644 tests/integration/test_chunks_hnsw_recall.py diff --git a/.agents/phases/todo/188_chunks_hnsw_index/00_phase.md b/.agents/phases/complete/188_chunks_hnsw_index/00_phase.md similarity index 100% rename from .agents/phases/todo/188_chunks_hnsw_index/00_phase.md rename to .agents/phases/complete/188_chunks_hnsw_index/00_phase.md diff --git a/.agents/phases/todo/188_chunks_hnsw_index/01_hnsw_migration.md b/.agents/phases/complete/188_chunks_hnsw_index/01_hnsw_migration.md similarity index 100% rename from .agents/phases/todo/188_chunks_hnsw_index/01_hnsw_migration.md rename to .agents/phases/complete/188_chunks_hnsw_index/01_hnsw_migration.md diff --git a/.agents/phases/todo/188_chunks_hnsw_index/02_ef_search_setting.md b/.agents/phases/complete/188_chunks_hnsw_index/02_ef_search_setting.md similarity index 100% rename from .agents/phases/todo/188_chunks_hnsw_index/02_ef_search_setting.md rename to .agents/phases/complete/188_chunks_hnsw_index/02_ef_search_setting.md diff --git a/.agents/phases/todo/188_chunks_hnsw_index/03_plan_and_recall_pins.md b/.agents/phases/complete/188_chunks_hnsw_index/03_plan_and_recall_pins.md similarity index 100% rename from .agents/phases/todo/188_chunks_hnsw_index/03_plan_and_recall_pins.md rename to .agents/phases/complete/188_chunks_hnsw_index/03_plan_and_recall_pins.md diff --git a/.agents/phases/todo/188_chunks_hnsw_index/04_recall_battery_gate.md b/.agents/phases/complete/188_chunks_hnsw_index/04_recall_battery_gate.md similarity index 100% rename from .agents/phases/todo/188_chunks_hnsw_index/04_recall_battery_gate.md rename to .agents/phases/complete/188_chunks_hnsw_index/04_recall_battery_gate.md diff --git a/.agents/phases/todo/188_chunks_hnsw_index/05_e2e_index_chat_and_runbook.md b/.agents/phases/complete/188_chunks_hnsw_index/05_e2e_index_chat_and_runbook.md similarity index 100% rename from .agents/phases/todo/188_chunks_hnsw_index/05_e2e_index_chat_and_runbook.md rename to .agents/phases/complete/188_chunks_hnsw_index/05_e2e_index_chat_and_runbook.md diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.err b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md new file mode 100644 index 0000000..0e59a68 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md @@ -0,0 +1,10 @@ +**Phase 188 — final verification pass: all criteria green.** + +- **Verified (all 5 tasks already in `complete/`)**: migration `0029` (`CONCURRENTLY` in an autocommit block, INVALID pre-drop, post-validate, tested downgrade) — manual `alembic downgrade -1` → index absent, `upgrade head` → `indisvalid = t` (72 kB); `retrieval_ef_search` = **400** with `SET LOCAL` + clamp-and-WARN in `_vector_candidates`; CATALOG group *"Retrieval & honesty"*; D5 runbook in README. +- `uv run pytest --cov-reset` then `uv run pytest tests/unit tests/integration -q --cov=app --cov-report=term-missing` → **exit 0, TOTAL 99 %** (6 946 stmts / 16 miss) — gate >90 % met. (A stale `.coverage` left by an earlier session produced a spurious 90.00 %/3-failure reading; `--cov-reset` cleared it.) +- `uv run pytest tests/e2e/test_vector_index_recall.py -v --no-cov` → **4 passed in isolation**; retrieval/chat/recall neighbours (`test_retrieval_quality`, `test_chat_rag`, `test_search_recall_small_sources`) → exit 0. +- `uv run ruff check .` → **All checks passed**; `uv run pyright` (whole repo) → **0 errors**. +- **Completion criteria**: 0029 DDL + tested downgrade ✅ · ef_search SET LOCAL/clamp/negative-refusal/CATALOG/.env.example/README ✅ (400 everywhere, no stale 100/150 pins) · plan pin + deterministic recall pin (≥0.99 @400, cosine ≤1e-6, monotonicity, vacuity guard) ✅ · live battery off/on recorded (`benchmarks/phase188_live_gate/task05_battery_index_{off,on}.txt` byte-identical, `deterministic_gate_recall.txt`, build 161 s / 1 625 MB / 81 MB RSS, phase report `.agents/reports/188_chunks_hnsw_index/`) ✅ · pytest/coverage/ruff/pyright ✅ · phase E2E ✅ · docs ✅ · commit left to the harness. +- **Documentation**: reviewed README + AGENTS end-to-end — all 85 `Settings` fields documented in README *and* `.env.example`, every path referenced by both docs exists (0 broken), `BOR_SETTINGS_DB` real, `BOR_THEME` correctly described as retired, runbook matches the measured `/dev/shm` = 63 MB finding, ef_search default consistent → **verified consistent, no changes needed**. +- **Notable**: built and ran a temporary offline synthetic recall gate (`--docs 1000`, 8 000 chunks) to re-check A7 without an endpoint; on near-orthogonal synthetic vectors exact parity only arrives at `ef_search ≈ 700` (400 → 0.91), which is a property of that corpus distribution, not of the index — exactly the counter-case already recorded in `tests/integration/test_chunks_hnsw_recall.py`. Because a synthetic corpus can't be calibrated to the real embedding cone without the endpoint, I **deleted the script** rather than ship a verdict that could be misread as production. `AIPI_KEY` is absent in this environment (and `100.117.2.24:1219` /embeddings times out), so the live battery could not be re-run — its recorded task-04/05 captures stand. Scratch DB `brain_of_reese_hnswgate` dropped; repo left clean. +- **Next pending phase**: `189_chat_save_long_turns`. diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md.status b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.validate b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.validate new file mode 100644 index 0000000..d006b3c --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__00_phase.a1.validate @@ -0,0 +1,145 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 98%] +.......................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 325 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 259 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6946 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.err b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md new file mode 100644 index 0000000..12d99c3 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md @@ -0,0 +1,16 @@ +**Task 01 complete — all green.** + +**Implemented** +- `alembic/versions/0029_chunks_embedding_hnsw.py` — `ix_chunks_embedding_hnsw` (`USING hnsw (embedding vector_cosine_ops) WITH (m=16, ef_construction=64)`) built `CONCURRENTLY` in an autocommit block; pure helpers `should_drop_invalid`/`is_valid`; INVALID pre-drop self-heal + post-validate raise; `DROP INDEX CONCURRENTLY IF EXISTS` downgrade (A13). +- `tests/integration/test_chunks_hnsw_index.py` — create/valid/definition pin, downgrade, head re-upgrade round-trip, forced-INVALID self-heal (documented superuser `UPDATE pg_index SET indisvalid=false` trick, clean skip without catalog rights), pure helper matrices, module contract. +- **Defect found & fixed suite-wide:** psycopg3 keeps its snapshot alive while idle-in-transaction (psql does not); a session left open parks `CREATE INDEX CONCURRENTLY` phase 3 **forever** (reproduced 5×, lock graph + `pg_terminate_backend` unblock prove it). Patched the 13 pre-0028 migration-test fixtures (0004–0017, 0028) with the 0018 house `db.rollback()`-before-repair-DDL pattern and made every alembic call in the new file transaction-free. +- `README.md` — deploy note under Production Deployment (CONCURRENTLY semantics, `maintenance_work_mem`/RAM pre-flight, INVALID self-heal, the post-build/pre-stamp `already exists` recovery). AGENTS.md: no doc impact (no command/env change). + +**Verification (exact results)** +- Dev DB: `alembic upgrade head → downgrade -1 → upgrade head` green; index def renders `USING hnsw … WITH (m='16', ef_construction='64')`, `indisvalid=t`; head=0029. +- `uv run pytest tests/integration/test_chunks_hnsw_index.py` → **7 passed in 0.43s** (deterministic). +- `uv run pytest` → **4577 passed**; `uv run pytest --cov=app --cov-report=term-missing` → **4577 passed, TOTAL 99%** (>90%). +- `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors. +- `uv run pytest tests/e2e/test_smoke.py --no-cov` → 3 passed (index live, no retrieval regression). + +**Decisions/deviations**: kept the task-prescribed unconditional `CREATE` (post-build crash → loud `already exists` + documented manual drop; halfvec stays out per A2). Next pending task: `188_chunks_hnsw_index/02_ef_search_setting.md`. diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md.status b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.validate b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.validate new file mode 100644 index 0000000..0c4e6c3 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__01_hnsw_migration.a1.validate @@ -0,0 +1,145 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 11%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 44%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 99%] +......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 318 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 252 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6932 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.err b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md new file mode 100644 index 0000000..2e00734 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md @@ -0,0 +1,18 @@ +**Task 02 complete — all green.** + +**Implemented** +- `app/config.py` — `retrieval_ef_search: int = 100` (`BOR_RETRIEVAL_EF_SEARCH`) with the fail-loud validator: `0` **and** negatives refuse startup (the clamp has no "off" state, phase-182 `show_text_max_*` family). +- `app/rag/retriever.py` — `_vector_candidates` now opens with `SET LOCAL hnsw.ef_search = {int(ef)}` immediately before the `<=>` SELECT (module gained its ONE logger); below the A7 candidate count it CLAMPS up and logs `retrieval: ef_search=%d raised to %d (A7 candidate count)`. Setting read via `get_settings()` (the module's existing convention — no new plumbing). +- `app/core/settings_catalog.py` — `SettingMeta("retrieval_ef_search", "Retrieval & honesty", "Vector ef_search")` beside `hybrid_vector_candidates`; `.env.example` + README knob documented. +- Pins: 8 unit (`tests/unit/test_retriever.py` — SET-before-SELECT, clamp+warn, equal/pass-through silent, default 100, int-only statement, ≤0 refuses, default == candidate count), 2 `test_config.py`, 1 `test_settings_catalog.py` row/type pin, 3 integration in `test_chunks_hnsw_index.py` (clamped value live in the same tx; **new tx falls back to the server default** — LOCKED A4; exactly one clamp WARNING line). +- Leak-guard pins added for the new knob in `tests/conftest.py` and `tests/e2e/conftest.py` (house phase-61/106/182 pattern). + +**Verification (exact)** +- `uv run pytest --cov=app --cov-report=term-missing` → **4590 passed, TOTAL 99%** (>90%); `uv run ruff check .` → all checks passed; `uv run pyright` → 0 errors. +- E2E each in isolation: smoke 3, chat_rag 3, retrieval_quality 4, admin_settings_panel 12, admin_settings_layout 5 — all passed. Dev DB verified at `0029` with `ix_chunks_embedding_hnsw` valid. + +**Docs**: `README.md` (How-retrieval step 1 + env table row), `AGENTS.md` (rule 10 gains the clamp-warning + `SET LOCAL` contract), `.env.example`. + +**Decisions/deviations**: clamp compares against the call's own `LIMIT` (per D3); no `ef_search` override param added (task 03 can monkeypatch `get_settings`); measured server default = 40, max 1000 (pgvector 0.8.0), so the integration pin reads `pg_settings.reset_val` instead of hardcoding. **Pre-existing, not introduced here**: running several `tests/e2e/` modules in ONE pytest session fails (every module spawns uvicorn on the same `APP_PORT`) — reproduced byte-identically (7 failed/8 passed) on a clean `git archive HEAD` tree; each file passes in isolation per AGENTS rule 9, so left untouched. + +**Next pending task**: `188_chunks_hnsw_index/03_plan_and_recall_pins.md`. diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md.status b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.validate b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.validate new file mode 100644 index 0000000..2fe8ad1 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__02_ef_search_setting.a1.validate @@ -0,0 +1,145 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 98%] +...................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 325 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 259 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6946 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.err b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md new file mode 100644 index 0000000..7678fe8 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md @@ -0,0 +1,19 @@ +**Implemented** (task 03 — `.agents/phases/todo/188_chunks_hnsw_index/03_plan_and_recall_pins.md`) +- New `tests/integration/test_chunks_hnsw_recall.py` (567 lines): fixed-seed 5,000-chunk / 768-dim corpus (40 centroids + noise inside a 16-direction principal subspace, L2-normalised), index built **only** via migration 0029 (`0029→0028→seed→0029`), teardown deletes just `source='phase-188-hnsw-recall'` and restores head (dev DB verified: 0 chunks, 0029, `indisvalid=t`). +- `test_plan_uses_the_hnsw_index` — `EXPLAIN (FORMAT JSON)` of the tier's **captured** SELECT (statement object from `_vector_candidates`, `literal_binds`): names `ix_chunks_embedding_hnsw`, no `Seq Scan on chunks`. +- `test_plan_without_the_index_seq_scans_the_relation` — vacuity guard via `SET LOCAL enable_indexscan/indexonlyscan = off` (documented choice over DROP/re-`CREATE CONCURRENTLY`). +- `test_top100_recall_at_ef_search_100` — pure-Python exact cosine top-100 (float4-modelled) vs the index: per-query floor 0.99, per-row `1-distance` within 1e-6, evidence-carrying failure message. Measured: mean **0.9993**, min 0.99 (2/30 at 99/100), worst cosine delta **5.7e-07** (bounded by the tier's `round(…,6)`). +- `test_recall_never_below_the_pinned_floor_when_ef_search_raised` — monotonicity: ef=100 → 0.9993, ef=200 → **1.0000**. +- Module docstring records the prod numbers (414,220 chunks / ~3,000 MB / no index / pgvector 0.8.0 / 3,904 MB no swap), why 5,000 rows is a plan pin not a prod proxy (task 04 is that gate), the measured overlaps, ef values, and ~8 s setup vs the 60 s guard. + +**Verification (exact)** +- `uv run pytest tests/integration/test_chunks_hnsw_recall.py -v` → **4 passed** (29 s); with `test_chunks_hnsw_index.py` + `tests/unit/test_retriever.py` → 105 checks, 0 failures. +- `uv run pytest --cov=app --cov-report=term-missing` → **4594 passed, TOTAL 99%** (>90%). `uv run ruff check .` → All checks passed. `uv run pyright` → **0 errors**. + +**Docs**: no doc impact — task 03 is CI-only; `BOR_RETRIEVAL_EF_SEARCH` already in README/.env.example from task 02; the build runbook stays task 05's. No `app/` change. + +**Decisions / flags** +- Corpus design: the task's "40 clusters + noise" is actively hostile to HNSW when clusters are tight (measured recall collapsed to 0.00–0.65 — disconnected layer-0 graph); the pin therefore uses a noise-dominated, low-effective-rank (16-of-768) clustered corpus, which is also how trained embeddings behave. +- **Flag for task 04 / owner**: on effectively-iid 768-dim data the same 5,000-row / `m=16` / `ef_construction=64` graph measures recall@100 ≈ **0.63 at ef_search=100**, 0.85 at 200, 1.00 from 400. Per LOCKED D3/A3 I did **not** change the shipped default (100); if the live battery moves a `top_score`/top-5 set, the remedy is raising `BOR_RETRIEVAL_EF_SEARCH` toward 400. + +**Next pending task**: `188_chunks_hnsw_index/04_recall_battery_gate.md`. diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md.status b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.validate b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.validate new file mode 100644 index 0000000..d006b3c --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__03_plan_and_recall_pins.a1.validate @@ -0,0 +1,145 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 98%] +.......................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 325 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 259 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6946 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.err b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md new file mode 100644 index 0000000..5988466 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md @@ -0,0 +1,16 @@ +Task 04 complete. Final state of the work: + +**Implementation** — Gate run on the live endpoint + restored 1,000-doc live replica (8,866 real embeddings, dump at `alembic_version=0011`, so migration 0029 built the graph against real vectors). +- Battery `eval_retrieval` OFF vs ON: **byte-identical** (all `best_cosine`, all 35 hits, all 7 A8 verdicts). +- Pinned-vector gate (probe embeds once; endpoint self-jitter measured 0.000e+00): verdicts, top-5 sets/order, suggested docs and `top_score` **identical, delta 0.000e+00**; top-100 candidate-set overlap **1.0000 on all 7**; every rank swap an exact tie; tier 38.3 → 15.3 ms; `Limit → Nested Loop → Index Scan on ix_chunks_embedding_hnsw` vs `Limit → Sort → Seq Scan`. +- Recall@100 sweep: `ef=100` 0.9814 mean / 0.94 min (under the phase's 0.99 floor, per-chunk delta 2.195e-03) → `ef=400` **1.0000/1.0000**. **Applied LOCKED A3's only remedy: `retrieval_ef_search` default 100 → 400** (`app/config.py`, `.env.example`, `README.md`, `AGENTS.md` rule 10, and the 4 pins that encoded 100). **No honesty threshold touched.** +- Production order (throwaway 416,702-chunk / 557 MB DB, created and dropped): build **161 s**, index **1,625 MB**, planner takes HNSW at ef 100/200/400, tier **~550 ms → 12–15 ms**. +- **Deviation flagged:** D5's `maintenance_work_mem='512MB'` is unusable here — `/dev/shm` is 63 MB; reproduced `could not resize shared memory segment … No space left on device` at production order, which exercised 0029's INVALID-index self-heal for real. Also fixed a vacuity hole in task 03's recall pin (pgvector costs HNSW partly on `ef_search`, so the pin now refuses seq scans and asserts the index is in the plan). + +**Tests / gates** — `uv run pytest --cov=app`: **4594 passed**, `app/` **99%** (>90%). `ruff check .`: All checks passed. `pyright`: 0 errors. `tests/e2e/test_smoke.py`: exit 0. + +**Docs** — new `benchmarks/phase188_live_gate/` (both battery captures, compare + recall outputs, pinned vectors, production-order build/latency logs, the 512 MB failure log, README) + pointer in `benchmarks/README.md`; report at `.agents/reports/188_chunks_hnsw_index/task04_recall_battery_gate.md` (+ the probe). + +**Deviations** — (1) the ×47 production-order corpus is valid for build/plan/latency but *not* recall (47 near-copies per real vector); recall verdict comes from the real-embedding replica, stated in both docs. (2) Phase task files untouched; no git index/history mutations — 34 paths left in the working tree. Dev DB returned to its start state (empty, 0029, index valid); scratch DBs dropped. + +**Next** — task 05 (`prod_index_build_runbook`): carry the 161 s / 1.6 GB figures, the shm constraint, and a post-build battery re-run on the real 414k KB. diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md.status b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.validate b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.validate new file mode 100644 index 0000000..d006b3c --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__04_recall_battery_gate.a1.validate @@ -0,0 +1,145 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 98%] +.......................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 325 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 259 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6946 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.err b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md new file mode 100644 index 0000000..1cfecc0 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md @@ -0,0 +1,12 @@ +All gates green. Final state: + +**Implemented (task 05)** +- `tests/e2e/test_vector_index_recall.py` (889 lines, mock-only, own app on 8205 + two ef-apps on 8206/8207 with stdout captured) — the four required tests: `test_answered_question_cites_the_same_document_indexed_or_not` (arms split by the real `0029` downgrade/upgrade, catalog state re-read inside each arm), `test_index_exists_and_is_valid_after_import`, `test_near_miss_question_still_deflects` (both arms), `test_ef_search_setting_is_honoured` (`=150` answers with no clamp line; `=50` answers identically and logs `retrieval: ef_search=50 raised to 100 (A7 candidate count)`). +- `tests/fixtures/hnswkb/` — tracked 11-note KB imported through the real importer with the index live; sentinels measured, not guessed (answered: cosine **0.6187**, fts 10, `hnswkb/infra/tilecache.md` first; near-miss: **0.1706**, fts 0). +- README: new **`Deploying migration 0029 (HNSW)`** runbook (pre-flight incl. `/dev/shm`, CONCURRENTLY semantics, 161 s / 1.6 GB / 3.9 KB-per-row, INVALID recovery, the `maintenance_work_mem > /dev/shm` trap, the 4 GB-box ordering pointer); **fixed a pre-existing docs bug** — the env table still said `BOR_RETRIEVAL_EF_SEARCH = 100` after task 04 shipped 400. AGENTS.md: new rule 16 (index owned only by `0029`, `SET LOCAL` only, never re-tune 400 or move a threshold, how a test may take the index off). `benchmarks/phase188_live_gate/`: task-05 parity section + captures. + +**Gates** — `test_vector_index_recall.py` 4 passed in isolation; `test_retrieval_quality` 4, `test_search_recall_small_sources` 7, `test_chat_rag` 3, `test_smoke` 3, all green in isolation; `uv run pytest --cov=app` **4594 passed, app/ coverage 99%**; `ruff check .` clean; `pyright` **0 errors**. Retrieval battery re-verified clean-room on the live replica (throwaway DB `bor_p188gate`, restored at `0011` → `0028` OFF → `head` ON, index valid/34 MB): the two captures are **byte-identical**; DB dropped, dev DB left as found (head `0029`, index valid, KB empty). + +**Deviations** — (1) no commit and no phase-dir move: git-mutating commands were disallowed for this run, so `00_phase.md`/task files and the commit remain untouched; (2) phase 119 LOCKED A1 gives a zero-read grounded turn **zero chips**, so test 1 pins "the same cited document" through the durable `query_log` record + the rendered "Nearby docs, in case" row + the `done` tiers (and pins `chips == 0` in both arms), stated in the module docstring; (3) one transient full-suite failure I caused by running the new E2E module concurrently with it (shared Postgres) — both pass serially, re-run clean. + +**Next pending** — task 05's step 5 (the atomic `--no-gpg-sign` commit + move `188_chunks_hnsw_index/` to `complete/`), then `.agents/phases/todo/189_chat_save_long_turns/`. diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md.status b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.validate b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.validate new file mode 100644 index 0000000..d006b3c --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/188_chunks_hnsw_index__05_e2e_index_chat_and_runbook.a1.validate @@ -0,0 +1,145 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 98%] +.......................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 325 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 259 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6946 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py b/.agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py new file mode 100644 index 0000000..ab8282c --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py @@ -0,0 +1,629 @@ +"""Phase 188 task 04 — the DETERMINISTIC index off/on gate probe (live KB, live endpoint). + +Why this probe exists next to `scripts/eval_retrieval.py` +--------------------------------------------------------- +`eval_retrieval` cannot gate this task on its own, for two reasons. It prints +five hits per question, while the difference between the exact and the +approximate search lives in the TAIL of the cosine tier's 100-row candidate set +(at `ef_search = 100` on the live replica the exact and approximate top-100 +agreed on 0.9814 of the rows on average, 0.94 at worst, with a per-chunk cosine +moved by up to 2.195e-03 while `top_score` stayed at 0.000e+00). Five printed rows +cannot show a 100-row candidate set, and they conflate the cosine tier with the RRF +fusion: an early off/on battery pair at `ef_search = 100` moved printed cosines by +up to ~3e-4 (above the 1e-4 tolerance) because a substituted tail candidate shifts +the fusion ranks and a document's several chunks let a hit keep its document while +changing its chunk. + +This probe embeds the battery ONCE, caches the vectors, and measures both sides from +the cache, so the access path is the only variable and all 100 rows are visible +separately from the fused five. It also reports the endpoint's own float jitter +(`endpoint_self_jitter`, one question embedded twice) rather than assuming it either +way — measured 0.000e+00 here, i.e. the endpoint is deterministic and every delta +below is the index's doing. LOCKED A3: the tolerance is never widened; the input is +pinned. + +What it measures per question (all through the real code path, +`app.rag.retriever._vector_candidates` / `retrieve`): + + * `exact` — the cosine tier with `enable_indexscan = off` (the exact + top-100 a seq scan produces, i.e. the pre-phase behaviour) + * `app` — the tier exactly as a request runs it (SET LOCAL ef_search + from Settings, planner free to use the index) + * `ef100` / `ef200` / `ef400` — the tier pinned at each ef_search + * latency of each of the above (median of `--reps`, own transaction) + * the full `retrieve()` pipeline: best_cosine, fts_hits, the A8 verdict, + the top-5 documents (the same best-per-doc order eval_retrieval prints) + * the EXPLAIN node types of the tier statement (anti-vacuity: `exact` + MUST seq-scan, `app` MUST name ix_chunks_embedding_hnsw when it exists) + +Gate for `compare` (LOCKED A3 — nothing here is ever relaxed to pass): + + * identical A8 verdicts, + * identical top-5 document lists and identical suggested documents, + * identical `top_score` / `best_cosine` within 1e-4, + * identical scores within 1e-4 for every rank inside the fused window + (`hybrid_top_k = 5`) — the ranks that reach RRF and the deflection gate, + * the cosine tier's top-100 candidate SET overlap at or above 0.99. + +Tail ranks are gated as a SET overlap rather than a score delta: an HNSW search +may hand back a different tail chunk at a near-identical cosine, which changes +which tail chunk the fusion could have raised without moving anything the turn +shows the model. That is exactly the difference this gate is allowed to see, and +the reason `eval_retrieval` alone cannot answer the task. + +Usage (repo root, live KB restored, the app's LLM key reachable — +`BOR_LLM_API_KEY` in `.env`, or `$AIPI_KEY` as the fallback app/config.py reads) +----------------------------------------------------------------- + PYTHONPATH=. uv run python \ + .agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py embed \ + --out /tmp/gate_vectors.json + # index OFF (alembic downgrade -1) + PYTHONPATH=. uv run python .../task04_recall_ab_probe.py measure \ + --vectors /tmp/gate_vectors.json --out /tmp/gate_off.json --reps 5 + # index ON (alembic upgrade head) + PYTHONPATH=. uv run python .../task04_recall_ab_probe.py measure \ + --vectors /tmp/gate_vectors.json --out /tmp/gate_on.json --reps 5 + PYTHONPATH=. uv run python .../task04_recall_ab_probe.py compare \ + --a /tmp/gate_off.json --b /tmp/gate_on.json + # recall@100 of the approximate index vs the exact top-100 + PYTHONPATH=. uv run python .../task04_recall_ab_probe.py recall \ + --path /tmp/gate_on.json +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import statistics +import sys +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv +from sqlalchemy import text +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import Session + +load_dotenv() + +BATTERY = Path("tests/fixtures/retrieval_battery.txt") +TOP_K = 100 # A7's cosine candidate count (the clamp floor) +EF_GRID = (100, 200, 400) +INDEX_NAME = "ix_chunks_embedding_hnsw" +CANDIDATE_SET_FLOOR = 0.99 # the phase's own recall floor (LOCKED A3) +TURN_TOP_K = 5 # app/config.py hybrid_top_k — what the turn actually fuses + + +def questions() -> list[str]: + lines = BATTERY.read_text(encoding="utf-8").splitlines() + return [ln.strip() for ln in lines if ln.strip() and not ln.lstrip().startswith("#")] + + +def cmd_embed(args: argparse.Namespace) -> int: + from app.config import get_settings + from app.db import db_available + from app.rag.llm import LLMClient + + if not db_available(): + print("probe: Postgres is down", file=sys.stderr) + return 1 + qs = questions() + vectors, jitter = asyncio.run(_embed_all(LLMClient(get_settings()), qs)) + out = {"questions": qs, "vectors": vectors, "endpoint_self_jitter": jitter} + Path(args.out).write_text(json.dumps(out)) + print(f"probe: embedded {len(qs)} questions -> {args.out} (endpoint self-jitter {jitter:.3e})") + return 0 + + +async def _embed_all(llm: Any, qs: list[str]) -> tuple[list[list[float]], float]: + """One loop for every embedding the probe needs (the SDK client is + loop-bound: a second asyncio.run with the same client raises + ``Event loop is closed``). Also embeds the first question twice more to + measure the endpoint's OWN run-to-run float jitter — the number that + explains why the plain off/on battery diff moves at all.""" + vectors = await llm.embed(qs) + first = (await llm.embed([qs[0]]))[0] + again = (await llm.embed([qs[0]]))[0] + jitter = max(abs(a - b) for a, b in zip(first, again, strict=True)) + await llm._client.close() # pyright: ignore[reportPrivateUsage] + return vectors, jitter + + +def _tier_statement(vector: list[float]) -> Any: + """The tier's own SELECT (statement object captured from the code).""" + from app.rag.retriever import _vector_candidates + + class Rec: + def __init__(self) -> None: + self.statements: list[Any] = [] + + def execute(self, statement: Any, params: Any = None) -> Rec: + self.statements.append(statement) + return self + + def all(self) -> list[Any]: + return [] + + rec = Rec() + _vector_candidates(rec, vector, TOP_K) # pyright: ignore[reportArgumentType] + return rec.statements[1].compile( + dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True} + ) + + +def _plan_nodes(session: Session, sql: Any) -> list[str]: + raw = session.execute(text(f"EXPLAIN (FORMAT JSON) {sql}")).scalar() + doc = json.loads(raw) if isinstance(raw, str) else raw + + def walk(plan: dict[str, Any]) -> Iterator[dict[str, Any]]: + yield plan + for child in plan.get("Plans") or []: + yield from walk(child) + + nodes = [] + for p in walk(doc[0]["Plan"]): + name = p.get("Index Name") or p.get("Relationship Name") or "" + nodes.append(f"{p['Node Type']}" + (f" on {name}" if name else "")) + return nodes + + +def _pipeline( + session: Session, question: str, vector: list[float], force_index: bool +) -> dict[str, Any]: + """The whole turn's retrieval, in ONE transaction, optionally with the + planner forced onto the HNSW index (``enable_seqscan = off``). + + Why the force: at ``ef_search = 100`` pgvector costs the approximate scan + ABOVE a sequential scan of the 8,866-chunk / 13 MB replica, so the planner + picks the seq scan and an "index ON" measurement would silently measure the + exact scan twice (a vacuous gate). Forcing the index measures the path + production (414,220 chunks, ~3,000 MB) actually takes. The force is + session-local (``SET LOCAL``) and rolls back with the transaction. + """ + from app.config import get_settings + from app.rag.retriever import retrieve, select_suggested + + settings = get_settings() + session.rollback() + session.execute(text("begin")) + if force_index: + session.execute(text("SET LOCAL enable_seqscan = off")) + # the plan a turn actually gets: the tier's own SET LOCAL, in the SAME + # transaction it runs in (EXPLAIN with the server's ef_search=40 would + # report a plan no request ever uses) + session.execute(text(f"SET LOCAL hnsw.ef_search = {int(settings.retrieval_ef_search)}")) + chunks = retrieve(session, question, vector) + best_cosine = max((c.cosine for c in chunks), default=0.0) + fts_hits = sum(1 for c in chunks if c.fts_hit) + best_by_doc: dict[str, Any] = {} + for c in chunks: + best_by_doc.setdefault(f"{c.document.source}/{c.document.path}", c) + out = { + "best_cosine": best_cosine, + "fts_hits": fts_hits, + "verdict": ( + "LOW (deflect)" + if best_cosine < settings.relevance_threshold and fts_hits == 0 + else "HIGH (answer)" + ), + "top5": list(best_by_doc)[:5], + "suggested": [f"{d.source}/{d.path}" for d in select_suggested(chunks)], + "plan": _plan_nodes(session, _tier_statement(vector)), + } + session.rollback() + return out + + +def _timed_tier( + session: Session, + vector: list[float], + setup: str | None, + reps: int, + ef_override: int | None = None, +) -> dict[str, Any]: + """Median/min/max milliseconds of the cosine tier, one transaction per rep. + + ``ef_override`` goes through the REAL knob (``BOR_RETRIEVAL_EF_SEARCH`` + + the ``get_settings`` cache), never a hand-written ``SET LOCAL``: the tier + issues its own ``SET LOCAL hnsw.ef_search`` from Settings, so a probe-side + ``SET`` would simply be overwritten by the line under test. + """ + import os + + from app.config import get_settings + from app.rag.retriever import _vector_candidates + + previous_env = os.environ.get("BOR_RETRIEVAL_EF_SEARCH") + if ef_override is not None: + os.environ["BOR_RETRIEVAL_EF_SEARCH"] = str(ef_override) + get_settings.cache_clear() + timings: list[float] = [] + rows: list[dict[str, Any]] = [] + try: + for i in range(reps): + session.rollback() # end any implicit transaction -> SET LOCAL is fresh + if setup: + session.execute(text("begin")) + session.execute(text(setup)) + t0 = time.perf_counter() + got = _vector_candidates(session, vector, TOP_K) + timings.append((time.perf_counter() - t0) * 1000.0) + if i == 0: + rows = [ + { + "rank": rank, + "chunk_id": str(c.chunk_id), + "cosine": c.cosine, + "doc": f"{c.document.source}/{c.document.path}", + } + for rank, c in enumerate(got, start=1) + ] + session.rollback() + finally: + if ef_override is not None: + if previous_env is None: + os.environ.pop("BOR_RETRIEVAL_EF_SEARCH", None) + else: + os.environ["BOR_RETRIEVAL_EF_SEARCH"] = previous_env + get_settings.cache_clear() + return { + "ms_median": round(statistics.median(timings), 3), + "ms_min": round(min(timings), 3), + "ms_max": round(max(timings), 3), + "rows": rows, + } + + +def cmd_measure(args: argparse.Namespace) -> int: + from app.config import get_settings + from app.db import SessionLocal, db_available + + if not db_available(): + print("probe: Postgres is down", file=sys.stderr) + return 1 + payload = json.loads(Path(args.vectors).read_text()) + qs: list[str] = payload["questions"] + vectors: list[list[float]] = payload["vectors"] + settings = get_settings() + with SessionLocal() as db: + index_present = db.execute( + text("select count(*) from pg_indexes where indexname = :n"), {"n": INDEX_NAME} + ).scalar() + ef_setting = settings.retrieval_ef_search + per_question: list[dict[str, Any]] = [] + for q, vec in zip(qs, vectors, strict=True): + record: dict[str, Any] = {"question": q} + record["exact"] = _timed_tier( + db, + vec, + "SET LOCAL enable_indexscan = off", + args.reps, + ) + record["app"] = _timed_tier(db, vec, None, args.reps) + # the APPROXIMATE path, forced (the planner declines it at this + # corpus size — see _pipeline's docstring), at each ef_search + for ef in EF_GRID: + record[f"forced{ef}"] = _timed_tier( + db, + vec, + "SET LOCAL enable_seqscan = off", + args.reps, + ef_override=ef, + ) + # anti-vacuity plans, same session, same vector + sql = _tier_statement(vec) + db.execute(text("begin")) + db.execute(text("SET LOCAL enable_indexscan = off")) + record["plan_exact"] = _plan_nodes(db, sql) + db.rollback() + db.execute(text("begin")) + record["plan_app"] = _plan_nodes(db, sql) + db.rollback() + # the pipeline a turn actually runs, with the A8 gate: once as the + # planner chooses, once forced onto the approximate index + record["pipeline"] = _pipeline(db, q, vec, force_index=False) + record["pipeline_forced"] = _pipeline(db, q, vec, force_index=True) + per_question.append(record) + print( + f" {q!r}: verdict={record['pipeline']['verdict']} " + f"exact={record['exact']['ms_median']}ms app={record['app']['ms_median']}ms " + f"plan_app={record['plan_app'][1] if len(record['plan_app']) > 1 else '?'}" + ) + Path(args.out).write_text( + json.dumps( + { + "index_present": int(index_present or 0), + "ef_search_setting": ef_setting, + "reps": args.reps, + "vector_candidates": settings.hybrid_vector_candidates, + "questions": qs, + "records": per_question, + } + ) + ) + print( + f"probe: measured {len(per_question)} questions -> {args.out} " + f"(index_present={index_present})" + ) + return 0 + + +def _docs(rec: dict[str, Any]) -> list[str]: + return [r["doc"] for r in rec["rows"]] + + +def cmd_compare(args: argparse.Namespace) -> int: + a = json.loads(Path(args.a).read_text()) + b = json.loads(Path(args.b).read_text()) + print(f"A={args.a} index_present={a['index_present']} ef={a['ef_search_setting']}") + print(f"B={args.b} index_present={b['index_present']} ef={b['ef_search_setting']}") + failures: list[str] = [] + max_score_delta = 0.0 + max_tier_delta = 0.0 # every rank of the tier, reported + max_window_delta = 0.0 # ranks <= hybrid_top_k, gated + for ra, rb in zip(a["records"], b["records"], strict=True): + assert ra["question"] == rb["question"] + q = ra["question"] + if ra["pipeline"]["verdict"] != rb["pipeline"]["verdict"]: + failures.append( + f"{q!r}: verdict {ra['pipeline']['verdict']} != " + f"{rb['pipeline']['verdict']}" + ) + if ra["pipeline"]["top5"] != rb["pipeline"]["top5"]: + failures.append( + f"{q!r}: top5 changed\n A={ra['pipeline']['top5']}\n" + f" B={rb['pipeline']['top5']}" + ) + if set(ra["pipeline"]["top5"]) != set(rb["pipeline"]["top5"]): + failures.append(f"{q!r}: top5 SET changed") + if ra["pipeline"]["suggested"] != rb["pipeline"]["suggested"]: + failures.append(f"{q!r}: suggested list changed") + delta = abs(ra["pipeline"]["best_cosine"] - rb["pipeline"]["best_cosine"]) + max_score_delta = max(max_score_delta, delta) + if delta > 1e-4: + failures.append( + f"{q!r}: top_score delta {delta:.9f} > 1e-4 " + f"(A={ra['pipeline']['best_cosine']:.9f} B={rb['pipeline']['best_cosine']:.9f})" + ) + # The candidate SET (the A7 tier's promise to the fusion) and the + # SCORES. Order is NOT the gate: `ORDER BY distance LIMIT 100` breaks + # ties by access path, so an index scan and a top-N sort legitimately + # return the same rows in a different order whenever two chunks share a + # cosine. What must not move is which chunks are in the list, their + # scores, and anything the turn shows the model. + da, dbb = ra["app"], rb["app"] + ids_a = {r["chunk_id"] for r in da["rows"]} + ids_b = {r["chunk_id"] for r in dbb["rows"]} + overlap = len(ids_a & ids_b) / max(1, len(ids_a)) + if overlap < CANDIDATE_SET_FLOOR: + failures.append( + f"{q!r}: the cosine tier's top-{TOP_K} candidate set overlap is " + f"{overlap:.4f}, below the {CANDIDATE_SET_FLOOR} floor " + f"(A-only={sorted(ids_a - ids_b)[:3]}…, B-only={sorted(ids_b - ids_a)[:3]}…)" + ) + for row_a, row_b in zip(da["rows"], dbb["rows"], strict=False): + d = abs(row_a["cosine"] - row_b["cosine"]) + max_tier_delta = max(max_tier_delta, d) + # The 1e-4 tolerance binds where the turn actually reads: the + # fused window is TOP_K=5 (app/config.py), and ranks 1-5 are the + # ones whose scores reach RRF and the deflection gate. Ranks below + # that only ever change WHICH tail chunk is in the list, and that + # is measured by the overlap floor above, not by a score delta. + if row_a["rank"] <= TURN_TOP_K: + max_window_delta = max(max_window_delta, d) + if row_a["rank"] <= TURN_TOP_K and d > 1e-4: + failures.append( + f"{q!r}: rank-{row_a['rank']} cosine delta {d:.9f} > 1e-4 " + f"(A={row_a['cosine']:.9f} B={row_b['cosine']:.9f})" + ) + tie_swaps = sum( + 1 + for a, b in zip(da["rows"], dbb["rows"], strict=False) + if a["chunk_id"] != b["chunk_id"] + ) + exact_ties = all( + abs(a["cosine"] - b["cosine"]) == 0.0 + for a, b in zip(da["rows"], dbb["rows"], strict=False) + if a["chunk_id"] != b["chunk_id"] + ) + same_top5 = "SAME" if ra["pipeline"]["top5"] == rb["pipeline"]["top5"] else "DIFFERENT" + same_sugg = ( + "SAME" if ra["pipeline"]["suggested"] == rb["pipeline"]["suggested"] else "DIFFERENT" + ) + print( + f" {q!r}: verdict {ra['pipeline']['verdict']} / " + f"{rb['pipeline']['verdict']} " + f"| top5 {same_top5} | suggested {same_sugg} " + f"| candidate-set overlap {overlap:.4f} " + f"| best_cosine delta {delta:.3e} " + f"| rank swaps {tie_swaps}" + f"{' (all exact ties)' if exact_ties else ' <-- NOT ties'}" + ) + print(f"\nmax top_score (best_cosine) delta = {max_score_delta:.3e} (gate <= 1e-4)") + print( + f"max in-window (rank <= {TURN_TOP_K}) tier cosine delta = " + f"{max_window_delta:.3e} (gate <= 1e-4)" + ) + print( + f"max tier cosine delta over all {TOP_K} ranks = {max_tier_delta:.3e} (reported only)" + ) + print( + f"tail note: ranks below the fused window are gated as the candidate-SET " + f"overlap (>= {CANDIDATE_SET_FLOOR}), not as a score delta: a graph search may " + "hand back a different tail chunk at a near-identical score without moving " + "anything the turn reads." + ) + for label, side in (("A", a), ("B", b)): + r0 = side["records"][0] + print(f"{label} plan(exact)={r0['plan_exact'][:2]} plan(app)={r0['plan_app'][:2]}") + print( + f"tier latency ms (median over questions): A exact={_med(a, 'exact')} " + f"app={_med(a, 'app')} | " + f"B exact={_med(b, 'exact')} app={_med(b, 'app')}" + ) + if failures: + print(f"\nGATE FAIL ({len(failures)} differences):") + for f in failures: + print(f" - {f}") + return 1 + print("\nGATE PASS: identical verdicts, identical top-5 sets and order, deltas <= 1e-4") + return 0 + + +def _med(side: dict[str, Any], key: str) -> float: + return round(statistics.median([r[key]["ms_median"] for r in side["records"]]), 3) + + +def cmd_recall(args: argparse.Namespace) -> int: + """THE APPROXIMATE-PATH GATE, run on the index-ON side. + + Three things per question, all on the SAME pinned vector: + + * recall@100 of the forced HNSW scan against the exact top-100 + * the per-chunk cosine delta at equal rank, and the best-cosine + (``top_score``) delta + * the A8 verdict, the top-5 documents and the suggested list of the + forced-index pipeline against the planner's own pipeline + + A vacuity guard is part of the gate: the forced plan MUST name + ``ix_chunks_embedding_hnsw`` and the exact plan MUST be a Sort over a seq + scan, otherwise the two sides measured the same thing twice. + """ + side = json.loads(Path(args.path).read_text()) + if not side["index_present"]: + print("recall: the index is not present — run this on the index-ON side", file=sys.stderr) + return 2 + failures: list[str] = [] + out: dict[str, Any] = {} + for ef in EF_GRID: + key = f"forced{ef}" + overlaps: list[float] = [] + deltas: list[float] = [] + best_deltas: list[float] = [] + for r in side["records"]: + exact = [x["chunk_id"] for x in r["exact"]["rows"]] + got = [x["chunk_id"] for x in r[key]["rows"]] + overlaps.append(len(set(exact) & set(got)) / max(1, len(exact))) + deltas.append( + max( + ( + abs(a["cosine"] - b["cosine"]) + for a, b in zip(r["exact"]["rows"], r[key]["rows"], strict=False) + ), + default=0.0, + ) + ) + best_deltas.append( + abs( + max(x["cosine"] for x in r["exact"]["rows"]) + - max(x["cosine"] for x in r[key]["rows"]) + ) + ) + out[key] = { + "mean_overlap": round(statistics.mean(overlaps), 6), + "min_overlap": round(min(overlaps), 6), + "max_cosine_delta": round(max(deltas), 9), + "max_top_score_delta": round(max(best_deltas), 9), + "ms_median": round( + statistics.median([r[key]["ms_median"] for r in side["records"]]), 3 + ), + } + print( + f" {key}: mean recall@100={out[key]['mean_overlap']:.4f} " + f"min={out[key]['min_overlap']:.4f} " + f"max per-chunk cosine delta={out[key]['max_cosine_delta']:.3e} " + f"max top_score delta={out[key]['max_top_score_delta']:.3e} " + f"latency={out[key]['ms_median']}ms" + ) + if ef == side["ef_search_setting"] and out[key]["min_overlap"] < 1.0: + print( + f" note: at the SHIPPED ef_search={ef} the approximate top-100 differs from the " + f"exact one in " + f"{round((1 - out[key]['min_overlap']) * 100, 2)}% of the worst question" + ) + print("\n pipeline parity (A8 verdict / top-5 / suggested), planner vs forced index:") + for r in side["records"]: + a, b = r["pipeline"], r["pipeline_forced"] + same_v = a["verdict"] == b["verdict"] + same_5 = a["top5"] == b["top5"] + same_s = a["suggested"] == b["suggested"] + d = abs(a["best_cosine"] - b["best_cosine"]) + print( + f" {r['question']!r}: verdict {'SAME' if same_v else 'DIFFERENT'} " + f"({a['verdict']} / {b['verdict']}) | top5 {'SAME' if same_5 else 'DIFFERENT'} " + f"| suggested {'SAME' if same_s else 'DIFFERENT'} | top_score delta {d:.3e}" + ) + if not same_v: + failures.append(f"{r['question']!r}: verdict moves with the index") + if not same_5: + failures.append( + f"{r['question']!r}: top-5 changes with the index\n" + f" planner ={a['top5']}\n index ={b['top5']}" + ) + if d > 1e-4: + failures.append( + f"{r['question']!r}: top_score delta {d:.3e} > 1e-4 on the forced index" + ) + print("\n plans (vacuity guards):") + r0 = side["records"][0] + print(f" exact: {r0['plan_exact'][:3]}") + print(f" forced index (pipeline): {r0['pipeline_forced']['plan'][:3]}") + if not any("ix_chunks_embedding_hnsw" in n for n in r0["pipeline_forced"]["plan"]): + failures.append( + "the forced pipeline did NOT use ix_chunks_embedding_hnsw — " + "the gate measured nothing" + ) + if not any("Seq Scan" in n for n in r0["plan_exact"]): + failures.append("the exact side was not a seq scan — the gate measured the same plan twice") + print( + f"\n latency ms over questions: exact(seq)={_med(side, 'exact')} " + f"planner={_med(side, 'app')} " + + " ".join(f"forced{ef}={_med(side, f'forced{ef}')}" for ef in EF_GRID) + ) + if failures: + print(f"\nGATE FAIL ({len(failures)}):") + for f in failures: + print(f" - {f}") + return 1 + print("\nGATE PASS: the approximate path returns the exact verdicts, top-5s and top_scores") + Path(args.path).with_suffix(".recall.json").write_text(json.dumps(out, indent=2)) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="task04_recall_ab_probe") + sub = p.add_subparsers(dest="cmd", required=True) + e = sub.add_parser("embed") + e.add_argument("--out", required=True) + m = sub.add_parser("measure") + m.add_argument("--vectors", required=True) + m.add_argument("--out", required=True) + m.add_argument("--reps", type=int, default=5) + c = sub.add_parser("compare") + c.add_argument("--a", required=True) + c.add_argument("--b", required=True) + r = sub.add_parser("recall") + r.add_argument("--path", required=True) + r = sub.add_parser("gate") + r.add_argument("--path", required=True) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.cmd == "embed": + return cmd_embed(args) + if args.cmd == "measure": + return cmd_measure(args) + if args.cmd == "compare": + return cmd_compare(args) + if args.cmd == "gate": + return cmd_recall(args) + return cmd_recall(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.agents/reports/188_chunks_hnsw_index/task04_recall_battery_gate.md b/.agents/reports/188_chunks_hnsw_index/task04_recall_battery_gate.md new file mode 100644 index 0000000..b3bc1ab --- /dev/null +++ b/.agents/reports/188_chunks_hnsw_index/task04_recall_battery_gate.md @@ -0,0 +1,227 @@ +# Phase 188 · Task 04 — Recall + deflection gate on the live retrieval battery + +**Status: complete.** The index changes nothing the turn shows the model, at every +`ef_search` measured. One knob moved as the task's remedy: `retrieval_ef_search` +ships at **400** instead of the candidate count. No honesty threshold moved. + +## Where the measurement ran (prerequisites, checked) + +Live `brain-of-reese_db_1` (**PostgreSQL 17.11**, Debian image, host port 5432) and the +**live embedding endpoint** — model `embed`, 768 dims, the key from +`BOR_LLM_API_KEY` in the local `.env` (never printed, never committed). The battery +and the probe both go through `app.rag.embeddings`, so the question vectors are the +same ones a real turn would compute. + +| Prerequisite | Measured | +| --- | --- | +| `SELECT extversion FROM pg_extension WHERE extname='vector'` | **0.8.0** (≥ 0.5.0 required) — the running server matches this checkout's pin | +| `alembic current` before the task | head `0028`, `ix_chunks_embedding_hnsw` absent | +| `maintenance_work_mem` | 64 MB (the container default; nobody ever applied D5's 512 MB) | +| `/dev/shm` inside `brain-of-reese_db_1` | **63 MB** | +| Planner | `seq_page_cost=1`, `random_page_cost=4` (default) | +| Disk on the Postgres data volume | **2.2 TB free of 3.7 TB** (41 % used) | +| Production relation, from the 2026-10-15 incident record (not measured here) | 414,220 chunks, heap **3,000 MB** | +| Restored gate corpus | **1,000** documents, **8,866** chunks, 8,866 embeddings, heap **13 MB** (95 MB plain-SQL dump) | + +The replica came from `data/bor_live_kb_replica.dump.sql` (plain-SQL `pg_dump`, +schema + data + its 10 own indexes, `CREATE EXTENSION IF NOT EXISTS vector`) via +the `restore-test-db` procedure, into a freshly dropped `public` schema, with a +disposable pre-restore backup taken first. + +**The one prerequisite that decides what this task can measure:** the dump carries +`alembic_version = 0011`, so the restored corpus lands on revision 0011 and +`alembic upgrade head` replays **0012…0029 into it** — which is exactly why the +gate is trustworthy on the shape of the real migration: 0029's +`CREATE INDEX CONCURRENTLY` ran against 8,866 **real** embeddings (seconds, valid +on the first attempt) rather than against an empty table. Restoring into a schema +already at head and skipping the upgrade would have built nothing and measured +nothing. The dev database was returned to the exact state it was in when the task +started (empty, at head `0029`, index valid) before the suite ran. + +## Step 1–2 — the battery, index OFF vs ON + +`uv run python -m scripts.eval_retrieval --from-file tests/fixtures/retrieval_battery.txt` +with the index dropped (`alembic downgrade -1`) and with it present +(`alembic upgrade head`). Captures: +[`battery_index_off.txt`](../../../benchmarks/phase188_live_gate/battery_index_off.txt), +[`battery_index_on.txt`](../../../benchmarks/phase188_live_gate/battery_index_on.txt). + +**The two captures are byte-identical** (7,193 bytes each, `diff` empty): every +`best_cosine`, every `fts_hits`, all 35 numbered hits, all 7 A8 verdicts. Battery +wall time 1.35 s with the index, 1.49 s without. + +**Why the battery alone cannot gate this, recorded because it nearly mis-measured +it.** Two reasons, both measured rather than assumed: + +* It prints **five** hits per question, while the difference between an exact and an + approximate search lives in the **tail** of the tier's 100-row candidate set. At + `ef_search = 100` the exact and approximate top-100 agreed on 0.9814 of rows on + average (0.94 worst) with a per-chunk cosine moved by up to 2.195e-03, while + `top_score` — the rank-1 number the honesty gate reads — stayed at 0.000e+00. The + battery prints a perfect match while the tier has changed its hand. +* At the task-02 default its OFF and ON captures disagreed by up to **~3e-4** on + printed cosines — above the 1e-4 tolerance — and the cause needed isolating before + anything could be concluded. The probe's own measurement of the endpoint + (`endpoint_self_jitter`, one question embedded twice) is **0.000e+00**: the endpoint + is deterministic, so that 3e-4 was **the index**. Which is what the pinned tier + then explained — at `ef_search = 100` a substituted tail candidate changes the RRF + ranks, and because a document contributes several chunks, a printed hit can keep + its document and change its chunk, moving the printed cosine while the top-5 list + and `top_score` stay identical. At the shipped 400 the two captures are + byte-identical: the recall result (1.0000) showing up in the report a human reads. + +Pinning is what makes that separation possible — with the vectors fixed, the tier can +be read on its own across all 100 rows instead of 5, and a delta can only mean the +access path. + +So the gate ran with the input pinned — `embed` once, cache, `measure` both sides +from the cache ([`question_vectors.json`](../../../benchmarks/phase188_live_gate/question_vectors.json)) +— which is also what LOCKED A3 demands: never widen the tolerance, remove the noise +instead. + +## Steps 3–4 — the deterministic gate (pinned vectors, same code path) + +[`deterministic_gate_compare.txt`](../../../benchmarks/phase188_live_gate/deterministic_gate_compare.txt) +— A = index dropped (the pre-phase `Limit → Sort → Hash Join → Seq Scan on chunks`), +B = index present at the shipped `ef_search`, identical question vectors: + +| Gate | Result | +| --- | --- | +| A8 verdicts, 7/7 | **identical** | +| Top-5 document sets **and order**, 7/7 | **identical** | +| Suggested documents, 7/7 | **identical** | +| `top_score` (`best_cosine`) delta | **0.000e+00** (≤ 1e-4) | +| Tier score delta, ranks 1–5 (`hybrid_top_k`, the fused window) | **0.000e+00** (≤ 1e-4) | +| Cosine-tier top-100 **candidate-set overlap** | **1.0000 on all 7** (floor 0.99) | +| Rank swaps that are not exact cosine ties | **0** | +| Tier latency, median over questions | **38.3 ms → 15.3 ms** | + +Ranks below the fused window are gated as a candidate **set** overlap rather than a +score delta, and that distinction earned its place: an HNSW search can hand back a +different tail chunk at a near-identical cosine, which moves which tail chunk the +fusion could have raised and nothing else. Every swap observed in the shipped +configuration was an exact cosine tie broken by the access path. + +[`deterministic_gate_recall.txt`](../../../benchmarks/phase188_live_gate/deterministic_gate_recall.txt) +— the tier forced onto the index at each depth, recall@100 against the exact scan, +**real embeddings**, two independent graph builds (the index is dropped and rebuilt +between the OFF and ON runs): + +| `ef_search` | mean recall@100 | min | max per-chunk cosine delta | max `top_score` delta | tier latency | +| --- | --- | --- | --- | --- | --- | +| 100 — task 02's default, `== LIMIT` | 0.9814 / 0.9843 | 0.94 | 2.195e-03 | **0.000e+00** | 13.8 ms | +| 200 | 1.0000 / 0.9971 | 1.00 / 0.99 | 0.0 | 0.0 | 14.1 ms | +| 400 — **shipped** | **1.0000 / 1.0000** | **1.00** | **0.0** | **0.0** | 14.6 ms | + +## Step 5 — the remedy, and what it did not touch + +Nothing user-visible moved at `ef_search = 100`, so the task's parity gate passed at +the task-02 default. What moved was the 100-row candidate set the cosine tier hands +the RRF fusion — 0.9814 mean / 0.94 minimum — below this phase's own acceptance +floor and matching the counter-case task 03 recorded for high-effective-rank data +(one substitution at rank ~89 of 100 reorders 22 tail ranks). **LOCKED A3 names one +remedy: raise `ef_search` (towards 400).** Applied: + +* `app/config.py` — `retrieval_ef_search: int = 400`, comment carrying the numbers. +* `.env.example`, `README.md` (retrieval section + env table), `AGENTS.md` (rule 10) + — the default, why it is 400, and that the thresholds stayed put. +* `tests/unit/test_config.py`, `tests/unit/test_retriever.py`, + `tests/unit/test_settings_catalog.py`, `tests/integration/test_chunks_hnsw_recall.py` + (`EF_PIN = 400`, `EF_RAISED = 600`) — every pin that encoded 100. +* **Untouched (LOCKED A3):** `relevance_threshold` 0.62, `lexical_support_floor` 0.60, + `name_hit_bonus` 0.005, `hybrid_top_k` 5, `hybrid_vector_candidates` 100, and every + recall floor and tolerance in this phase's tests. + +## Production-order probe — and the two things it found + +The replica is 13 MB of heap, so the planner legitimately declines a 34 MB index +below ~17 MB at `random_page_cost = 4`; the replica cannot answer "does production +use it, and what does it cost". A throwaway database (`bor_scale188b`, created and +dropped in the task) reproduced the production row count from the replica's own +embeddings ×47, each copy multiplied element-wise by a random near-one jitter vector +so all **416,702** vectors are distinct and the heap is 557 MB +([`production_order_build.log`](../../../benchmarks/phase188_live_gate/production_order_build.log), +[`production_order_latency.txt`](../../../benchmarks/phase188_live_gate/production_order_latency.txt)): + +| Measurement | Value | +| --- | --- | +| Build through `uv run alembic upgrade head`, `maintenance_work_mem = 64 MB` | **161 s**, alembic process peak RSS 81 MB | +| Index size | **1,625 MB** — ~3.9 KB/row at `m=16`, 768-dim float4: plan for **~1.6 GB** on the real 414k | +| Plan at `ef_search` 100 / 200 / 400 | **HNSW index scan at every depth** (cost 3,333 / 5,621 / 9,806) | +| Tier latency | **~550 ms seq → 12.0–14.9 ms** with the index (~40×), the ~1 ms `SET LOCAL` included | + +**Finding 1 — `SET maintenance_work_mem = '512MB'` (D5) is unusable in this +container, and the failure was reproduced at production order** +([`build_maintenance_work_mem_512MB_failure.log`](../../../benchmarks/phase188_live_gate/build_maintenance_work_mem_512MB_failure.log)): +`could not resize shared memory segment "/PostgreSQL.3366409952" to 533761504 bytes: +No space left on device` — `/dev/shm` is **63 MB**, and the dying build leaves an +**INVALID** index. That made the self-heal run for real: migration 0029 found +`indisvalid = false`, logged `dropping INVALID … rebuilding`, dropped it, and +rebuilt it valid (`pg_stat_progress_create_index` watched the second build at +860,544 of 872,448 version-0 tuples). So the D5 pre-check stays, but the recorded +procedure is "run `alembic upgrade head` with the container default"; raising +`maintenance_work_mem` only makes sense together with `--shm-size`. The container +default costs time, not correctness. + +**Finding 2 — a recall pin must not depend on the planner cooperating.** pgvector +costs an HNSW scan partly on `ef_search`: at production order the index path cost +**3,333 / 5,621 / 9,806** at `ef_search` 100 / 200 / 400 respectively, so a deeper +value makes the index a less attractive plan and, on a relation small enough for a +top-N sort to win, it can lose. Task 03's statement pin EXPLAINs the tier's SELECT +without the tier's own `SET LOCAL hnsw.ef_search`, so what it proves is the plan the +*server* default produces — a recall run at a depth the planner declines would score +the exact scan twice and report 1.0000. `test_top100_recall_at_the_shipped_ef_search` +now refuses sequential scans for the measurement (session-scoped, reset in a +`finally`) and asserts `ix_chunks_embedding_hnsw` is in the plan before scoring, so +it fails loudly instead of passing vacuously. On the live replica the planner did take +the index at the shipped depth (`Limit → Nested Loop → Index Scan on +ix_chunks_embedding_hnsw`, with the OFF side correctly +`Limit → Sort → Seq Scan`), and at production order it took it at all three depths. + +**Limitation, stated plainly:** the ×47 jittered corpus is *not* a valid recall +substrate — "the exact top-100" of 47 near-copies of 8,866 vectors is 100 knife-edge +slots inside a few clusters, which is not a real KB. Its overlap numbers (0.23–0.64) +are excluded from the recall verdict, which comes from the live replica where every +vector is a real chunk. After the production build the owner should re-run the +battery on the real 414k KB exactly as here; that is task 05's runbook step. + +## Verification + +| Gate | Result | +| --- | --- | +| `uv run pytest -q --cov=app --cov-report=term-missing` | **all passed**, `app/` coverage **99%** (> 90%) | +| `uv run ruff check .` | **All checks passed!** | +| `uv run pyright` | **0 errors, 0 warnings** | +| `uv run pytest tests/e2e/test_smoke.py -v --no-cov` | **exit 0** (app boots indexed, corpus assertion skips on the empty dev KB) | +| `uv run alembic downgrade -1 && uv run alembic upgrade head` | round trip verified twice on the live replica; index valid after each | + +## Files + +* `app/config.py` — `retrieval_ef_search` default `100 → 400` with the evidence. +* `.env.example`, `README.md`, `AGENTS.md` — the new default and why. +* `tests/unit/test_config.py`, `tests/unit/test_retriever.py`, + `tests/unit/test_settings_catalog.py`, + `tests/integration/test_chunks_hnsw_recall.py` (+ its module docstring's measured + numbers) — pins moved to the shipped depth, vacuity guard added. +* `benchmarks/phase188_live_gate/` — the captures, the gate outputs, the + production-order logs, the failure log, the pinned vectors, and the README that + ties them together; `benchmarks/README.md` gained a "Retrieval gate" pointer. +* `.agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py` — the probe + (`embed` / `measure` / `compare` / `recall` / `gate`). + +## Reproduce + +```bash +# index OFF +uv run alembic downgrade -1 +uv run python -m scripts.eval_retrieval --from-file tests/fixtures/retrieval_battery.txt +uv run alembic upgrade head +uv run python -m scripts.eval_retrieval --from-file tests/fixtures/retrieval_battery.txt +# deterministic, one embedding per question +P=.agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py +PYTHONPATH=. uv run python $P embed --out /tmp/v.json +PYTHONPATH=. uv run python $P measure --vectors /tmp/v.json --out /tmp/off.json # index dropped +PYTHONPATH=. uv run python $P measure --vectors /tmp/v.json --out /tmp/on.json # index present +PYTHONPATH=. uv run python $P compare --a /tmp/off.json --b /tmp/on.json +PYTHONPATH=. uv run python $P gate --path /tmp/on.json +``` diff --git a/.env.example b/.env.example index 28e9e22..4ba8d4c 100644 --- a/.env.example +++ b/.env.example @@ -96,6 +96,7 @@ BOR_EMBED_BATCH_SIZE=16 # --- Hybrid retrieval (vector + Postgres FTS, RRF-fused) --- BOR_HYBRID_VECTOR_CANDIDATES=100 # cosine list width for the fusion +# BOR_RETRIEVAL_EF_SEARCH=400 # HNSW ef_search for the cosine tier (phase 188): SET LOCAL per transaction, never a global GUC. Default 400 — the depth the live battery (benchmarks/phase188_live_gate/) measured EXACT top-100 parity at; the candidate count (100) recovered 0.9814 mean / 0.94 min of it on real embeddings, and the honesty thresholds were NOT touched to get there. Below BOR_HYBRID_VECTOR_CANDIDATES it is CLAMPED up to it with one WARNING line; > 0 (0/negative fails startup); max 1000 BOR_HYBRID_LEXICAL_CANDIDATES=30 # FTS list width for the fusion BOR_RRF_K=60 # Reciprocal Rank Fusion damping constant BOR_RECENCY_BOOST=0.0007 # max additive fused-score bonus a zero-age doc gets (phase 106); 0 = off (pre-phase ranking byte-identical) diff --git a/AGENTS.md b/AGENTS.md index c508703..3f6a9ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -140,7 +140,31 @@ clause) therefore satisfies `peak >= current`, never a peak below the run's own footprint, and an unknowable peak is never fabricated from the current reading); UI-side, - honor the "never stale" feedback contract (PLAN §7.4). + honor the "never stale" feedback contract (PLAN §7.4). Phase 188 adds + the retrieval tier's ONE operator-visible line: when + `BOR_RETRIEVAL_EF_SEARCH` sits below the A7 candidate count + `_vector_candidates` logs `retrieval: ef_search=%d raised to %d (A7 + candidate count)` at WARNING and runs the search at the candidate + count anyway — the clamp is never silent, and the healthy + (ef_search >= candidate count) turn logs nothing. The value itself + reaches Postgres as `SET LOCAL hnsw.ef_search` inside the turn's own + transaction (LOCKED A4: no global GUC, nothing survives the request or + a pool recycle). Task 04's live gate (the 1,000-document live-replica KB, + the live embedding endpoint, question vectors PINNED so the access path is the + one variable — `benchmarks/phase188_live_gate/`) found no A8, top-5 or + `top_score` difference at any + depth and shipped `retrieval_ef_search = 400`: at the candidate count the + graph recovered the exact top-100 for 0.9814 of rows on average (0.94 + worst), at 400 for all of them, and NO threshold + (`relevance_threshold`, `lexical_support_floor`, `name_hit_bonus`, + `hybrid_top_k`, `hybrid_vector_candidates`) moved to get there (LOCKED A3). + Two things the operator needs from the same run: building the index costs + ~161 s per 417k chunks and ~1.6 GB (3.9 KB/row at `m = 16`, 768-dim), and + `maintenance_work_mem = '512MB'` is NOT applicable in this container — + `/dev/shm` is 63 MB, so a raise above that dies with `could not resize + shared memory segment … No space left on device` and leaves an INVALID + index for the migration's self-heal to drop and rebuild (observed live); + the container's 64 MB default builds it fine. 11. **Never block the event loop:** anything blocking (a `subprocess`, synchronous file/network I/O, a heavy walk) reached from an `async` handler or a spawned background coroutine goes through @@ -246,6 +270,44 @@ stored text, a changed file re-indexes and the fast path still fires after). +16. **The cosine tier is indexed — keep it indexed, and keep `0029` the ONE + definition of that index:** `ix_chunks_embedding_hnsw` (HNSW over + `chunks.embedding`, `vector_cosine_ops`, `m = 16`, + `ef_construction = 64`) exists because every turn's + `ORDER BY embedding <=> :vec LIMIT 100` full-scanned ~3 GB on the + interactive path (phase 188). Build and drop it ONLY through migration + `0029` (`CREATE/DROP INDEX CONCURRENTLY` inside + `op.get_context().autocommit_block()`, which pre-drops an INVALID remnant + and raises if the build does not validate) — never an inline + `CREATE INDEX` in app code, a test, or a runbook step; never a second + vector index, a `halfvec` cast, or a rewritten `ORDER BY` (the `<=>` + operator is what the graph serves, LOCKED A3/A6). A test that needs the + index OFF moves the SCHEMA (`command.downgrade(cfg, "0028")` / + `command.upgrade(cfg, "0029")`) after rolling back its own session — an + open snapshot parks a `CONCURRENTLY` build indefinitely — and asserts the + catalog state it claims (`pg_index.indisvalid`, `pg_get_indexdef`). + `hnsw.ef_search` is set with `SET LOCAL` inside the turn's own transaction + and the `int()`-coerced setting is the only source (no global `SET`, no + pool-wide state, nothing survives a recycle); the shipped default is + **400** because that is the depth the live gate measured exact-top-100 + parity at, so "tuning" it back to the candidate count is a recall change, + and closing a recall gap by moving an honesty threshold is forbidden + (LOCKED A3). The `ef_search` a turn gets is + `max(retrieval_ef_search, hybrid_vector_candidates)` with a WARNING when + clamped. Pinned by `tests/integration/test_chunks_hnsw_index.py` (DDL + round trip + INVALID self-heal + drift guard), + `tests/integration/test_chunks_hnsw_recall.py` (the tier's captured + statement names the index, `SET LOCAL` reaches the connection, recall at + the shipped depth with seq scans refused), and + `tests/e2e/test_vector_index_recall.py` (the owner-visible arm: the SAME + cited document with the index dropped and restored, the index VALID after a + real importer run, the near-miss still deflecting in both arms, and the + clamp read off the app's own captured stdout at `=150` / `=50`). That + suite's sentinels are calibrated against + `tests/fixtures/hnswkb/` — an 11-note tracked KB whose BYTES own the + quoted cosine/lexical margins, so editing a note there moves the numbers in + the module docstring and the pins together. + ## Quick reference ```bash podman compose up -d db # start Postgres 17 + pgvector diff --git a/README.md b/README.md index 9d200b0..a236f77 100644 --- a/README.md +++ b/README.md @@ -963,7 +963,20 @@ curl -s -X DELETE localhost:8000/api/steering/ # remove Every question is embedded and also lexically tokenized (OR-joined, English stemming) and searched **twice** against Postgres: -1. **Vector** — pgvector cosine top-N (default `BOR_HYBRID_VECTOR_CANDIDATES=100`) +1. **Vector** — pgvector cosine top-N (default `BOR_HYBRID_VECTOR_CANDIDATES=100`) over + the `ix_chunks_embedding_hnsw` HNSW index (phase 188), searched at + `BOR_RETRIEVAL_EF_SEARCH` (default **`400`**, set per transaction with + `SET LOCAL` — never a global setting). 400 is not a guess: the live battery + in [`benchmarks/phase188_live_gate/`](benchmarks/phase188_live_gate/README.md) + measured the graph recovering the exact top-100 for only 0.9814 of rows on + average (0.94 worst) at `100` — the candidate count — against 1.0000 at 400, + with every A8 verdict, top-5 set and `top_score` unchanged at both depths + (the honesty thresholds were never touched). A value below the candidate + count is **clamped up** to it with one WARNING line in the server log + (`retrieval: ef_search=… raised to …`): a smaller `ef_search` returns + fewer candidates than the tier promises, which is a silent recall + regression the honesty gate can turn into a deflect. Raise it (max 1000) + if recall matters more than the tier's latency. 2. **Lexical** — a stored `tsvector` (GIN-indexed) matched with `to_tsquery`, top-N by `ts_rank` (default `BOR_HYBRID_LEXICAL_CANDIDATES=30`), led by **name hits** — documents whose *path* matches a question token (`gitea` @@ -1736,6 +1749,7 @@ startup (the DB rows shadow the env, `.env` is never rewritten). | Env | Default | Meaning | |-----|---------|---------| | `BOR_HYBRID_VECTOR_CANDIDATES` | `100` | Cosine list width for the RRF fusion | +| `BOR_RETRIEVAL_EF_SEARCH` | `400` | HNSW `ef_search` for the cosine tier (phase 188) — `SET LOCAL` per transaction; clamped UP to `BOR_HYBRID_VECTOR_CANDIDATES` with a WARNING when lower (that clamp is the recall floor, so `0`/negative fails startup); raise it for more recall, Postgres caps it at 1000. 400 is the measured depth at which the graph recovers the exact top-100 (see [How retrieval works](#how-retrieval-works-hybrid)) | | `BOR_HYBRID_LEXICAL_CANDIDATES` | `30` | FTS list width for the RRF fusion | | `BOR_RRF_K` | `60` | RRF damping constant (`1/(k + rank)`) | | `BOR_RECENCY_BOOST` | `0.0007` | Max additive fused-score bonus a zero-age doc gets (phase 106); `0` = off | @@ -1982,6 +1996,14 @@ start while either is empty. The entrypoint runs `alembic upgrade head` automatically on start. +> **Deploy note (migration `0029`, phase 188):** this step builds +> `ix_chunks_embedding_hnsw` — an HNSW index over `chunks.embedding` — and +> it is the one migration in this repo that takes **minutes** on a full +> corpus and needs free RAM to do it. It will not lock the table, it can +> be interrupted, and it self-heals — but read +> [Deploying migration 0029 (HNSW)](#deploying-migration-0029-hnsw) before +> running it on the production box. + Or run the whole stack from compose (app + db): ```bash podman compose --profile prod up -d --build @@ -1994,6 +2016,127 @@ Production hardening: app runs as non-root (uid 10001), slim image, healthcheck on `/api/health`, debugpy off unless `DEBUGPY=1`, all assets served locally (no CDN), `BOR_ENVIRONMENT=production`. +### Deploying migration 0029 (HNSW) + +Migration `0029` builds `ix_chunks_embedding_hnsw` — an HNSW graph over the +existing `vector(768)` column (`m = 16`, `ef_construction = 64`), so the +cosine tier of every chat turn stops sequential-scanning +[`benchmarks/phase188_live_gate/`](benchmarks/phase188_live_gate/README.md). +Everything changes about the **plan**; nothing changes about the **answer** +(the measured parity is in the benchmark link above and in +`tests/integration/test_chunks_hnsw_recall.py`). It is still the migration +to run deliberately rather than as a side effect of a deploy. + +**Pre-flight — all four, on the machine that runs it.** + +```bash +free -m # free RAM, whole box +psql "$BOR_DATABASE_URL" -c "show shared_buffers;" # + maintenance_work_mem +psql "$BOR_DATABASE_URL" -c "select extversion from pg_extension where extname='vector';" +podman exec brain-of-reese_db_1 df -h /dev/shm # the one people miss +``` + +* **pgvector ≥ 0.5.0** (`0.8.0` measured). HNSW does not exist before 0.5 — + the migration fails with `access method "hnsw" does not exist`. +* **Disk:** budget **~3.9 KB per chunk row** — the real ~414k chunks need + roughly **1.6 GB** for the index on top of the ~3 GB heap. A build that + runs out of disk is the one failure the self-heal cannot talk its way out + of. +* **RAM:** the build itself is not the hungry part — a 416,702-row build + peaked at **81 MB** of RSS in the `alembic` process at the container's + default `maintenance_work_mem` (64 MB), and 1.6 GB of index is page cache + afterwards, which Linux drops under pressure. What the box needs is room + for the build to finish while the app keeps serving. +* **`/dev/shm`:** see the `maintenance_work_mem` note below — inside this + project's compose container it is **63 MB**, and that is what caps the + knob, not the box's RAM. + +**Build it.** + +```bash +uv run alembic upgrade head +``` + +`CREATE INDEX CONCURRENTLY` takes `ShareUpdateExclusiveLock`: reads and +writes continue, the app keeps answering through the whole build, and the +build takes **~161 s for 416,702 vectors** at `maintenance_work_mem = 64 MB` +(measured, `production_order_build.log`) — budget minutes, not seconds, and +more on a 4 GB-class box. + +*On `maintenance_work_mem`:* + +* Never rely on the PostgreSQL **4 MB** default if you can set the knob at + all — a 4 MB graph build spills and is dramatically slower. +* **But do not raise it above `/dev/shm` in this container.** D5's original + suggestion (`SET maintenance_work_mem = '512MB'`) was run at production + order and failed: + `could not resize shared memory segment "/PostgreSQL.3366409952" to 533761504 bytes: No space left on device` + — the container's `/dev/shm` is **63 MB**, and the dying build leaves an + **INVALID** index behind (`build_maintenance_work_mem_512MB_failure.log`). + The migration's self-heal then did its job (dropped the INVALID graph, + rebuilt it valid), so nothing was lost, but the recorded procedure for + THIS container is: **`alembic upgrade head` with the container default + (64 MB)**, which costs time and not correctness. Raising + `maintenance_work_mem` only makes sense together with a larger + `--shm-size` / `shm_size:` on the db container. + +**`CONCURRENTLY` semantics, in the owner's terms.** No read or write is +blocked. The build is long, and if it dies — OOM killer, cancel, network +drop, container restart — Postgres does **not** roll it back: it leaves the +index present and marked `indisvalid = false`. That state is silent (it +costs insert time on every sync and serves no query), which is why `0029` +post-validates its own build and **raises** instead of reporting success. + +**Recovery.** + +```bash +# "did it actually build?" — one row, `t` is the answer +psql "$BOR_DATABASE_URL" -c "select i.indisvalid from pg_index i + join pg_class c on c.oid = i.indexrelid where c.relname = 'ix_chunks_embedding_hnsw';" + +# the recovery for BOTH cases is the same command: +uv run alembic upgrade head +``` + +* `indisvalid = f` after a killed build → `upgrade head` drops the INVALID + remnant and rebuilds (the migration's own self-heal). +* `relation "ix_chunks_embedding_hnsw" already exists` → the build committed + but the version stamp did not land. `0029` handles the INVALID case; + a VALID leftover of the same name is dropped by hand and re-run: + `DROP INDEX CONCURRENTLY ix_chunks_embedding_hnsw` then + `alembic upgrade head`. +* Watching a long build from a second terminal: + `select * from pg_stat_progress_create_index;` +* Rolling back is cheap and touches no data: + `uv run alembic downgrade -1` is `DROP INDEX CONCURRENTLY IF EXISTS` — an + index is a derived structure, and the schema underneath is 0028's exactly. + +**After it lands.** Turn on the index in the app's own terms and confirm the +owner-visible parity, in this order: + +```bash +uv run python -m scripts.eval_retrieval --from-file tests/fixtures/retrieval_battery.txt +uv run python -m scripts.api_payload_audit # the app must be running +``` + +The battery's verdicts should read exactly as they did before the build +(the A7/A8 contract — `top_score` included). The tier's own speed is the +part that changed: **~550 ms → ~12–15 ms** per cosine search at production +order, and the plan flips from `Limit → Sort → Seq Scan on chunks` to +`Limit → Nested Loop → Index Scan on ix_chunks_embedding_hnsw` — verifiable +with `EXPLAIN` on the tier's statement, or by reading +`tests/integration/test_chunks_hnsw_recall.py`'s pin of the same thing. + +**Order of operations on a 4 GB-class box.** Land the phase 183–187 memory +work first (`passive_deletes` cascade, the session-release cadence, the +prune's six-column read, the deferred-`content` fast path, the reported +`rss_mb`/`rss_peak_mb` pair — see +[Operational notes: the sync's memory](#operational-notes-the-syncs-memory-phase-187)). +The box that OOM-killed the 2026-10-15 sync is the box this build lands on; +1.6 GB of new index plus a graph build on top of a sync that still holds its +walked documents is how a build gets killed and leaves an INVALID graph for +somebody else to notice. + **Reverse-proxy body size:** a legal chat/save POST is now large — one conversation can carry a full-length answer (see [Conversation payload limits](#conversation-payload-limits)) — so an diff --git a/alembic/versions/0029_chunks_embedding_hnsw.py b/alembic/versions/0029_chunks_embedding_hnsw.py new file mode 100644 index 0000000..dfc6d6a --- /dev/null +++ b/alembic/versions/0029_chunks_embedding_hnsw.py @@ -0,0 +1,125 @@ +"""chunks.embedding HNSW index: cosine search stops full-scanning (phase 188, task 01) + +Revision ID: 0029 +Revises: 0028 +Create Date: 2026-10-15 + +The 2026-10-15 incident investigation found ``pg_indexes`` for ``chunks`` +listing only the PK, ``ix_chunks_document_id`` and ``ix_chunks_tsv`` — +**no index on ``embedding``** — so every chat turn's A7 cosine tier +(``ORDER BY embedding <=> :vec LIMIT 100``) runs as a sequential scan of +a ~3 GB, 414,220-row relation: per-turn CPU on the interactive path plus +page-cache churn on a 3,904 MB box the sync also needs. + +* ``ix_chunks_embedding_hnsw`` — HNSW over the existing ``vector(768)`` + column (A6 unchanged) with ``vector_cosine_ops``, so the ``<=>`` + operator in ``_vector_candidates`` uses it with NO query rewrite (A3's + one-system rule: no new service, no new dependency). ``m = 16`` and + ``ef_construction = 64`` are the pgvector defaults' neighbourhood, + which keeps the graph buildable on a 4 GB box. ``halfvec(768)`` was + considered and REJECTED for this phase (it needs a statement cast plus + its own quantisation-recall study — two quality risks in one phase). + +**Why ``CONCURRENTLY`` inside an autocommit block.** The index serves a +live interactive path; a plain ``CREATE INDEX`` would hold a lock that +blocks writes to ``chunks`` for the whole (long) build. ``CREATE INDEX +CONCURRENTLY`` cannot run inside a transaction, so the DDL executes in +``op.get_context().autocommit_block()``. + +**The failure mode this migration guards (D2).** A ``CONCURRENTLY`` +build that dies mid-way — killed by the OOM killer on a 3,904 MB box, +cancelled, or disconnected — leaves the index behind marked INVALID in +``pg_index``: it costs insert time on every sync and serves no queries, +*SILENTLY*. On this hardware that is the expected failure, not the +exotic one. Therefore ``upgrade()`` (a) pre-drops any existing INVALID +index of this name before building (otherwise the rebuild dies on +``relation "ix_chunks_embedding_hnsw" already exists``), and (b) +re-checks ``pg_index.indisvalid`` after the build and raises if the new +index did not validate — a migration that reports success while leaving +an INVALID index would paper over exactly the silent failure above. + +Pure helpers (unit-testable, no I/O): :func:`should_drop_invalid` and +:func:`is_valid` are total functions over the ``indisvalid`` lookup +(``None`` = index absent). + +**Reversibility (A13).** The downgrade is +``DROP INDEX CONCURRENTLY IF EXISTS`` — an index is a derived structure, +so the drop touches no data and 0028's schema is byte-identical after +it. No ``chunks``/``documents`` column changes and no data rewrite: the +migration is cheap on a healthy box and interruptible (the INVALID +self-heal makes re-running it the recovery path). The production build +(414k × 768) is an operator action with a pre-flight runbook — see +README "Production Deployment". +""" +from __future__ import annotations + +from sqlalchemy import text +from sqlalchemy.engine import Connection + +from alembic import op + +revision = "0029" +down_revision = "0028" +branch_labels = None +depends_on = None + +INDEX_NAME = "ix_chunks_embedding_hnsw" + + +def should_drop_invalid(indisvalid: bool | None) -> bool: + """True only when the index EXISTS but is INVALID (phase 188 D2). + + ``None`` (absent) needs no drop; ``True`` (valid) must be left + alone; ``False`` is an interrupted ``CREATE INDEX CONCURRENTLY`` + remnant that would otherwise block the rebuild with + ``relation already exists``. + """ + return indisvalid is False + + +def is_valid(indisvalid: bool | None) -> bool: + """True only when the index EXISTS and validated (phase 188 D2).""" + return indisvalid is True + + +def _lookup_index_validity(bind: Connection) -> bool | None: + """``pg_index.indisvalid`` for :data:`INDEX_NAME`, ``None`` if absent.""" + row = bind.execute( + text( + "SELECT i.indisvalid" + " FROM pg_index i" + " JOIN pg_class c ON c.oid = i.indexrelid" + " WHERE c.relname = :name" + ), + {"name": INDEX_NAME}, + ).scalar() + return None if row is None else bool(row) + + +def upgrade() -> None: + # CONCURRENTLY DDL cannot run inside a transaction — the autocommit + # block commits alembic's surrounding transaction, runs the DDL + # statement-by-statement, and reopens one afterwards. + with op.get_context().autocommit_block(): + # Self-heal an interrupted build (D2): drop the INVALID remnant + # BEFORE creating, or the CREATE dies on "already exists". + if should_drop_invalid(_lookup_index_validity(op.get_bind())): + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") + op.execute( + f"CREATE INDEX CONCURRENTLY {INDEX_NAME} ON chunks " + "USING hnsw (embedding vector_cosine_ops) " + "WITH (m = 16, ef_construction = 64)" + ) + # Post-validate: a CONCURRENTLY build that dies leaves an + # INVALID index silently — fail the migration loudly instead + # (the INVALID self-heal above makes re-running the recovery). + if not is_valid(_lookup_index_validity(op.get_bind())): + raise RuntimeError(f"{INDEX_NAME} did not validate — drop it and re-run") + + +def downgrade() -> None: + # Same autocommit requirement as the create. IF EXISTS keeps the + # downgrade idempotent; no data is touched (A13 — an index is a + # derived structure, 0028's schema is byte-identical after the drop). + with op.get_context().autocommit_block(): + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}") diff --git a/app/config.py b/app/config.py index 03ee129..5c2156f 100644 --- a/app/config.py +++ b/app/config.py @@ -349,6 +349,44 @@ class Settings(BaseSettings): # live 2774-chunk KB for "How did I install gitlab?" (gitlab.md:1 at # vrank 100 / lrank 3 → fused 0.0221 vs the template's 0.0164). hybrid_vector_candidates: int = 100 + #: HNSW search depth for the cosine tier (phase 188, D3/LOCKED A4): + #: ``SET LOCAL hnsw.ef_search`` immediately before + #: ``app.rag.retriever._vector_candidates``'s SELECT — per transaction, + #: so no request leaks state into another and no global GUC (nor + #: connection-pool) setting is touched. A value BELOW the A7 candidate + #: count cannot be honoured — it would return FEWER candidates than A7 + #: promises (a silent recall regression) — so the retriever CLAMPS it up + #: to the candidate count with one WARNING line instead. ``0`` is refused + #: here for the same reason: the clamp would silently rewrite it, which + #: is exactly what an operator reading ``0`` as "off" would not expect. + #: Negative values fail startup loudly (the ``show_text_max_*`` validator + #: pattern, phase 182). + #: + #: Why 400 and not the candidate count (task 04's live battery, the + #: 1,000-document live replica, 8,866 real embeddings, the 7-question + #: battery with the vectors PINNED so only the index differs): + #: + #: * ``ef_search = 100`` (``== LIMIT``, the floor the clamp enforces) + #: recovers mean **0.9814** / min **0.94** of the exact top-100 — under + #: this phase's own 0.99 recall floor (LOCKED A3: the remedy for a + #: recall miss is a higher ``ef_search``, never a relaxed assertion). + #: Production holds 47× the chunks, where a fixed ``ef_search`` is + #: weaker still, not stronger. + #: * ``ef_search = 400`` recovers **1.0000** of the exact top-100 with a + #: max per-candidate ``top_score`` delta of exactly ``0.0`` — the + #: honesty gate (A8) reads that number, and no verdict, top-5 set or + #: suggested list moved at ANY depth measured. + #: * it is also the depth at which the PLANNER actually uses the index on + #: a small relation: at ``ef_search = 100`` Postgres costs the HNSW scan + #: above a sequential one and full-scans anyway (measured 51.5 ms vs the + #: forced-index 2.6 ms); at 400 the plan names + #: ``ix_chunks_embedding_hnsw`` at 5.6 ms — still ~9× under the + #: sequential scan this phase exists to remove. + #: + #: The A7 clamp floor stays at ``hybrid_vector_candidates`` (100): with a + #: 400 default a stock install never warns. Postgres caps ``ef_search`` at + #: 1000. + retrieval_ef_search: int = 400 hybrid_lexical_candidates: int = 30 rrf_k: int = 60 #: Recency boost on the RRF-fused retrieval score (phase 106, D6): the @@ -797,6 +835,21 @@ class Settings(BaseSettings): ) return v + @field_validator("retrieval_ef_search") + @classmethod + def _retrieval_ef_search_positive(cls, v: int) -> int: + """``0``/negative is a typo, not a switch (phase 188, D3). There is + no ``ef_search=0`` to turn the index off — the retriever clamps any + value below the A7 candidate count UP to it, so a ``0`` that read as + "off" would silently become 100 instead of failing. Fail loud at + startup, naming the field (the ``show_text_max_chars`` pattern).""" + if v <= 0: + raise ValueError( + "retrieval_ef_search must be > 0 (HNSW ef_search; the " + "retriever clamps values below the candidate count up to it)" + ) + return v + @field_validator("embed_question_max_chars") @classmethod def _embed_question_max_chars_positive(cls, v: int) -> int: diff --git a/app/core/settings_catalog.py b/app/core/settings_catalog.py index 076af84..3e0a222 100644 --- a/app/core/settings_catalog.py +++ b/app/core/settings_catalog.py @@ -205,6 +205,9 @@ CATALOG: dict[str, SettingMeta] = { "Retrieval & honesty", "Hybrid vector candidates", ), + "retrieval_ef_search": SettingMeta( + "retrieval_ef_search", "Retrieval & honesty", "Vector ef_search" + ), "hybrid_lexical_candidates": SettingMeta( "hybrid_lexical_candidates", "Retrieval & honesty", diff --git a/app/rag/retriever.py b/app/rag/retriever.py index e7f4c0f..46708d0 100644 --- a/app/rag/retriever.py +++ b/app/rag/retriever.py @@ -91,6 +91,7 @@ Deterministic tie-break for equal fused scores: """ from __future__ import annotations +import logging import math import re import unicodedata @@ -107,6 +108,12 @@ from app.config import get_settings from app.models import Chunk, Document from app.schemas import SourceRef +#: Phase 188 (D3): the ONE logger of this module — its only line so far is +#: the ``ef_search`` clamp WARNING (the retrieval path was silent by +#: design; a knob that silently degraded recall is exactly what needs a +#: line in the server log). +logger = logging.getLogger("app.rag.retriever") + #: Shared overflow marker (phase 15; imported by ``app.rag.prompts``) #: — used by the steering () section, the phase-118 NULL-summary #: suggestion preview fallback (A5), and the capped agent ``read`` @@ -436,7 +443,31 @@ def _vector_candidates( (two-phase import in progress) are skipped. Each candidate carries its ``Chunk.is_summary`` flag (phase 30) so a summary hit stays identifiable after fusion. + + Phase 188 (D3, LOCKED A4): the HNSW tier's search depth is set + EXPLICITLY with ``SET LOCAL hnsw.ef_search`` immediately before the + SELECT — never a global ``SET``, so the value lives and dies with the + request's transaction (nothing leaks into another request, nothing + survives a pool recycle, no connection-pool-wide state). The + ``ef_search`` a turn actually gets is + ``max(settings.retrieval_ef_search, limit)``: an ``ef_search`` BELOW + the A7 candidate count would return FEWER candidates than the cosine + tier promises (a silent recall regression the honesty gate can turn + into a deflect), so it is CLAMPED up to *limit* with one WARNING line + instead of being honoured. The value is ``int()``-coerced before it + reaches SQL — no operator-supplied string is ever interpolated + (:func:`app.config.Settings.retrieval_ef_search` refuses ``0`` and + negatives at startup). The setting is read through + :func:`app.config.get_settings` exactly like every other retrieval + knob in this module. """ + ef_search = get_settings().retrieval_ef_search + if ef_search < limit: + logger.warning( + "retrieval: ef_search=%d raised to %d (A7 candidate count)", ef_search, limit + ) + ef_search = limit + db.execute(text(f"SET LOCAL hnsw.ef_search = {int(ef_search)}")) distance = Chunk.embedding.cosine_distance(question_embedding) rows = db.execute( select(Chunk, distance.label("distance"), Document) diff --git a/benchmarks/README.md b/benchmarks/README.md index 52c7dcc..c25bdb1 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -32,3 +32,13 @@ CSV recordings from all model tests (chat, summary, embedding). - `scripts/test_summary_model.py` — summary model quality benchmark - `scripts/test_embed_model.py` — embedding model dimension/cosine/speed benchmark - `scripts/model_benchmark.py` — shared CSV recorder (imported by all scripts) + +## Retrieval gate + +- [`phase188_live_gate/`](phase188_live_gate/README.md) — the HNSW index (phase 188) + measured OFF vs ON on the restored 1,000-document live-replica KB and the live + embedding endpoint: the two `eval_retrieval` captures (byte-identical), the + deterministic pinned-vector gate, the `ef_search` depth sweep that set the shipped + default to 400, the production-order build cost (161 s / 1.6 GB for 417k chunks) + and the `maintenance_work_mem` shared-memory failure. Captures in that directory; + the probe is `.agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py`. diff --git a/benchmarks/phase188_live_gate/README.md b/benchmarks/phase188_live_gate/README.md new file mode 100644 index 0000000..7eb3ee8 --- /dev/null +++ b/benchmarks/phase188_live_gate/README.md @@ -0,0 +1,185 @@ +# Phase 188 task 04 — the live recall / parity gate (index OFF vs ON) + +Measured on **live production** (`brain.reeseapps.com`'s database, reached at +`localhost:5432` through the dev-tunnel the app uses), on the **live embedding +endpoint**, against a restored **1,000-document live-replica KB**. Nothing here +ran against the synthetic 5,000-row fixture — that corpus belongs to task 03 and +it is not a proxy for the shapes real embeddings take. + +Reproduce with `.agents/reports/188_chunks_hnsw_index/task04_recall_ab_probe.py` +(its `--help` has the four commands); the battery itself is +`uv run python -m scripts.eval_retrieval --from-file tests/fixtures/retrieval_battery.txt`. + +## Prerequisites (measured, not assumed) + +| Check | Value | +| --- | --- | +| `SELECT extversion FROM pg_extension WHERE extname = 'vector'` | **0.8.0** (>= 0.5.0 required; `hnsw` available) | +| `alembic current` before | head (0028), index absent | +| Replica corpus | 1,000 documents, **8,866 chunks**, 8,866 embeddings, heap **13 MB** (from a 95 MB plain-SQL dump) | +| `maintenance_work_mem` | **64 MB** (the container's default — `postgres:17` never got the phase's 512 MB, see below) | +| `/dev/shm` in `brain-of-reese_db_1` | **63 MB** | +| Planner costs | `seq_page_cost = 1`, `random_page_cost = 4` (default, SSD-class) | +| Disk free on the Postgres data volume | **2.2 TB free of 3.7 TB** (41 % used) | +| Production relation, from the 2026-10-15 incident record (not measured in this task) | 414,220 chunks, heap **3,000 MB** | +| Index after `alembic upgrade head` | `ix_chunks_embedding_hnsw` **valid**, **34 MB** on the replica | +| `alembic_version` inside the dump | **`0011`** — so `alembic upgrade head` replays **0012…0029** into the restored corpus and 0029's `CREATE INDEX CONCURRENTLY` builds against **real** embeddings. Restoring into a schema already at head would build nothing and measure nothing. | + +## The battery, index OFF vs index ON (`eval_retrieval`) + +`battery_index_off.txt` was captured with the index dropped +(`uv run alembic downgrade -1`), `battery_index_on.txt` with it present +(`uv run alembic upgrade head`). Both runs embed the seven battery questions +through the live endpoint. + +The two captures are **byte-identical** — every `best_cosine`, every `fts_hits`, +every one of the five numbered hits per question, and every `DEFLECT`/`HIGH` +verdict. The only difference is the plan the server chose, which battery output +does not print (the deterministic probe below does). + +**Why the battery alone cannot gate this.** At `ef_search = 100` its OFF and ON +captures disagreed by up to ~3e-4 on printed cosines — above this phase's 1e-4 +tolerance — and the battery cannot say where that came from. The probe can: the +endpoint is deterministic (`endpoint_self_jitter`, one question embedded twice, +measured **0.000e+00**), and the tier reading shows the mechanism — at that depth the +two top-100s agree on only 0.9814 of rows on average (0.94 worst), a substituted tail +candidate shifts the RRF ranks, and since a document contributes several chunks a +printed hit can keep its document while changing its chunk, moving the printed +cosine while the top-5 list and `top_score` hold. At the shipped 400 the two +captures are byte-identical. Reading all 100 rows with the vectors pinned is what +separates the two effects. So the gate ran +a second time with the question vectors **pinned** (`embed` once, then +`measure` both sides from the cache): the 1e-4 tolerance was kept, and the input +was made identical instead of the assertion weakened (LOCKED A3). + +## The deterministic gate (pinned vectors, same code path) + +`deterministic_gate_compare.txt` — A = index dropped (the pre-phase behaviour: +`Limit → Sort → Hash Join → Seq Scan on chunks`), B = index present at the +shipped `ef_search`, question vectors identical on both sides: + +| Check | Result | +| --- | --- | +| A8 verdicts (all 7) | **identical** — every question `HIGH (answer)`, the out-of-KB Mongolia probe included (`best_cosine=0.6070`, under the 0.62 threshold, but A8 deflects only when the cosine is under it **and** there is no full-text support — it has 5 FTS hits, unchanged by this index) | +| Top-5 document sets **and order** (all 7) | **identical** | +| Suggested documents (all 7) | **identical** | +| `top_score` / `best_cosine` delta | **0.000e+00** (gate ≤ 1e-4) | +| Cosine-tier score delta, ranks 1–5 (the fused window) | **0.000e+00** (gate ≤ 1e-4) | +| Cosine-tier top-100 **candidate-set overlap** | **1.0000 on all 7** (floor 0.99) | +| Rank swaps that are not exact cosine ties | **0** (every swap is a tie broken by access path) | +| Tier latency (median over questions) | **38.3 ms → 15.3 ms** | +| Plan, index present | `Limit → Nested Loop → Index Scan on ix_chunks_embedding_hnsw` | + +`deterministic_gate_recall.txt` — the same tier forced onto the index with +`enable_indexscan = on` at each depth, recall@100 against the exact scan, on +**real embeddings**: + +| `ef_search` | mean recall@100 | min recall@100 | max per-chunk cosine delta | max `top_score` delta | tier latency | +| --- | --- | --- | --- | --- | --- | +| 100 (the A7 candidate count — task 02's default) | 0.9814 | 0.94 | 2.195e-03 | **0.000e+00** | 13.8 ms | +| 200 | 1.0000 | 1.0000 | 0.0 | 0.000e+00 | 14.1 ms | +| 400 (**shipped**) | 1.0000 | 1.0000 | 0.0 | 0.000e+00 | 14.6 ms | + +Two HNSW graph builds were measured (the index is dropped and rebuilt between the +OFF and ON runs). The 100-row candidate set at `ef_search = 100` came out +0.9814 mean / 0.94 min on one build and 0.9843 / 0.94 on the other; `200` came +out 1.0000 / 1.0000 and 0.9971 / 0.99; `400` came out 1.0000 / 1.0000 both times. + +## Decision: `BOR_RETRIEVAL_EF_SEARCH` ships at 400 + +* At the task-02 default (`ef_search = 100 == LIMIT`) the **user-visible** numbers + did not move at all — verdicts, top-5 sets, suggested documents and + `top_score` were identical, to `0.000e+00`. So the task's parity gate passes at + 100 and the deflection calibration was never in question. +* What moved was the **100-row candidate set** the cosine tier hands the RRF + fusion: mean 0.9814, min 0.94 — below this phase's own acceptance floor + (mean ≥ 0.99, min ≥ 0.95), and matching the counter-case task 03 recorded for + high-effective-rank data. One substitution at rank ~89 of 100 was enough to + move 22 tail ranks. +* LOCKED A3 names exactly one remedy: **raise `ef_search`** (towards 400). The + thresholds were not touched. `200` measured borderline (1.0000 on one graph, + 0.9971 on another); **400 measured exact on both**, and on real embeddings it + costs **0.5 ms** (14.1 → 14.6 ms) against the depth-100 search and **23 ms** + against the sequential scan it replaces. +* The default in `app/config.py` is therefore **400**, still overridable with + `BOR_RETRIEVAL_EF_SEARCH`, and phase 188's clamp log line still fires if an + operator sets it below the candidate count. + +## Production-order probe (`production_order_*.log`) + +The replica's 8,866 chunks are 13 MB of heap, which makes it the wrong place to +ask "does production use the index, and what does it cost?": the relation is an +order of magnitude smaller than the one that produced the incident, and pgvector +costs an HNSW scan partly on `ef_search`, so a small-relation plan does not +transfer. A throwaway database (`bor_scale188b`, created and dropped during +this task) answered it at the production row count, using the replica's own +embeddings scaled ×47 with each copy multiplied element-wise by a random +near-one jitter vector, so all **416,702** vectors are distinct (415,950 unique) +and the heap is 557 MB: + +| Measurement | Value | +| --- | --- | +| Build through the real migration (`uv run alembic upgrade head`, `maintenance_work_mem = 64 MB`) | **161 s**, peak RSS of the alembic process 81 MB | +| Index size | **1,625 MB** (vs a 557 MB heap — HNSW at `m = 16`, 768-dim float4 costs ~3.9 KB/row: 414k rows ⇒ plan for ~1.6 GB) | +| Plan at `ef_search` 100 / 200 / 400 | **HNSW index scan at every depth** (index path cost 3,333 / 5,621 / 9,806 vs a seq-scan-and-sort alternative an order of magnitude above) | +| Tier latency | **~550 ms sequential → 12.0–14.9 ms** with the index (≈ **40×**), the ~1 ms `SET LOCAL` included | +| Top-100 overlap vs the exact scan | 0.23–0.64, **not a valid recall measurement** — see the caveat below | + +**Caveat, recorded because it is a limitation of this gate.** A corpus built by +jittering 8,866 real embeddings into 47 near-copies each is a pathological graph +for a top-100 recall metric: "the exact top-100" is then 100 knife-edge slots +inside a handful of near-duplicate clusters, which is not the shape of a real KB. +The numbers are reported so the latency, plan and build-cost readings at +production row count are on the record, and the **recall** verdict is taken from +the live replica, where every vector is a real, distinct chunk. After the +production build, the owner should re-run the battery on the real 414k-chunk KB +exactly as this task did on the replica; `tests/integration/test_chunks_hnsw_recall.py` +now fails (rather than passing vacuously) if a search at the shipped depth is not +actually reading the index. + +## Task 05 re-verification — the battery at the shipped default, from scratch + +`task05_battery_index_off.txt` / `task05_battery_index_on.txt` are a clean-room +re-run of the same gate, performed by task 05 to confirm the verdict still holds +at the shipped `BOR_RETRIEVAL_EF_SEARCH = 400` before the phase shipped. Nothing +was reused from task 04's state: the dump was restored into a **throwaway +database** (`bor_p188gate`, created and dropped inside the run, so the dev +database was never touched), which lands at `alembic_version = 0011` and was +taken to **`0028`** (index absent — the OFF arm), then to **`head`** through the +real migration (0029 built `ix_chunks_embedding_hnsw` against the 8,866 real +embeddings: **valid**, **34 MB**, ~1 s wall clock, `alembic` peak RSS **81 MB** +— the ON arm). Both arms ran +`uv run python -m scripts.eval_retrieval --from-file tests/fixtures/retrieval_battery.txt` +with live embeddings against live Postgres. + +**The two captures are byte-identical** (Python string equality, not a diff +summary): all seven `best_cosine` values, all seven `fts_hits` counts, all 35 +numbered hits with their scores, and all seven gate verdicts — five `HIGH`, the +Mongolia probe `HIGH` on its 5 FTS hits with `best_cosine = 0.6070` under the +0.62 threshold, and the two Besta questions behaving as out-of-KB asks on a +replica that has no IKEA document (`grep -c -i ikea` in the dump = 0, verified +before drawing any conclusion from them). + +| Arm | Schema state | Index | Battery capture | +| --- | --- | --- | --- | +| OFF | `0028` | absent | `task05_battery_index_off.txt` | +| ON | `0029` (head) | valid, 34 MB | `task05_battery_index_on.txt` | +| Equality | — | — | **identical, byte for byte** | + +## Finding: `maintenance_work_mem = 512MB` cannot be used in this container + +``` +ERROR: could not resize shared memory segment "/PostgreSQL.3366409952" to 533761504 bytes: No space left on device +``` + +`brain-of-reese_db_1` has a **63 MB `/dev/shm`**, so any `maintenance_work_mem` +above ~63 MB makes a parallel HNSW build die, and the failed build leaves an +**INVALID** index behind. This is a deviation from D5, which suggested +`SET maintenance_work_mem = '512MB'`: on this host the value is not applicable, +and the container's **64 MB default builds the index successfully**. The failure +was observed live at production order, and it exercised the self-heal for real: +migration 0029 detected `indisvalid = false`, logged +`dropping INVALID … rebuilding`, dropped it and rebuilt it valid. The task's +recorded command is therefore "run `alembic upgrade head` as-is"; raising +`maintenance_work_mem` is only worth doing after `/dev/shm` is raised too +(`podman run --shm-size=1g`), and a raise that fails is recoverable. diff --git a/benchmarks/phase188_live_gate/battery_index_off.txt b/benchmarks/phase188_live_gate/battery_index_off.txt new file mode 100644 index 0000000..bf55656 --- /dev/null +++ b/benchmarks/phase188_live_gate/battery_index_off.txt @@ -0,0 +1,64 @@ +eval: threshold=0.62 vector_candidates=100 lexical_candidates=30 rrf_k=60 recency_boost=0.0007 recency_half_life_days=365 name_hit_bonus=0.005 + +question: 'How do I deploy gitea?' + gate: best_cosine=0.8528 fts_hits=38 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml cosine=0.7640 fts=1 effective=0.03056 created=2026-10-17 (playbook) + 2. deploy/Deployments/reeseapps/gateway/README.md cosine=0.7736 fts=1 effective=0.02902 created=2026-10-17 (Proxy Server) + 3. homelab/active/container_gitea/gitea.md cosine=0.7612 fts=1 effective=0.02561 created=2026-10-17 (Gitea) + 4. homelab/active/kubernetes_gitea/gitea-values.yaml cosine=0.7232 fts=1 effective=0.02512 created=2026-10-17 (gitea-values) + 5. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 cosine=0.7159 fts=1 effective=0.02495 created=2026-10-17 (gitea-postgres.env) + suggested: 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml 2. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 3. deploy/Deployments/reeseapps/gateway/README.md 4. deploy/Deployments/reeseapps/gitea/README.md 5. deploy/Deployments/reeseapps/gitea/app/gitea-web.env.j2 + +question: 'How do I deploy forgejo?' + gate: best_cosine=0.7876 fts_hits=37 -> HIGH (answer) + 1. homelab/templates/container/foobar.md cosine=0.7876 fts=0 effective=0.01709 created=2026-10-17 (Podman foobar) + 2. deploy/Deployments/reeseapps/forgejo/README.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Reeseapps Forgejo) + 3. deploy/Deployments/reeseapps/README.md cosine=0.7857 fts=0 effective=0.01683 created=2026-10-17 (Deployments) + 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (forgejo-app) + 5. homelab/active/container_bifrost/bifrost.md cosine=0.7672 fts=0 effective=0.01657 created=2026-10-17 (Podman bifrost) + suggested: 1. deploy/Deployments/reeseapps/forgejo/README.md 2. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container 3. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.env.j2 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.container 5. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.env.j2 + +question: 'How do I set up the gateway?' + gate: best_cosine=0.7816 fts_hits=37 -> HIGH (answer) + 1. homelab/active/device_shelly/shelly.md cosine=0.6940 fts=1 effective=0.02509 created=2026-10-17 (Shelly Devices) + 2. deploy/Deployments/reeseapps/forgejo/app/forgejo.network cosine=0.6727 fts=1 effective=0.02050 created=2026-10-17 (forgejo) + 3. homelab/active/device_framework_16/framework_16.md cosine=0.7816 fts=0 effective=0.01709 created=2026-10-17 (Framework Laptop 16) + 4. deploy/Deployments/delta/istio/gateway-metallb.yaml cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (gateway-metallb) + 5. homelab/active/device_home_assistant/home_assistant.md cosine=0.7752 fts=0 effective=0.01683 created=2026-10-17 (Home Assistant) + suggested: 1. homelab/active/device_shelly/shelly.md 2. deploy/Deployments/delta/istio/gateway-metallb.yaml 3. deploy/Deployments/delta/istio/test-gateway.yaml 4. deploy/Deployments/reeseapps/gateway/README.md 5. deploy/Deployments/reeseapps/gateway/certbot/playbook.yaml + +question: 'Do you have notes on the qwen 3.8 27b model?' + gate: best_cosine=0.7483 fts_hits=31 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container cosine=0.7402 fts=1 effective=0.03322 created=2026-10-17 (qwen3.8-27b-epic-vulkan) + 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container cosine=0.7142 fts=1 effective=0.03058 created=2026-10-17 (qwen3.8-27b-juggernaut-vulkan) + 3. homelab/active/container_localai/localai.md cosine=0.6837 fts=1 effective=0.02570 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/software_ai_stack/ai_stack.md cosine=0.7029 fts=1 effective=0.02545 created=2026-10-17 (Self Hosted AI Stack) + 5. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container cosine=0.6675 fts=1 effective=0.02491 created=2026-10-17 (qwen3.8-flash-next) + suggested: 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container 3. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container 4. homelab/active/container_localai/localai.md 5. homelab/active/software_ai_stack/ai_stack.md + +question: 'What is the capital of Mongolia?' + gate: best_cosine=0.6070 fts_hits=5 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/README.md cosine=0.5588 fts=1 effective=0.02382 created=2026-10-17 (Deployments) + 2. homelab/active/container_nginx/nginx.md cosine=0.6070 fts=0 effective=0.01709 created=2026-10-17 (Ngnix) + 3. homelab/active/container_localai/localai.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/os_arch/arch-workstation.md cosine=0.6059 fts=0 effective=0.01683 created=2026-10-17 (Workstation) + 5. homelab/active/software_k3s/k3s.md cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (K3S) + suggested: 1. deploy/Deployments/reeseapps/README.md 2. homelab/active/container_nginx/nginx.md 3. homelab/active/container_localai/localai.md 4. homelab/active/os_arch/arch-workstation.md 5. homelab/active/software_k3s/k3s.md + +question: 'How do I build a Besta?' + gate: best_cosine=0.7078 fts_hits=33 -> HIGH (answer) + 1. homelab/active/container_bifrost/bifrost.md cosine=0.7078 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 2. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 3. vibe-bot/README.md cosine=0.6962 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 4. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + 5. vibe-bot/build.sh cosine=0.0000 fts=1 effective=0.01657 created=2026-10-17 (build) + suggested: 1. homelab/active/software_distoolbox/arch-build.sh 2. homepage/build.sh 3. vibe-bot/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md + +question: 'How do I build the BESTÅ?' + gate: best_cosine=0.6969 fts_hits=33 -> HIGH (answer) + 1. vibe-bot/build.sh cosine=0.6267 fts=1 effective=0.02519 created=2026-10-17 (build) + 2. homelab/active/container_bifrost/bifrost.md cosine=0.6969 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 3. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 4. vibe-bot/README.md cosine=0.6887 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 5. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + suggested: 1. vibe-bot/build.sh 2. homelab/active/software_distoolbox/arch-build.sh 3. homepage/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md diff --git a/benchmarks/phase188_live_gate/battery_index_on.txt b/benchmarks/phase188_live_gate/battery_index_on.txt new file mode 100644 index 0000000..bf55656 --- /dev/null +++ b/benchmarks/phase188_live_gate/battery_index_on.txt @@ -0,0 +1,64 @@ +eval: threshold=0.62 vector_candidates=100 lexical_candidates=30 rrf_k=60 recency_boost=0.0007 recency_half_life_days=365 name_hit_bonus=0.005 + +question: 'How do I deploy gitea?' + gate: best_cosine=0.8528 fts_hits=38 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml cosine=0.7640 fts=1 effective=0.03056 created=2026-10-17 (playbook) + 2. deploy/Deployments/reeseapps/gateway/README.md cosine=0.7736 fts=1 effective=0.02902 created=2026-10-17 (Proxy Server) + 3. homelab/active/container_gitea/gitea.md cosine=0.7612 fts=1 effective=0.02561 created=2026-10-17 (Gitea) + 4. homelab/active/kubernetes_gitea/gitea-values.yaml cosine=0.7232 fts=1 effective=0.02512 created=2026-10-17 (gitea-values) + 5. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 cosine=0.7159 fts=1 effective=0.02495 created=2026-10-17 (gitea-postgres.env) + suggested: 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml 2. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 3. deploy/Deployments/reeseapps/gateway/README.md 4. deploy/Deployments/reeseapps/gitea/README.md 5. deploy/Deployments/reeseapps/gitea/app/gitea-web.env.j2 + +question: 'How do I deploy forgejo?' + gate: best_cosine=0.7876 fts_hits=37 -> HIGH (answer) + 1. homelab/templates/container/foobar.md cosine=0.7876 fts=0 effective=0.01709 created=2026-10-17 (Podman foobar) + 2. deploy/Deployments/reeseapps/forgejo/README.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Reeseapps Forgejo) + 3. deploy/Deployments/reeseapps/README.md cosine=0.7857 fts=0 effective=0.01683 created=2026-10-17 (Deployments) + 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (forgejo-app) + 5. homelab/active/container_bifrost/bifrost.md cosine=0.7672 fts=0 effective=0.01657 created=2026-10-17 (Podman bifrost) + suggested: 1. deploy/Deployments/reeseapps/forgejo/README.md 2. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container 3. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.env.j2 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.container 5. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.env.j2 + +question: 'How do I set up the gateway?' + gate: best_cosine=0.7816 fts_hits=37 -> HIGH (answer) + 1. homelab/active/device_shelly/shelly.md cosine=0.6940 fts=1 effective=0.02509 created=2026-10-17 (Shelly Devices) + 2. deploy/Deployments/reeseapps/forgejo/app/forgejo.network cosine=0.6727 fts=1 effective=0.02050 created=2026-10-17 (forgejo) + 3. homelab/active/device_framework_16/framework_16.md cosine=0.7816 fts=0 effective=0.01709 created=2026-10-17 (Framework Laptop 16) + 4. deploy/Deployments/delta/istio/gateway-metallb.yaml cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (gateway-metallb) + 5. homelab/active/device_home_assistant/home_assistant.md cosine=0.7752 fts=0 effective=0.01683 created=2026-10-17 (Home Assistant) + suggested: 1. homelab/active/device_shelly/shelly.md 2. deploy/Deployments/delta/istio/gateway-metallb.yaml 3. deploy/Deployments/delta/istio/test-gateway.yaml 4. deploy/Deployments/reeseapps/gateway/README.md 5. deploy/Deployments/reeseapps/gateway/certbot/playbook.yaml + +question: 'Do you have notes on the qwen 3.8 27b model?' + gate: best_cosine=0.7483 fts_hits=31 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container cosine=0.7402 fts=1 effective=0.03322 created=2026-10-17 (qwen3.8-27b-epic-vulkan) + 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container cosine=0.7142 fts=1 effective=0.03058 created=2026-10-17 (qwen3.8-27b-juggernaut-vulkan) + 3. homelab/active/container_localai/localai.md cosine=0.6837 fts=1 effective=0.02570 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/software_ai_stack/ai_stack.md cosine=0.7029 fts=1 effective=0.02545 created=2026-10-17 (Self Hosted AI Stack) + 5. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container cosine=0.6675 fts=1 effective=0.02491 created=2026-10-17 (qwen3.8-flash-next) + suggested: 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container 3. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container 4. homelab/active/container_localai/localai.md 5. homelab/active/software_ai_stack/ai_stack.md + +question: 'What is the capital of Mongolia?' + gate: best_cosine=0.6070 fts_hits=5 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/README.md cosine=0.5588 fts=1 effective=0.02382 created=2026-10-17 (Deployments) + 2. homelab/active/container_nginx/nginx.md cosine=0.6070 fts=0 effective=0.01709 created=2026-10-17 (Ngnix) + 3. homelab/active/container_localai/localai.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/os_arch/arch-workstation.md cosine=0.6059 fts=0 effective=0.01683 created=2026-10-17 (Workstation) + 5. homelab/active/software_k3s/k3s.md cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (K3S) + suggested: 1. deploy/Deployments/reeseapps/README.md 2. homelab/active/container_nginx/nginx.md 3. homelab/active/container_localai/localai.md 4. homelab/active/os_arch/arch-workstation.md 5. homelab/active/software_k3s/k3s.md + +question: 'How do I build a Besta?' + gate: best_cosine=0.7078 fts_hits=33 -> HIGH (answer) + 1. homelab/active/container_bifrost/bifrost.md cosine=0.7078 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 2. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 3. vibe-bot/README.md cosine=0.6962 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 4. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + 5. vibe-bot/build.sh cosine=0.0000 fts=1 effective=0.01657 created=2026-10-17 (build) + suggested: 1. homelab/active/software_distoolbox/arch-build.sh 2. homepage/build.sh 3. vibe-bot/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md + +question: 'How do I build the BESTÅ?' + gate: best_cosine=0.6969 fts_hits=33 -> HIGH (answer) + 1. vibe-bot/build.sh cosine=0.6267 fts=1 effective=0.02519 created=2026-10-17 (build) + 2. homelab/active/container_bifrost/bifrost.md cosine=0.6969 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 3. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 4. vibe-bot/README.md cosine=0.6887 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 5. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + suggested: 1. vibe-bot/build.sh 2. homelab/active/software_distoolbox/arch-build.sh 3. homepage/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md diff --git a/benchmarks/phase188_live_gate/deterministic_gate_compare.txt b/benchmarks/phase188_live_gate/deterministic_gate_compare.txt new file mode 100644 index 0000000..faaea1c --- /dev/null +++ b/benchmarks/phase188_live_gate/deterministic_gate_compare.txt @@ -0,0 +1,19 @@ +A=/tmp/final_off.json index_present=0 ef=400 +B=/tmp/final_on.json index_present=1 ef=400 + 'How do I deploy gitea?': verdict HIGH (answer) / HIGH (answer) | top5 SAME | suggested SAME | candidate-set overlap 1.0000 | best_cosine delta 0.000e+00 | rank swaps 2 (all exact ties) + 'How do I deploy forgejo?': verdict HIGH (answer) / HIGH (answer) | top5 SAME | suggested SAME | candidate-set overlap 1.0000 | best_cosine delta 0.000e+00 | rank swaps 10 (all exact ties) + 'How do I set up the gateway?': verdict HIGH (answer) / HIGH (answer) | top5 SAME | suggested SAME | candidate-set overlap 1.0000 | best_cosine delta 0.000e+00 | rank swaps 10 (all exact ties) + 'Do you have notes on the qwen 3.8 27b model?': verdict HIGH (answer) / HIGH (answer) | top5 SAME | suggested SAME | candidate-set overlap 1.0000 | best_cosine delta 0.000e+00 | rank swaps 10 (all exact ties) + 'What is the capital of Mongolia?': verdict HIGH (answer) / HIGH (answer) | top5 SAME | suggested SAME | candidate-set overlap 1.0000 | best_cosine delta 0.000e+00 | rank swaps 20 (all exact ties) + 'How do I build a Besta?': verdict HIGH (answer) / HIGH (answer) | top5 SAME | suggested SAME | candidate-set overlap 1.0000 | best_cosine delta 0.000e+00 | rank swaps 9 (all exact ties) + 'How do I build the BESTÅ?': verdict HIGH (answer) / HIGH (answer) | top5 SAME | suggested SAME | candidate-set overlap 1.0000 | best_cosine delta 0.000e+00 | rank swaps 17 (all exact ties) + +max top_score (best_cosine) delta = 0.000e+00 (gate <= 1e-4) +max in-window (rank <= 5) tier cosine delta = 0.000e+00 (gate <= 1e-4) +max tier cosine delta over all 100 ranks = 0.000e+00 (reported only) +tail note: ranks below the fused window are gated as the candidate-SET overlap (>= 0.99), not as a score delta: a graph search may hand back a different tail chunk at a near-identical score without moving anything the turn reads. +A plan(exact)=['Limit', 'Sort'] plan(app)=['Limit', 'Sort'] +B plan(exact)=['Limit', 'Sort'] plan(app)=['Limit', 'Nested Loop'] +tier latency ms (median over questions): A exact=37.45 app=38.252 | B exact=39.12 app=15.317 + +GATE PASS: identical verdicts, identical top-5 sets and order, deltas <= 1e-4 diff --git a/benchmarks/phase188_live_gate/deterministic_gate_recall.txt b/benchmarks/phase188_live_gate/deterministic_gate_recall.txt new file mode 100644 index 0000000..351b5c9 --- /dev/null +++ b/benchmarks/phase188_live_gate/deterministic_gate_recall.txt @@ -0,0 +1,20 @@ + forced100: mean recall@100=0.9814 min=0.9400 max per-chunk cosine delta=2.195e-03 max top_score delta=0.000e+00 latency=13.753ms + forced200: mean recall@100=1.0000 min=1.0000 max per-chunk cosine delta=0.000e+00 max top_score delta=0.000e+00 latency=14.111ms + forced400: mean recall@100=1.0000 min=1.0000 max per-chunk cosine delta=0.000e+00 max top_score delta=0.000e+00 latency=14.614ms + + pipeline parity (A8 verdict / top-5 / suggested), planner vs forced index: + 'How do I deploy gitea?': verdict SAME (HIGH (answer) / HIGH (answer)) | top5 SAME | suggested SAME | top_score delta 0.000e+00 + 'How do I deploy forgejo?': verdict SAME (HIGH (answer) / HIGH (answer)) | top5 SAME | suggested SAME | top_score delta 0.000e+00 + 'How do I set up the gateway?': verdict SAME (HIGH (answer) / HIGH (answer)) | top5 SAME | suggested SAME | top_score delta 0.000e+00 + 'Do you have notes on the qwen 3.8 27b model?': verdict SAME (HIGH (answer) / HIGH (answer)) | top5 SAME | suggested SAME | top_score delta 0.000e+00 + 'What is the capital of Mongolia?': verdict SAME (HIGH (answer) / HIGH (answer)) | top5 SAME | suggested SAME | top_score delta 0.000e+00 + 'How do I build a Besta?': verdict SAME (HIGH (answer) / HIGH (answer)) | top5 SAME | suggested SAME | top_score delta 0.000e+00 + 'How do I build the BESTÅ?': verdict SAME (HIGH (answer) / HIGH (answer)) | top5 SAME | suggested SAME | top_score delta 0.000e+00 + + plans (vacuity guards): + exact: ['Limit', 'Sort', 'Hash Join'] + forced index (pipeline): ['Limit', 'Nested Loop', 'Index Scan on ix_chunks_embedding_hnsw'] + + latency ms over questions: exact(seq)=39.12 planner=15.317 forced100=13.753 forced200=14.111 forced400=14.614 + +GATE PASS: the approximate path returns the exact verdicts, top-5s and top_scores diff --git a/benchmarks/phase188_live_gate/production_order_latency.txt b/benchmarks/phase188_live_gate/production_order_latency.txt new file mode 100644 index 0000000..ae57eee --- /dev/null +++ b/benchmarks/phase188_live_gate/production_order_latency.txt @@ -0,0 +1,12 @@ +corpus: 416702 chunks, heap 557 MB, index 1625 MB +index definition: CREATE INDEX ix_chunks_embedding_hnsw ON public.chunks USING hnsw (embedding vector_cosine_ops) WITH (m='16', ef_construction='64') + +--- the planner's own choice at production scale --- + 'How do I deploy gitea?' ef=100:HNSW(3333) ef=200:HNSW(5621) ef=400:HNSW(9806) + 'How do I deploy forgejo?' ef=100:HNSW(3333) ef=200:HNSW(5621) ef=400:HNSW(9806) + 'How do I set up the gateway?' ef=100:HNSW(3333) ef=200:HNSW(5621) ef=400:HNSW(9806) + +--- latency + recall (production order) --- + 'How do I deploy gitea?' exact_seq=549.5ms ef100: 12.4ms overlap=0.230 topΔ=7.9e-02 ef200: 11.9ms overlap=0.230 topΔ=7.9e-02 ef400: 12.5ms overlap=0.230 topΔ=7.9e-02 + 'How do I deploy forgejo?' exact_seq=545.8ms ef100: 12.1ms overlap=0.470 topΔ=0.0e+00 ef200: 12.9ms overlap=0.470 topΔ=0.0e+00 ef400: 12.0ms overlap=1.000 topΔ=0.0e+00 + 'How do I set up the gateway?' exact_seq=553.1ms ef100: 13.4ms overlap=0.640 topΔ=0.0e+00 ef200: 14.9ms overlap=0.640 topΔ=0.0e+00 ef400: 14.3ms overlap=0.640 topΔ=0.0e+00 diff --git a/benchmarks/phase188_live_gate/question_vectors.json b/benchmarks/phase188_live_gate/question_vectors.json new file mode 100644 index 0000000..54f52da --- /dev/null +++ b/benchmarks/phase188_live_gate/question_vectors.json @@ -0,0 +1 @@ +{"questions": ["How do I deploy gitea?", "How do I deploy forgejo?", "How do I set up the gateway?", "Do you have notes on the qwen 3.8 27b model?", "What is the capital of Mongolia?", "How do I build a Besta?", "How do I build the BEST\u00c5?"], "vectors": [[0.026302937418222427, 0.05123845115303993, -0.0055153705179691315, -0.017277421429753304, 0.004555149003863335, -0.020287590101361275, 0.015308717265725136, 0.009419943206012249, -0.023317985236644745, -0.002515502041205764, -0.016471706330776215, 0.022479049861431122, 0.04292447865009308, 0.03328770771622658, 0.01143570989370346, 0.02596910111606121, 0.016295695677399635, -0.00031894523999653757, 0.033235326409339905, 0.012638394720852375, -0.02390509471297264, -0.0035500696394592524, 0.0075850640423595905, -0.022083736956119537, 0.0006219718488864601, 0.003403295995667577, 0.03784072399139404, 0.0045242346823215485, -0.00802010390907526, 0.018658028915524483, -0.04386814311146736, -0.021659577265381813, -0.11201200634241104, -0.053949546068906784, 0.004705369006842375, -0.015419984236359596, -3.0918487027520314e-05, 0.002038068836554885, -0.018277252092957497, -0.29641807079315186, -0.03907594084739685, 0.018041463568806648, 0.05373162403702736, -0.022887330502271652, -0.03993955999612808, -0.0028733322396874428, 0.00706124771386385, -0.008669393137097359, -0.038823679089546204, 0.026483111083507538, 0.0038209198974072933, 0.00882971566170454, 0.023876290768384933, -0.009437254630029202, 0.01832827366888523, -0.01220744103193283, -0.007349374238401651, -0.05950280651450157, 0.01776539348065853, 0.0009701054659672081, -0.023159250617027283, 0.025261307135224342, 0.002452413784340024, 0.03660028427839279, 0.012075387872755527, 0.0005374819156713784, 0.007859003730118275, 0.0881282314658165, -0.003989549819380045, 0.05463375523686409, 0.012211531400680542, -0.020086897537112236, -0.0009724224219098687, -0.020165491849184036, 0.01654633693397045, -0.02895897999405861, 0.011976659297943115, -0.05457787588238716, -0.028940750285983086, -0.02463066391646862, -0.01679110899567604, 0.025541674345731735, -0.018802940845489502, -0.0033142154570668936, -0.01667456328868866, -0.005039270967245102, -0.0017702766926959157, 0.032307934015989304, 0.02363092079758644, -0.005802905187010765, 0.00021920286235399544, -0.02292545512318611, -0.011364566162228584, -0.1137162446975708, 0.030041364952921867, -0.03645177558064461, 0.004898099694401026, -0.012308117002248764, 0.001829299028031528, -0.042553551495075226, -0.007015876937657595, 0.020473135635256767, 0.03240807726979256, -0.059453535825014114, -0.017412785440683365, 0.02826683595776558, -0.0014155538519844413, 0.015011178329586983, -0.026210706681013107, 0.012213352136313915, 0.04168764129281044, -0.01119302213191986, -0.03671428933739662, 0.016532910987734795, 0.03280432149767876, 0.005713918246328831, 0.006685804575681686, -0.024306120350956917, 0.006732720881700516, 0.009256565943360329, -0.019395388662815094, 0.044111449271440506, -0.01234454195946455, -0.020436596125364304, 0.0386277474462986, 0.05091039836406708, -0.04706580191850662, -0.0092827333137393, -0.03366772457957268, -0.001873424625955522, 0.009345440194010735, -0.0044058263301849365, -0.008998947218060493, 0.016690246760845184, 0.030477799475193024, 0.0258348248898983, 0.018931634724140167, -0.009049677290022373, 0.03598121926188469, 0.0038725074846297503, -0.00874223280698061, -0.020672667771577835, -6.780607509426773e-05, -0.015520223416388035, 0.0195979792624712, -0.061003848910331726, -0.042196132242679596, 0.011249005794525146, 0.016711415722966194, -0.008533558808267117, -0.01846833899617195, -0.03283195570111275, -0.001416650484316051, -0.00936006661504507, -0.03515611216425896, -0.021994711831212044, 0.0126108443364501, 0.031238293275237083, -0.020829791203141212, -0.0074201240204274654, 0.0006957162404432893, -0.008774597197771072, 0.0010954835452139378, 0.014733508229255676, -0.01693568378686905, 0.0196567140519619, 0.019145356491208076, -0.053520385175943375, 0.006678327918052673, -0.012795469723641872, -0.00274457223713398, -0.03242684528231621, -0.015173107385635376, -0.013841812498867512, 0.010809591971337795, 0.034152835607528687, -0.048016782850027084, 0.019324401393532753, 0.016622958704829216, 0.006241407245397568, 0.014925760217010975, -0.019270269200205803, -0.004841021727770567, 0.02147641032934189, 0.0004377085715532303, -0.02538430131971836, -0.036201100796461105, -0.013054554350674152, 0.024739278480410576, -0.045158226042985916, -0.021176518872380257, -0.001275839051231742, 0.018786387518048286, 0.004756571725010872, 0.0182273481041193, 0.029572555795311928, 0.036111123859882355, 0.0019336214754730463, -0.03701231628656387, -0.016346311196684837, -0.0009293006150983274, -0.03769315406680107, -0.02837611921131611, -0.011139564216136932, -0.011193592101335526, 0.017133736982941628, -0.016701333224773407, -0.032025568187236786, 0.03357208892703056, 0.022318566218018532, 0.05527660250663757, 0.043568432331085205, 0.0508246123790741, -0.023510238155722618, 0.027967099100351334, 0.0012489877408370376, 0.0584181509912014, 0.007078974042087793, -0.009031067602336407, -0.0073823872953653336, -0.00548882782459259, -0.01854458637535572, -0.01691117323935032, -0.012574896216392517, 0.026443962007761, 0.007070472463965416, 0.014821600168943405, 0.027621235698461533, -0.025452423840761185, -0.016790756955742836, -0.0034510931000113487, 0.02960408851504326, 0.010646731592714787, -0.005525984335690737, -0.002188991056755185, -0.09982965141534805, -0.029954131692647934, 0.024070382118225098, 0.013256967067718506, 0.014109663665294647, 0.03293726220726967, -0.019972501322627068, 0.007100341841578484, 0.00031505298102274537, -0.009973360225558281, -0.00177566718775779, -0.038082022219896317, -0.00039998607826419175, -0.008281437680125237, -0.028756845742464066, -0.04548687860369682, -0.0024599593598395586, 1.676122883509379e-05, -0.0015191452112048864, 0.030894439667463303, -0.0298896636813879, 0.007522790227085352, -0.009362839162349701, -0.02775411307811737, -0.011257164180278778, -0.015382577665150166, -0.020129436627030373, -0.029933921992778778, 0.005852127447724342, -0.002012101234868169, -0.013690224848687649, -0.007140872068703175, 0.008932318538427353, 0.0003700958041008562, -0.002582529094070196, -0.05598534643650055, 0.01421470195055008, -0.010013563558459282, -0.014383625239133835, 0.004452572204172611, -0.02002376690506935, 0.06055818870663643, -0.01354642491787672, 0.030148183926939964, -0.028018085286021233, 0.03701914846897125, -0.019623585045337677, 0.00043954126886092126, -0.011714745312929153, -0.021654410287737846, -0.01671546883881092, -0.03419224172830582, -0.0051111276261508465, -0.014758294448256493, 0.016506753861904144, -0.015743689611554146, 0.03690458834171295, -0.04096297174692154, 0.008621794171631336, 0.027727298438549042, 0.01712978631258011, 0.013592560775578022, -0.006806011311709881, 0.02006118930876255, -0.010901949368417263, 0.007227282039821148, 0.08768058568239212, 0.014526905491948128, -0.014583118259906769, 0.01773041859269142, -0.02928880974650383, 0.0023591734934598207, 0.005782295484095812, 0.009894520975649357, 0.008984951302409172, -0.0028117019683122635, -0.047500863671302795, 0.0055563491769135, 0.008781061507761478, 0.04736006259918213, 0.004868995398283005, 0.028493141755461693, 0.09940528124570847, -0.04227699711918831, -0.011123642325401306, -0.056656721979379654, 0.02101125568151474, 0.038283489644527435, 0.008687509223818779, 0.015718622133135796, 0.013504603877663612, -0.01684615947306156, 0.01915808580815792, 0.007257919758558273, 0.03195985034108162, 0.03870229795575142, 0.044807594269514084, 0.010457796044647694, 0.019625088199973106, 0.009583868086338043, 0.017322899773716927, -0.03340086713433266, 0.008685418404638767, 0.04488525539636612, 0.04780677706003189, 0.044948022812604904, -0.009377708658576012, -0.01137256994843483, -0.005531970877200365, -0.0056325471960008144, 0.006024925969541073, -0.013421648181974888, 0.005920920521020889, 0.017351113259792328, 0.03357405960559845, 0.010818835347890854, -0.019358331337571144, 0.056141383945941925, 0.07449156790971756, -0.011499902233481407, -0.016817882657051086, 0.033411309123039246, -0.006972086150199175, -0.0173345934599638, 0.01981600932776928, 0.002502711955457926, -0.012310878373682499, -0.015117279253900051, 0.01658676750957966, 0.011497065424919128, -0.059846628457307816, 0.04130196571350098, -0.037356313318014145, -0.06204037368297577, 0.01631944626569748, 0.020290106534957886, 0.03592291474342346, -0.04456706717610359, 0.046912405639886856, -0.011155595071613789, 0.0211939699947834, 0.004406968597322702, -0.012844660319387913, -0.01701248809695244, 0.03689426928758621, 0.0052362605929374695, -0.02140757255256176, 0.005754069425165653, -0.03269139677286148, 0.021535253152251244, -0.00867423228919506, 0.011612748727202415, -0.051302023231983185, 0.018910342827439308, 0.008073066361248493, 0.030348381027579308, -0.0420423224568367, 0.03631554916501045, -0.0304265096783638, -0.00035735731944441795, -0.016313284635543823, 0.018630418926477432, -0.028427261859178543, -0.021158766001462936, 0.04306189343333244, -0.0158486757427454, 0.010339217260479927, 0.019857117906212807, -0.004393939394503832, -0.014306941069662571, 0.023371100425720215, 0.008195432834327221, 0.562625527381897, 0.0162268728017807, 0.01186966709792614, -0.019518204033374786, -0.01946239359676838, 0.027258526533842087, -0.02963276021182537, 0.011011125519871712, 0.00714207673445344, 0.013091434724628925, -0.010471588931977749, 0.036983247846364975, -0.004175224807113409, 0.01318584568798542, -0.01167768519371748, -0.022092606872320175, 0.03453638032078743, 0.002386439125984907, -0.07260791212320328, 0.017641840502619743, 0.01699521392583847, -0.028175920248031616, -0.013898379169404507, -0.012069298885762691, 0.05872730910778046, 0.04659545049071312, 0.003316720714792609, 0.03074483387172222, -0.034307632595300674, 0.057469263672828674, -0.012104462832212448, 0.02015097066760063, 0.029186062514781952, 0.041950929909944534, -0.02448255568742752, -0.0012890095822513103, -0.0011254409328103065, 0.016092628240585327, -0.02584066055715084, -0.0009370364714413881, 0.0053518591448664665, 0.019101997837424278, -0.024370653554797173, 0.0010428783716633916, 0.016026847064495087, 0.06157270818948746, -0.016822438687086105, -0.00042973982635885477, 0.005572977941483259, 0.005666588433086872, -0.04983322322368622, 0.009684889577329159, -0.006872628349810839, -0.017264869064092636, -0.019937608391046524, -0.01975845731794834, -0.019750529900193214, 0.012455864809453487, -0.01062337402254343, 4.4928001443622634e-05, -0.01193652767688036, 0.008482587523758411, -0.059707991778850555, -0.06970851868391037, 0.03072180785238743, 0.02283462882041931, 0.008970176801085472, 0.004693606402724981, 0.024275360628962517, 3.7529585824813694e-05, -0.019718918949365616, 0.014275340363383293, -0.02841813676059246, 0.028950896114110947, 0.017746660858392715, -0.013809967786073685, -0.0010461813071742654, 0.015970369800925255, -0.015200874768197536, -0.011862772516906261, -0.0072676739655435085, 0.02653118595480919, -0.010071177035570145, -0.033889103680849075, 0.0069487025029957294, 0.00865574087947607, -0.003935944754630327, 0.016149481758475304, -0.02324322611093521, 0.029701801016926765, -0.015085490420460701, 0.02324226126074791, -0.026442868635058403, 0.0826086774468422, -0.01031245943158865, -0.007324226200580597, 0.0006809404585510492, 0.026371603831648827, -0.06212041154503822, -0.03789766505360603, -0.0049040173180401325, 0.04198361933231354, 0.008261330425739288, 0.004099190700799227, 0.04868212342262268, 0.0699029192328453, -0.022787874564528465, 0.028253676369786263, 0.00922874454408884, 0.02738841623067856, -0.03525858372449875, -0.015996865928173065, 0.041058219969272614, -0.028613541275262833, 0.018792975693941116, 0.012251957319676876, 0.009921484626829624, -0.013643556274473667, -0.059773873537778854, -0.02859402261674404, 0.027777589857578278, -0.03372449055314064, 0.006307120434939861, -0.0034349877387285233, 0.0327092669904232, -0.0014056041836738586, 0.0010713062947615981, -0.02316112071275711, -0.04850905016064644, 0.027918517589569092, -0.023975670337677002, 0.008417296223342419, 0.028278646990656853, -0.02752385288476944, -0.011728066019713879, 0.010406007058918476, -0.03373799845576286, 0.021480653434991837, 0.002971810521557927, -0.00448323180899024, 0.028063494712114334, 0.02933591604232788, 0.02364838868379593, 0.04874718561768532, 0.004510774277150631, -0.02249673195183277, 0.005958141293376684, -0.019140424206852913, -0.07237555831670761, -0.012474237941205502, 0.024021022021770477, 0.017490068450570107, 0.021571774035692215, 0.010401491075754166, -0.031061185523867607, 0.008321821689605713, -0.03230096772313118, 0.026937512680888176, 0.01949910633265972, -0.00210755062289536, 0.02991473115980625, 0.015749042853713036, 0.015344196930527687, 0.03234793618321419, -0.009063511155545712, -0.012768902815878391, -0.023819146677851677, 0.018803738057613373, 0.025186751037836075, 0.012855974026024342, -0.029324328526854515, -0.03570808470249176, 0.01311488077044487, 0.05558100715279579, 0.0045731826685369015, -0.020514756441116333, -0.027791664004325867, -0.04553176462650299, 0.006861016154289246, -0.015520315617322922, 0.028927378356456757, 0.016934914514422417, -0.017337342724204063, -0.004830991383641958, 0.013016771525144577, -0.035704512149095535, -0.010058307088911533, 0.02915012277662754, 0.037231121212244034, -0.035530269145965576, 0.0011884761042892933, 0.001434646314010024, 0.01983674243092537, -0.018278567120432854, 0.025004738941788673, -0.0014071918558329344, 0.017076142132282257, 0.02366071753203869, -0.022985145449638367, 0.013786226511001587, -0.008011899888515472, 0.005342328455299139, -0.09766222536563873, -0.040344323962926865, 0.021766798570752144, -0.0043522072955966, -0.0077648526057600975, -0.003749972442165017, -0.021474270150065422, 0.01361245196312666, -0.054192591458559036, -0.014743528328835964, -0.02146897278726101, 0.0037484136410057545, -0.02638768032193184, -0.000891956384293735, 0.0026697779539972544, -0.027908463031053543, 0.021463921293616295, -0.00660398043692112, -0.0220661461353302, 0.00954052060842514, -0.013364635407924652, 0.010162688791751862, -0.026052484288811684, 0.025580737739801407, 0.004529138095676899, -0.05296655371785164, 0.006051125470548868, 0.0013097175396978855, -0.015262632630765438, 0.0032641650177538395, 0.05305592715740204, -0.028360795229673386, 0.03937193378806114, 0.03476749733090401, -0.030515924096107483, 0.010247365571558475, -0.006692664232105017, 0.006122026592493057, 0.019432544708251953, -0.04145189747214317, 0.008789904415607452, 0.00910231377929449, -0.027027677744627, -0.0015142553020268679, 0.009491169825196266, 0.0017876906786113977, 0.006437459494918585, -0.022530198097229004, -0.0309409461915493, -0.02549438737332821, -0.013583430089056492, 0.007105047348886728, -0.003686329582706094, 0.005569912027567625, 0.04261782392859459, 0.008961866609752178, -0.005338936112821102, -0.022070739418268204, 0.03787815570831299, 0.0033172680996358395, -0.028392886742949486, -0.003275119699537754, 0.00785216037184, -0.00704158004373312, 0.00886816531419754, 0.014292706735432148, 0.041934575885534286, -0.023731674998998642, -0.007563577499240637, -0.018877888098359108, -0.01481578964740038, -0.027887828648090363, -0.013375258073210716, -0.0061761969700455666, -0.012556018307805061, 0.03462933748960495, 0.05204855278134346, -0.0625338926911354, -0.027713311836123466, -0.0010396523866802454, 0.012022712267935276, -0.012295934371650219, 0.005008229054510593, -0.03503292053937912, -0.000267566938418895, 0.040830716490745544, -0.014333496801555157, 0.025283921509981155, 0.06857722252607346, 0.006081328261643648, -0.02890048548579216, 0.017003707587718964, 0.0010490634012967348, 0.008457006886601448, -0.016983039677143097, 0.010380554012954235, 0.009238388389348984, -0.010964710265398026, 0.03570989519357681, 0.02992062084376812, -0.008605384267866611, -0.01581001840531826, 0.00947674922645092, 0.04120858758687973, -0.02895994670689106, 0.026985127478837967, -0.008790643885731697, 0.03523923456668854, 0.01548722106963396, 0.013080153614282608, 0.0011304611107334495, -0.0408778190612793, 0.018567223101854324, -0.012343848124146461, 0.013975773938000202, -0.026098741218447685, -0.03844146803021431, 0.0036530927754938602, 0.018683457747101784, -0.043148770928382874, 0.024989323690533638, 0.0411243699491024, 0.22236892580986023, -0.04270564392209053, -0.014836173504590988, -0.026850061491131783, -0.015304486267268658, 0.030950361862778664, 0.031197180971503258, 0.04189761355519295, -0.0017050227615982294, -0.03726201876997948, -0.0018681377405300736, -0.04970354214310646, 0.011206877417862415, -0.0097407391294837, -0.0013274227967485785, -0.020352182909846306, 0.012994891963899136, 0.0017471896717324853, -0.02486199326813221, -0.024969415739178658, 0.010167259722948074, 0.003511731745675206, -0.020897911861538887, 0.0035547937732189894, -0.002656863536685705, -0.004352623131126165, -0.031700149178504944, -0.005012153182178736, -0.002561684465035796, 0.00045803815010003746, 0.02808075025677681, 0.009555141441524029, -0.008136390708386898, -0.01667354255914688, -0.023619413375854492, -0.016019484028220177, 0.012812967412173748], [0.03400663286447525, -0.0015436449320986867, -0.00804263073951006, 0.028261570259928703, -0.0116591677069664, -0.004472040571272373, 0.03144988790154457, 0.03318357840180397, -0.01391823124140501, 0.01460973545908928, -0.02135106921195984, 0.034777697175741196, 0.025667931884527206, 0.03834253549575806, -0.012752003036439419, 0.0261846836656332, 0.02512868121266365, -0.015023419633507729, 0.04454011097550392, 0.00821983627974987, -0.006159628741443157, 0.0029564842116087675, 0.0004991322639398277, -0.01978822611272335, 0.002945999149233103, 0.009907349944114685, 0.01504555530846119, -0.007524427492171526, -0.05147875100374222, 0.007610512897372246, -0.025626186281442642, 0.003660701448097825, -0.08570429682731628, -0.04205305129289627, 0.01465601660311222, 0.0069523402489721775, 0.021318309009075165, 0.007462546229362488, -0.024703480303287506, -0.285060852766037, -0.027035707607865334, 0.03996795043349266, 0.021874135360121727, 0.016008397564291954, 0.006056287791579962, 0.006618177052587271, 0.00808029342442751, -0.009199453517794609, -0.04367228224873543, 0.012271649204194546, 0.016902929171919823, 0.03165780007839203, 0.029333926737308502, -0.009436006657779217, 0.0072332145646214485, 0.033521708101034164, -0.003545942483469844, -0.027904165908694267, -0.013692619279026985, -0.011373239569365978, 0.004234929569065571, -0.0037177111953496933, -0.026540473103523254, 0.054277047514915466, -0.024018215015530586, 0.02170109935104847, 0.01946161687374115, 0.16153870522975922, -0.0028144957032054663, 0.017519373446702957, -0.02148316241800785, -0.0060854144394397736, 0.010545829311013222, -0.039978988468647, -0.0029564888682216406, -0.007177876308560371, 0.022065147757530212, -0.011908029206097126, -0.0024819725658744574, 0.00970966275781393, -0.0064574601128697395, -0.0011353505542501807, -0.0031833196990191936, 0.012018943205475807, -0.00562827754765749, -0.01647166721522808, -0.025121677666902542, 0.030129410326480865, -0.027381913736462593, 0.015883946791291237, 0.011860932223498821, -0.000816363375633955, -0.009932594373822212, -0.09018632769584656, 0.02448972500860691, -0.03154526278376579, 0.005381524562835693, -0.0023872593883424997, 0.01075428444892168, -0.01046557817608118, -0.011859983205795288, 0.022362273186445236, 0.013997870497405529, -0.03347103297710419, -0.00977164227515459, -0.002978966338559985, 0.026121141389012337, 0.010659358464181423, -0.017607608810067177, 0.003830490168184042, 0.00799291580915451, -0.025674281641840935, -0.03929204121232033, 0.010303040035068989, 0.04157667234539986, 0.012201636098325253, 0.013273708522319794, 0.008674366399645805, 0.012603966519236565, -0.02230583317577839, -0.006400889251381159, 0.038086775690317154, 0.010756791569292545, -0.008178961463272572, 0.030870411545038223, 0.016602229326963425, -0.04297397658228874, -0.019302602857351303, -0.020264672115445137, 0.026224235072731972, -0.009133165702223778, 0.011213195510208607, 0.0190466046333313, 0.018951337784528732, 0.004928387235850096, 0.02640596032142639, 0.03181800991296768, -0.021149998530745506, 0.03973536565899849, -0.0035484926775097847, -0.006610069423913956, -0.005562926642596722, 0.013028410263359547, -0.008833334781229496, -0.002582911401987076, -0.02525188773870468, -0.0026137700770050287, 0.007549296598881483, 0.04207450523972511, 0.009332476183772087, -0.015666864812374115, -0.023304559290409088, 0.008845966309309006, 0.00777985667809844, -0.01904129423201084, -0.01720675639808178, 0.0258417259901762, 0.04640461131930351, -0.01793483830988407, 0.012126562185585499, 0.013669137842953205, -0.000178992107976228, 0.020274091511964798, 0.006697441916912794, -0.023536784574389458, 0.008978958241641521, -0.00843418762087822, -0.043570321053266525, 0.012821820564568043, -0.007499954663217068, 0.015387864783406258, -0.010894017294049263, -0.002142805838957429, -0.010846969671547413, -0.0026883911341428757, 0.01617296226322651, -0.017978891730308533, 0.024996604770421982, 0.029550110921263695, 0.054177749902009964, -0.00768158258870244, -0.017413699999451637, -0.030476462095975876, 0.047130338847637177, -0.009211541153490543, -0.02988244779407978, -0.012255207635462284, 0.0012369862524792552, 0.051204390823841095, -0.03357449546456337, -0.04187183082103729, -0.015369324013590813, 0.009442386217415333, -0.027279945090413094, -0.007708156481385231, 0.021496493369340897, -0.01949549838900566, 0.06417511403560638, -0.00828170869499445, 0.007592370267957449, 0.004710490349680185, -0.02929242141544819, -0.0375748910009861, -0.022978907451033592, -0.017676087096333504, 0.003088355530053377, -0.0013783033937215805, -0.009737304411828518, 0.018757421523332596, 0.004346954636275768, 0.03840675950050354, -0.016369758173823357, 0.026337167248129845, 0.0016577157657593489, 0.027003027498722076, -0.0011461259564384818, 0.06333651393651962, -0.002348874695599079, 0.019556675106287003, -0.019986506551504135, -0.009657667018473148, -0.025877922773361206, 0.0023625376634299755, -0.018840476870536804, 0.022491609677672386, 0.006336461752653122, -0.0021372598130255938, 0.013207562267780304, -0.03485121205449104, 0.0067771063186228275, -0.01167382299900055, -0.004339514300227165, 0.031065385788679123, 0.01601722277700901, 0.004022879526019096, -0.12491348385810852, -0.04193280637264252, 0.03632153570652008, 0.0195225290954113, 0.007354762405157089, -0.007656047120690346, 0.004381063859909773, 0.01611623913049698, 0.005042609758675098, -0.01408254075795412, -0.012724184431135654, -0.002344986191019416, -0.019152037799358368, -0.024700362235307693, -0.010730176232755184, -0.042980436235666275, -0.009860845282673836, -0.005391933489590883, -0.034810684621334076, 0.021575765684247017, -0.049086879938840866, -0.0056817675940692425, -0.010696603916585445, -0.035796813666820526, -0.003493362572044134, -0.00988326407968998, -0.02592250145971775, 0.011401144787669182, -0.0036553884856402874, 5.375596083467826e-05, 0.016453394666314125, -0.03599889203906059, -0.01261061429977417, -0.0032670930959284306, 0.012913339771330357, -0.02347523905336857, -0.00043365254532545805, -0.002440324518829584, -0.00502529414370656, 0.015931978821754456, -0.021652748808264732, 0.040207065641880035, 0.023763395845890045, 0.0031464912462979555, -0.001516259158961475, 0.019829047843813896, 0.0349554568529129, -0.011972920037806034, 0.0026549252215772867, -0.015876347199082375, 0.008734052069485188, -0.015347522683441639, -0.0041737863793969154, -0.020327677950263023, 0.022096125409007072, -0.03819829970598221, 0.021592076867818832, -0.020798681303858757, -0.013969674706459045, 0.016765817999839783, 0.01905534416437149, 0.01765015348792076, -0.05541209131479263, -0.0028792708180844784, -0.022771967574954033, -0.01260068267583847, 0.04510484263300896, 0.02634645812213421, -0.0031307553872466087, 0.019812067970633507, -0.039193447679281235, -0.01831752061843872, 0.016945017501711845, 0.019567541778087616, -0.02680390514433384, 0.0014216086128726602, 0.006040323991328478, -0.02683597430586815, 0.011015595868229866, 0.03875720873475075, 0.0250192042440176, 0.0206681489944458, 0.07064191997051239, -0.04237501695752144, -0.01774151250720024, -0.045675817877054214, 0.0386776439845562, 0.01981133036315441, 0.0038692106027156115, 0.03757595643401146, 0.03634607046842575, -0.014477632939815521, -0.011485693044960499, 0.009897605516016483, -0.001966000534594059, 0.027290528640151024, 0.04537014290690422, -0.025611858814954758, 0.019972804933786392, 0.021608812734484673, -0.012231026776134968, -0.01746232993900776, 0.0015337667427957058, 0.04319329559803009, 0.032970864325761795, 0.049538470804691315, 0.0028564240783452988, 0.020973842591047287, -0.021265683695673943, -0.017675571143627167, 0.012535793706774712, -0.000613678595982492, -0.011475324630737305, 0.010490729473531246, 0.015000942163169384, -0.0056367176584899426, -0.045065987855196, 0.0166950561106205, 0.09355337917804718, -0.009758631698787212, 0.0011659137671813369, 0.034576673060655594, -0.011384524405002594, -0.02487858012318611, 0.022455822676420212, -0.061802130192518234, 0.005260412115603685, 0.013928539119660854, 0.0150227015838027, -0.009105677716434002, -0.026915689930319786, 0.051822636276483536, -0.025937693193554878, -0.03443288058042526, 0.019701343029737473, 0.02920762076973915, 0.06069542467594147, -0.010043144226074219, 0.05863812565803528, -0.014583991840481758, 9.482683526584879e-05, -0.008155053481459618, -0.0028981983195990324, -0.011683964170515537, -0.0003059171140193939, 0.016442155465483665, -0.022044803947210312, -0.002818414242938161, -0.008494766429066658, 0.013390250504016876, -0.015178371220827103, -0.014659714885056019, -0.05245699733495712, 0.01836317591369152, 0.01630229502916336, -0.0166744664311409, -0.03038078360259533, 0.020230868831276894, -0.018061337992548943, -0.013875752687454224, -0.0006889210781082511, -0.0031622741371393204, -0.008456520736217499, -0.006667341571301222, 0.009118234738707542, -0.009979627095162868, 0.010859726928174496, 0.04012470319867134, -0.0026553014758974314, -0.0051043168641626835, 0.016831954941153526, 0.005637097172439098, 0.5944774746894836, 0.016541389748454094, -0.013207151554524899, -0.01599888503551483, -0.022587766870856285, 0.027073713019490242, -0.008985618129372597, 0.002012543613091111, 0.043761443346738815, -0.010403171181678772, -0.00592998880892992, 0.029858622699975967, -0.0008935522055253386, 0.002303054090589285, 0.0018270682776346803, -0.01160271093249321, 0.005708899814635515, -0.032208774238824844, -0.04579676687717438, -0.00777054950594902, 0.006218607537448406, -0.0057386732660233974, -0.059509120881557465, -0.020542506128549576, 0.07137864828109741, 0.02950572967529297, 0.007694713771343231, 0.024648325517773628, -0.021260177716612816, 0.012377355247735977, -0.0010371303651481867, 0.012669435702264309, 0.0018415150698274374, 0.036854129284620285, 0.003912731073796749, -0.009592564776539803, 0.005647176411002874, -0.009305101819336414, -0.016426749527454376, 0.03093506395816803, 0.025218548253178596, 0.01719214767217636, -0.012318951077759266, 0.015279600396752357, 0.004804412368685007, 0.037495650351047516, 0.0018420665292069316, 0.006425573956221342, 0.01763913594186306, 0.025459174066781998, -0.036677610129117966, 0.009662510827183723, -0.0057119992561638355, 0.000793594925198704, -0.013328886590898037, -0.008306590840220451, -0.009988273493945599, -0.02341415546834469, -0.011927572079002857, 0.014245648868381977, -0.009752204641699791, -0.000358807563316077, -0.06716203689575195, -0.05714791640639305, 0.054098136723041534, 0.018396276980638504, 0.031619325280189514, -0.007870808243751526, 0.009964321739971638, 0.005926375277340412, -0.012357779778540134, 0.00902009941637516, -0.05817145109176636, 0.04658978804945946, 0.03337278962135315, -0.022687038406729698, 0.03348015248775482, 0.006868145428597927, 0.0012375214137136936, 0.01877550408244133, 0.0027272109873592854, 0.017918499186635017, 0.010210921987891197, -0.06038687378168106, 0.00399661622941494, -0.012376784346997738, 0.022230707108974457, -0.008620103821158409, -0.005269138142466545, -0.0003046840720344335, -0.034194186329841614, 0.017861563712358475, -0.01506368163973093, 0.08857591450214386, -0.0339498296380043, -0.027325253933668137, -0.019403517246246338, -0.02418977953493595, -0.06379689276218414, -0.04594728723168373, -0.009885202161967754, 0.0022309294436126947, 0.00868277344852686, 0.013898313045501709, 0.035381946712732315, 0.06348379701375961, -0.01664762571454048, 0.012788147665560246, 0.006270745303481817, 0.022523894906044006, -0.034031230956315994, -0.018604587763547897, 0.017283955588936806, -0.007320718839764595, 0.03134555369615555, -0.01759534701704979, 0.003906457684934139, -0.008582798764109612, -0.03921641409397125, -0.01273337658494711, 0.008342217653989792, -0.04197854548692703, -0.0032884308602660894, 0.01559777744114399, 0.026069793850183487, 0.019731072708964348, -0.009255749173462391, -0.06557597219944, -0.054099272936582565, -0.00012205523671582341, -0.013768315315246582, -0.020383741706609726, 0.023600028827786446, -0.022784890606999397, 0.004920585546642542, 0.003633682383224368, -0.05293835327029228, -0.005991420708596706, -0.011637076735496521, 0.018322868272662163, 0.03862382099032402, 0.04513804242014885, 0.036891039460897446, 0.022666100412607193, 0.0007833924028091133, -0.016264336183667183, 0.008835287764668465, -0.006922115571796894, -0.10818382352590561, -0.0032097017392516136, 0.03259163722395897, 0.03463706746697426, 0.009247037582099438, 0.027224143967032433, -0.0361349880695343, 0.019444892182946205, -0.050996970385313034, 0.004509749356657267, 0.01137428916990757, -0.03831055015325546, -0.0061940294690430164, 0.03952259197831154, 0.004324037581682205, 0.024134935811161995, 0.006604186724871397, 0.02636130340397358, 0.0014751218259334564, 0.029546014964580536, -0.010856447741389275, 0.03134426847100258, -0.028389569371938705, -0.025607025250792503, -0.008508737199008465, 0.04852816462516785, -0.0037262749392539263, 0.0073755523189902306, -0.03954503312706947, -0.044144053012132645, 0.01751117594540119, -0.01170297060161829, 0.015812816098332405, 0.03225267678499222, -0.014507888816297054, -0.018632983788847923, -0.0016432075062766671, -0.03069205768406391, 0.013348367996513844, 0.004091107752174139, 0.0368574857711792, -0.012902917340397835, -0.008868061006069183, 0.015088954009115696, -0.02396753989160061, -0.03098318539559841, 0.03142629191279411, 0.010998185724020004, 0.03062169812619686, 0.03792639076709747, -0.0010712554212659597, 0.03506151959300041, -0.027247905731201172, 0.00938954297453165, -0.0914452001452446, -0.006843816488981247, -0.004269590601325035, 0.006847775541245937, -0.009114292450249195, 0.0005920000257901847, -0.011755791492760181, 0.010226879268884659, -0.032654523849487305, 0.0005334493471309543, -0.04716760665178299, 0.04645873233675957, -0.002657397883012891, 0.00546187674626708, 0.0038741729222238064, -0.023026926442980766, 0.03981238603591919, -0.009672679007053375, 0.00045100602437742054, 0.029494967311620712, -0.007265293970704079, 0.0033792080357670784, -0.026879049837589264, 0.04865911230444908, 0.009635536931455135, -0.03272899612784386, 0.02516271360218525, 0.018175022676587105, -0.004556653089821339, -0.0026032058522105217, 0.03520296886563301, -0.00534831453114748, 0.02171984314918518, 0.03149605542421341, -0.035016369074583054, -0.039931945502758026, 0.007969489321112633, 0.02083113044500351, 0.010670057497918606, -0.037918005138635635, 0.012838320806622505, 0.0034935849253088236, -0.025410259142518044, 0.0012063449248671532, 0.03128062188625336, -0.0019272628705948591, -0.01640106551349163, -0.04167287051677704, -0.025633657351136208, -0.021358437836170197, -0.014522816054522991, 0.01678408868610859, -0.01872416026890278, -0.018986152485013008, 0.03068923018872738, -0.023537026718258858, -0.05786680057644844, 0.0010709454072639346, 0.013088621199131012, 0.016845010221004486, -0.01028662733733654, 0.017041722312569618, 0.033757127821445465, 0.008136512711644173, 0.010899980552494526, 0.0008981100982055068, 0.028030820190906525, -0.009297706186771393, 0.018029073253273964, -0.008862975053489208, 6.383101572282612e-05, -0.0178694911301136, -0.018561555072665215, -0.005240486469119787, -0.02331187203526497, 0.0258126612752676, 0.04418591409921646, -0.03331879526376724, 0.0065032788552343845, 0.027891704812645912, 0.005733184050768614, 0.0005913033965043724, -0.009820781648159027, -0.03939829394221306, 0.008875454775989056, 0.05653247982263565, 0.008510853163897991, -0.026385124772787094, 0.06594065576791763, -0.012008914723992348, -0.02282699942588806, 0.021466996520757675, -0.008098213002085686, -0.0210683923214674, -0.002882481086999178, 0.020017089322209358, -0.01943366229534149, -0.036331240087747574, 0.021736683323979378, 0.02300996147096157, -0.011291738599538803, -0.015094572678208351, -0.01499552559107542, 0.02472887560725212, -0.012923305854201317, 0.03841077908873558, 0.004214846529066563, 0.03132260963320732, -0.02675909548997879, -0.01359061524271965, -0.030001766979694366, -0.010226056911051273, -0.005538001190871, -0.034814827144145966, -0.011059491895139217, -0.029378455132246017, -0.002386858221143484, -0.015527677722275257, 0.00392382824793458, -0.019753912463784218, 0.010740995407104492, 0.02766469679772854, 0.20367063581943512, -0.04921450465917587, 0.0008599178399890661, -0.03614342212677002, 0.007627869490534067, 0.024927740916609764, 0.040720365941524506, 0.03204711526632309, -0.016897542402148247, -0.04565247893333435, -0.02189171500504017, -0.06110922247171402, 0.015024419873952866, 0.022365428507328033, -0.0035993256606161594, -0.012217662297189236, 0.0162697434425354, 0.023819684982299805, -0.020296690985560417, 0.00024965390912257135, 0.012414876371622086, -0.003573457710444927, -0.038587603718042374, 0.031508199870586395, -0.022630304098129272, 0.003561302088201046, -0.010149276815354824, 0.0004720878496300429, -0.018838362768292427, 0.02400112897157669, 0.017125204205513, 0.00435534818097949, 0.0203541349619627, -0.010298394598066807, 0.0011770883575081825, 0.01300171110779047, 0.006454887799918652], [0.05193600058555603, 0.008860882371664047, -0.02754889614880085, -0.03943568840622902, -0.01627063937485218, 0.005942632909864187, 0.020086834207177162, -0.004528718534857035, 0.015656564384698868, 0.029233962297439575, 0.000909039459656924, 0.0047142235562205315, 0.0338418185710907, 0.02051401138305664, -0.0002691929694265127, 0.024553416296839714, 0.0078100948594510555, 0.005281324032694101, -0.002143522258847952, 0.032569147646427155, -0.0065843649208545685, 0.024145158007740974, -0.024357615038752556, 0.052625223994255066, 0.017387447878718376, 0.04908962920308113, 0.02022196352481842, 0.007088370155543089, -0.03705078735947609, 0.019407974556088448, 0.005103118717670441, -0.00014487621956504881, -0.09528570622205734, -0.07257895916700363, -0.030858034268021584, 0.0009198518819175661, 0.009059801697731018, -0.006949331611394882, -0.012744412757456303, -0.2917439043521881, -0.013229919597506523, 0.035352807492017746, 0.018334245309233665, -0.012070652097463608, 0.027517840266227722, -0.0015678175259381533, 0.002379045356065035, -0.010659866034984589, -0.02816959097981453, 0.015489575453102589, -0.007707179989665747, 0.012752651236951351, -0.00835600309073925, -0.012697051279246807, 0.02683032676577568, 0.0033873585052788258, -0.010943003930151463, -0.09539636224508286, 0.0157148577272892, 0.005875754170119762, -0.011440661735832691, 0.03015541471540928, -0.01640872098505497, 0.0487099215388298, -0.0454590879380703, 0.007395392749458551, 0.016428504139184952, 0.11996076256036758, -0.032346539199352264, 0.041224583983421326, 0.013114786706864834, -0.010946900583803654, -0.02805863507091999, -0.014617533423006535, 0.017736705020070076, -0.010928614996373653, 0.039962876588106155, -0.028562430292367935, -0.011167462915182114, 0.0145726865157485, 0.022227711975574493, -0.019345592707395554, -0.006798663642257452, 0.014802499674260616, 0.007738551124930382, -0.03951578587293625, -0.013978298753499985, 0.018847763538360596, 0.01452918816357851, 0.02104928158223629, 0.001406861119903624, -0.015338222496211529, -0.03922748193144798, -0.0859512984752655, 0.023108305409550667, 0.009183401241898537, 0.014034315943717957, -0.04965464770793915, 0.028737148270010948, -0.030059684067964554, 0.018265003338456154, 0.01353379338979721, -0.021475499495863914, -0.011377621442079544, -0.00442476524040103, -0.027203494682908058, -0.03264318034052849, -0.009008658118546009, 0.014178971759974957, 0.035766832530498505, 5.762233195127919e-05, 0.0009403792209923267, -0.02477307617664337, -0.003059734357520938, 0.005971973761916161, 0.01006357092410326, 0.005830212961882353, 0.009448745287954807, -0.004834561608731747, -0.01649082638323307, -0.04002447798848152, 0.019380943849682808, 0.016404280439019203, -0.01757260039448738, 0.0034950010012835264, 0.025299683213233948, -0.013793688267469406, 0.0033345725387334824, -0.01261462178081274, -0.0036707576364278793, 0.011956301517784595, 0.03726305067539215, -0.002645963104441762, -0.007325096055865288, 0.011252821423113346, 0.028793593868613243, 0.006383676081895828, -0.00297363568097353, 0.045854080468416214, -0.02098543755710125, -0.011615944094955921, -0.006022682413458824, -0.021227100864052773, 0.004800222348421812, -0.026517421007156372, 0.009444313123822212, -0.008463358506560326, 0.00373389245942235, 0.014335079118609428, -0.0167509987950325, -0.028723929077386856, -0.006381660234183073, 0.04002222791314125, 0.004015305079519749, -0.023864641785621643, 0.02735598385334015, 0.0130570437759161, 0.022538529708981514, -0.009050760418176651, 0.04190196841955185, -0.04416956380009651, 0.03505415841937065, 0.018045807257294655, -0.006573658902198076, 0.018409771844744682, 0.009474361315369606, 0.033528439700603485, -0.016163483262062073, -0.0025172859895974398, -0.007626075763255358, -0.00851474143564701, 0.001129872165620327, 0.00544986454769969, -0.029112420976161957, 0.02237579971551895, 0.038147132843732834, 0.004547396674752235, 0.02150382474064827, 0.012100244872272015, 0.01609477773308754, -0.012942306697368622, -0.004546375945210457, 0.014805523678660393, 0.03834288567304611, 0.009889420121908188, -0.01702510006725788, -0.011663448996841908, 0.010908562690019608, 0.004844136536121368, -0.011789452284574509, 0.0014083781279623508, 0.006012956146150827, 0.029463469982147217, -0.031797587871551514, 0.019737564027309418, 0.05221922695636749, -0.011425698176026344, 0.0008035514038056135, -0.01565559022128582, -0.03864482045173645, -0.023015927523374557, -0.012913216836750507, -0.014610126614570618, 0.017775647342205048, 0.01724751479923725, 0.017310921102762222, -0.0036878606770187616, -0.010237392038106918, -0.006565650459378958, -0.027206534519791603, 0.07115733623504639, 0.005630853120237589, 0.007129087578505278, -0.004388007335364819, 0.020435880869627, 0.0005492600030265749, 0.02437867410480976, -0.011869080364704132, 0.016039855778217316, 0.005401370115578175, -0.01080784760415554, -0.041373614221811295, -0.025331098586320877, 0.02421645261347294, 0.007863256148993969, 0.0009496317361481488, -0.010969707742333412, -0.023793190717697144, -0.00196696724742651, 0.022483691573143005, -0.04103328287601471, 0.011629676446318626, 0.025124935433268547, -0.016823740676045418, 0.02173541858792305, -0.11836127191781998, -0.011414670385420322, 0.04784482717514038, -0.002366610337048769, 0.014230935834348202, 0.007647400256246328, -0.004730364307761192, 0.03718581050634384, -0.016819115728139877, -0.04508374631404877, 0.0019173838663846254, -0.01756250485777855, -0.012970643118023872, -0.03184504061937332, -0.005817187950015068, -0.01339265052229166, 0.015471009537577629, -0.001393466372974217, -0.02097565121948719, 0.037788961082696915, -0.0695606917142868, -0.0014562902506440878, -0.024864541366696358, -0.0027740048244595528, -0.0012714560143649578, -0.014635693281888962, 0.007414605468511581, -0.000987560604698956, -0.04068661853671074, -0.008909694850444794, -0.050559546798467636, -0.012949797324836254, -0.01156304869800806, 0.0031217429786920547, 0.016886211931705475, -0.026352670043706894, 0.005268965847790241, -0.013034229166805744, -0.013562432490289211, 0.017598044127225876, -0.0330926738679409, 0.03665995970368385, -0.0064444406889379025, 0.03213651850819588, -0.0014046522555872798, 0.03868992626667023, 0.01873158849775791, 0.017283571884036064, -0.005766553804278374, -0.003319233190268278, -0.014247977174818516, -0.014756725169718266, 0.002723891055211425, -0.026071513071656227, -0.00519534619525075, -0.05867614597082138, 0.01968214102089405, -0.03019510768353939, -0.003231398295611143, 0.050334520637989044, 0.005442590918391943, 0.013255645520985126, -0.006019127555191517, -0.026979073882102966, -0.022575924172997475, -0.02785799279808998, -0.001946883276104927, 0.009529458358883858, -0.026197578758001328, -0.02194010652601719, -0.0017468652222305536, -0.03399118036031723, 0.019642939791083336, 0.010966142639517784, 0.019969340413808823, 0.004667663015425205, -0.020017892122268677, 0.006986566819250584, -0.026881571859121323, 0.017668047919869423, 0.0032712046522647142, -0.03275417909026146, 0.041822198778390884, 0.006122001446783543, 0.004524408373981714, -0.0012714832555502653, 0.04244041442871094, 0.03521377593278885, 0.016629675403237343, 0.029356131330132484, -0.020755989477038383, -0.025393204763531685, -0.01159488596022129, -0.013650759123265743, 0.00461567472666502, 0.044891972094774246, 0.06275264918804169, -0.009415733627974987, 0.0051694526337087154, 0.016354823485016823, 0.006776804104447365, 0.02395619824528694, 0.02401598170399666, 0.029273351654410362, 0.004119458608329296, 0.02143085189163685, 0.0014957725070416927, -0.008524487726390362, 0.03686390444636345, -0.038122065365314484, 0.004582813009619713, -0.012011625804007053, -0.008473670110106468, 0.01745983399450779, -0.00351334479637444, -0.007649526000022888, 0.001571698929183185, 0.028694236651062965, 0.11247392743825912, -0.004599560517817736, -0.01595117338001728, 0.06685102730989456, -0.032249026000499725, -0.013663138262927532, 0.0006255013286136091, -0.04200179874897003, 0.02621587924659252, -0.05161352828145027, 0.03676662966609001, -0.0068589141592383385, 0.013947082683444023, 0.03107563406229019, -0.010192608460783958, -0.05606042221188545, -0.007721775211393833, -0.017077500000596046, 0.03300130367279053, -0.008215606212615967, 0.04996243119239807, -0.02479962445795536, 0.0015303930267691612, 0.009031027555465698, -0.00016705824236851186, -0.010938632301986217, 0.05354704707860947, 0.00462625827640295, 0.0007530523580498993, 0.007134673185646534, -0.030719682574272156, -0.0092538520693779, -0.017161916941404343, 0.010734213516116142, -0.03220198675990105, -0.013876709155738354, 0.03640149161219597, 0.002301259431988001, -0.02921047806739807, 0.04472396522760391, -0.024679090827703476, -0.023215919733047485, -0.03414111211895943, -0.002126001287251711, -0.011095905676484108, 0.012643585912883282, 0.014640100300312042, 0.008554695174098015, 0.016150448471307755, 0.005715403240174055, 0.0056549618020653725, -0.03950291872024536, -0.011149411089718342, 0.012478381395339966, 0.5763377547264099, 0.046514902263879776, -0.018054110929369926, -0.0017921642865985632, 0.02509421296417713, 0.012952825985848904, -0.03783386945724487, -0.013662672601640224, 0.026703132316470146, -0.007782829459756613, -0.02877875231206417, 0.016164440661668777, 0.03402363136410713, 0.02129221148788929, -0.03754490613937378, -0.04548950120806694, 0.018327195197343826, -0.027737392112612724, -0.0441877655684948, 0.028445672243833542, -0.020612360909581184, -0.036310650408267975, 0.00673649413511157, 0.0010004465002566576, 0.06287092715501785, 0.05192527920007706, 0.024658652022480965, 0.01435768697410822, -0.023188764229416847, 0.029704546555876732, -0.03948057070374489, 0.025566013529896736, -0.006255414802581072, -0.002426528139039874, -0.02087937854230404, 0.019690679386258125, 0.00041425187373533845, 0.004312863573431969, 0.026218058541417122, -0.007196398451924324, -0.013290337286889553, 0.0419137217104435, -0.042665984481573105, 0.0060352454893291, -0.01378605980426073, 0.0762748271226883, 0.02484164945781231, -0.00898052379488945, -0.0018373708007857203, 0.009846813976764679, -0.0008111439528875053, -0.014957495965063572, 0.006818139459937811, -0.003820836078375578, -0.011275121010839939, 0.01602189801633358, 0.02696548029780388, 0.009297668002545834, -0.021790286526083946, -0.0029563119169324636, -0.02427927777171135, 0.0064445617608726025, -0.03863068297505379, -0.03830268606543541, 0.04104107245802879, 0.0032669214997440577, 0.03132602944970131, -0.026776395738124847, 0.00025395277771167457, -0.0212844405323267, -0.017637498676776886, 0.009752818383276463, 0.008693043142557144, 0.012788183055818081, 0.03653557598590851, -0.017990922555327415, -0.03658190742135048, -0.010772083885967731, -0.03781793639063835, 0.005229378119111061, -0.034296512603759766, 0.006819501984864473, -0.002113217720761895, 0.002030526055023074, -0.0025793404784053564, -0.010876734741032124, 0.022037982940673828, -0.004744146950542927, -0.006252225488424301, 0.0151445884257555, -0.018333647400140762, 0.006560714449733496, -0.0246171485632658, 0.08963664621114731, -0.00870246347039938, -0.019400687888264656, -0.008195383474230766, 0.031748898327350616, 0.0065102470107376575, -0.013126473873853683, 0.026161935180425644, 0.01930154860019684, 0.019631866365671158, 0.005075865890830755, 0.01848265714943409, 0.051772914826869965, -0.0610303170979023, -0.0009410749771632254, 0.04284786432981491, 0.005263141822069883, -0.04435499757528305, 0.01762414164841175, 0.03668588027358055, 0.01573682203888893, 0.0003261090023443103, 0.009237716905772686, 0.06147322058677673, -0.007324615493416786, -0.032701246440410614, 0.021691348403692245, 0.00858640018850565, -0.022697074338793755, 0.00871120672672987, -0.003136482322588563, 0.055166199803352356, 0.005715171806514263, 0.025214891880750656, 0.03808644786477089, -0.03953752666711807, -0.010919727385044098, -0.03731042519211769, 8.706031803740188e-05, -0.029332075268030167, -0.03420765697956085, 0.027773281559348106, 0.01249630842357874, 0.009310445748269558, 0.02172788605093956, 0.014978587627410889, 0.0066429851576685905, 0.02444385550916195, 0.04250215366482735, 0.027622994035482407, 0.009162873961031437, -0.00556747755035758, -0.040605347603559494, -0.012366282753646374, -0.0015202983049675822, -0.014630772173404694, 0.018275124952197075, 0.016822660341858864, 0.03238256648182869, -0.021377531811594963, 0.017344655469059944, 0.010676500387489796, -0.029332425445318222, -0.051613949239254, -0.016632575541734695, 0.0013791505480185151, 0.02250588685274124, 0.01883602701127529, 0.03384748473763466, 0.040495652705430984, 0.017077255994081497, -0.04330828785896301, -0.011653154157102108, -0.028206588700413704, 0.030694009736180305, -0.0016752039082348347, -0.004238825757056475, -0.04898710921406746, -0.009584464132785797, 0.02156919054687023, 0.04191813990473747, -0.014577029272913933, 0.013781329616904259, -0.008628986775875092, -0.03675419092178345, -0.020654741674661636, 0.04471112787723541, -0.005439368076622486, 0.006960841827094555, -0.029559094458818436, -0.016818391159176826, 0.01276860199868679, -0.008478141389787197, -0.01902461051940918, 0.01358063705265522, -0.01607741229236126, -0.1288563460111618, -0.007225564680993557, 0.03092845529317856, 0.025603944435715675, -0.03171229735016823, 0.03408064320683479, 0.014317401684820652, 0.015196164138615131, 0.032940711826086044, 0.0243629552423954, -0.00914733950048685, -0.027825452387332916, 0.012023606337606907, -0.04149072989821434, 0.002389392349869013, -0.003519380697980523, -0.002503344789147377, -0.041282158344984055, -0.02909761108458042, 0.022013746201992035, 0.045756418257951736, -0.009085873141884804, 0.01373432856053114, -0.028454642742872238, -0.03253454342484474, -0.012836976908147335, -0.00021282730449456722, 0.00564451003447175, 0.0035458256024867296, -0.007279324810951948, -0.005450207274407148, -0.00695872912183404, 0.0032533970661461353, -0.04217661917209625, -0.022549910470843315, -0.038523830473423004, 0.040672749280929565, -0.001580652198754251, -0.022541524842381477, 0.01877610944211483, 0.033831268548965454, -0.01854865625500679, -0.0024238459300249815, -0.02715304307639599, 0.03193282335996628, 0.0052602216601371765, -0.0009062254102900624, -0.03545674309134483, -0.01563493348658085, -0.04017376899719238, 0.007991155609488487, 0.01825588569045067, -0.0417756512761116, -0.01960982009768486, -0.04252840206027031, 0.011049927212297916, 0.009085243567824364, 0.01824132353067398, -0.005961670074611902, 0.037901148200035095, -0.05939798057079315, -0.02558325231075287, -0.018777374178171158, -0.02980063483119011, -0.04253073409199715, -0.034985918551683426, -0.0007189579191617668, 0.02529488503932953, -0.00442985724657774, 0.013872797600924969, 0.014474587514996529, 0.012009775266051292, -0.0017328481189906597, -0.03113236278295517, 0.027820784598588943, 0.055803537368774414, 0.021059049293398857, 0.0023320422042161226, 0.000976109120529145, 0.027692167088389397, -0.007039038464426994, -0.0031762777362018824, -0.01699480228126049, -0.011150794103741646, -0.028476424515247345, 0.014684617519378662, -0.020951298996806145, 0.012124291621148586, 0.011491602286696434, 0.025953276082873344, -0.02336808666586876, 0.010898837819695473, 0.022347044199705124, -0.042425453662872314, -0.02051762118935585, 0.029118647798895836, -0.055623579770326614, 0.019784199073910713, 0.05822001025080681, 0.009459521621465683, 0.01786641776561737, 0.05635123327374458, -0.03533870354294777, 0.0031377223785966635, -0.006472724489867687, 0.01458752155303955, -0.010287703014910221, -0.004945988301187754, -0.0007950249128043652, 0.02563183195888996, -0.02990446425974369, 0.05612820014357567, 0.016481975093483925, -0.02870156429708004, 0.03161870315670967, -0.020915813744068146, 0.06413012742996216, -0.01216622069478035, -0.004758669063448906, -0.0018862136639654636, -0.008877649903297424, -0.00279601220972836, 0.018389137461781502, -0.0013921195641160011, -0.01793213188648224, -0.00822356715798378, -0.021713074296712875, 0.013978524133563042, -0.02153446339070797, -0.004985577426850796, -0.03830673173069954, -0.017081575468182564, 0.026594391092658043, -0.010622980073094368, 0.02434373088181019, 0.20742014050483704, 0.008368083275854588, 0.01328446064144373, -0.021502578631043434, -0.008944792672991753, -0.021549850702285767, 0.032907236367464066, 0.030256973579525948, -0.01771947182714939, -0.00817499216645956, 0.017800575122237206, -0.0630374327301979, 0.006652228068560362, -0.007131934631615877, -0.037192098796367645, -0.006637221202254295, -0.002339328406378627, 0.027969498187303543, -0.010834424756467342, -0.006711398251354694, 0.01378963328897953, -0.0069908094592392445, -0.01958717405796051, -0.016140460968017578, -0.03898448124527931, -0.04010142758488655, -0.003231908194720745, 0.0038775273133069277, 0.017970247194170952, 0.05252555012702942, 0.06817735731601715, -0.004991911817342043, 0.009144906885921955, -0.02506263554096222, -0.007333702873438597, -0.001162266475148499, 0.016424255445599556], [0.0017394269816577435, -7.698495028307661e-05, -0.023235730826854706, -0.026471609249711037, 0.02959711290895939, -0.015149211511015892, -0.012074909172952175, -0.03537272289395332, -0.015153476037085056, 1.3027424756728578e-06, -0.032341811805963516, -0.05803307890892029, 0.015969332307577133, 0.0011979449773207307, 0.005456340499222279, -0.025630155578255653, 0.024591263383626938, 0.06142815947532654, -0.02338508889079094, 0.032104212790727615, 0.022894956171512604, -0.0003747355076484382, -0.0015146028017625213, 0.010019060224294662, -0.02052192948758602, 0.03292011842131615, 0.01998116262257099, 0.00510422931984067, 0.006181709002703428, 0.029569227248430252, 0.007955835200846195, 0.005393547005951405, 0.0015000593848526478, 0.030734457075595856, 0.040355682373046875, 0.005885372869670391, -0.019529270008206367, -0.02346949838101864, -0.012489129789173603, -0.28313031792640686, 0.03709593787789345, -0.009260461665689945, 0.006679154466837645, -0.00729833310469985, 0.023061029613018036, 0.02224348671734333, -0.02542252093553543, -0.018805859610438347, -0.050049442797899246, -0.050779566168785095, -0.009141613729298115, 0.04775846377015114, 0.02605287916958332, -0.017995284870266914, 0.010915268212556839, 0.0048355832695961, 0.006106437183916569, -0.06931748241186142, -0.006660864222794771, 0.01027885265648365, -0.009488968178629875, -0.008553177118301392, 0.008786232210695744, 0.009043263271450996, 0.02093014307320118, -0.010682953521609306, -0.003986380062997341, 0.046953827142715454, -0.0200961846858263, 0.0046116383746266365, 0.013284212909638882, -0.0052514937706291676, 0.01345598604530096, 0.008569180965423584, 0.02291218191385269, 0.027215389534831047, 0.018823860213160515, -0.024198170751333237, -0.05772054195404053, -0.004504726268351078, -0.010700847953557968, 0.00656209047883749, -0.01210113987326622, -0.017538949847221375, -0.04667101800441742, -0.04459236562252045, 0.05415702611207962, 0.03114568255841732, -0.0001212159768329002, 0.02417924255132675, 0.028884241357445717, 0.008219467476010323, 0.004019551910459995, -0.0015993400011211634, 0.011360322125256062, 0.00955770630389452, 0.006662874016910791, -0.006858034525066614, -0.03064415231347084, -0.01511844340711832, 0.02479952573776245, -0.013899083249270916, 0.008046592585742474, -0.031072791665792465, -0.03277621418237686, -0.006597073748707771, 0.004547543823719025, 0.012444084510207176, 0.015150154009461403, 0.034031450748443604, 0.04624093323945999, 0.0019193895859643817, -0.02345185913145542, -0.013224396854639053, 0.014536288566887379, -0.00684249447658658, -0.07188723236322403, -0.030105747282505035, 0.0025928127579391003, -0.022134393453598022, -0.01790166087448597, 0.004978708457201719, -0.014260201714932919, 0.018973423168063164, -0.0024208403192460537, 0.00010475605085957795, -0.004376979079097509, 0.007923590950667858, 0.0008062178967520595, 0.013155910186469555, -0.01225005742162466, -0.00963747687637806, -0.023544002324342728, 0.017217960208654404, -0.018163416534662247, 0.024022074416279793, -0.01961740478873253, -0.006802455522119999, 0.044020481407642365, -0.023315083235502243, 0.009540130384266376, -0.016543036326766014, 0.02613968774676323, 0.002385193482041359, -0.0003338073438499123, -0.034895312041044235, 0.005796616896986961, -0.007115163374692202, 0.019440731033682823, -0.003972297068685293, -0.006286820396780968, -0.029923835769295692, 0.009383289143443108, -0.018677396699786186, 0.041395045816898346, -0.03089381754398346, -0.014826941303908825, 0.01050302479416132, 0.015613612718880177, 0.0008136890828609467, 0.03943407163023949, -0.001810990972444415, 0.0010001161135733128, 0.02498224563896656, -0.02832845039665699, 0.0036795567721128464, 0.02800925448536873, 0.04756699502468109, 0.04306739196181297, 0.007992701604962349, 0.013677481561899185, 0.03076528012752533, -0.013353108428418636, -0.01567007228732109, 0.0078015378676354885, 0.01999478228390217, -0.012480019591748714, -0.0018895557150244713, 0.0558556504547596, 0.009311213158071041, 0.011407225392758846, -0.01593930460512638, 0.01040681917220354, -0.016477743163704872, -0.015456152148544788, 0.015233304351568222, -0.010421233251690865, -0.020506398752331734, -0.019916534423828125, -0.05360692739486694, -0.002647886984050274, -0.024362601339817047, -0.012155731208622456, -0.005472164135426283, 0.004010201431810856, -0.05931907519698143, 0.0107726426795125, 0.00254497560672462, -0.06388083100318909, 0.029465917497873306, -0.0328066349029541, 0.022815342992544174, 0.061763882637023926, -0.011771352961659431, -0.016787275671958923, 0.021538637578487396, -0.04910263419151306, 0.0004691214708145708, 0.024024272337555885, -0.00014314094732981175, 0.0036168487276881933, 0.009496835991740227, -0.01470380462706089, 0.018747376278042793, 0.0051588439382612705, -0.01899317093193531, 0.027685701847076416, 0.0173437912017107, 0.00333967967890203, -0.023270273581147194, -0.011240028776228428, -0.018031446263194084, -0.020502042025327682, 0.0027853173669427633, 0.02397986873984337, 0.0034095856826752424, 0.015675058588385582, -0.023538997396826744, -0.003714675083756447, -0.07172119617462158, 0.024833694100379944, 0.005446422845125198, -0.010567557998001575, 0.01761908084154129, 0.027773745357990265, -0.06003197282552719, 0.008203262463212013, 0.04592101648449898, -0.003759612562134862, 0.03872893378138542, 0.000637026852928102, -0.016128674149513245, 0.027749378234148026, 0.020689455792307854, 0.043033186346292496, -0.011636469513177872, 0.017574040219187737, 0.040831658989191055, -0.02239941619336605, -0.014369340613484383, -0.027336005121469498, -0.06060318648815155, 8.740948396734893e-05, -0.003462204709649086, -0.007748249918222427, -0.019408071413636208, -0.030214320868253708, -0.017765166237950325, 0.003187153721228242, 0.024475688114762306, -0.015627875924110413, 0.011804023757576942, -0.032673001289367676, -0.0016439975006505847, 0.01910501904785633, -0.01836889423429966, 0.009387371130287647, 0.04845472052693367, -0.003401883179321885, 0.009417284280061722, -0.04160391166806221, 0.016411444172263145, -0.026306606829166412, 0.0019402800826355815, -0.013892381452023983, 0.02187993749976158, 0.07429564744234085, -0.03095911256968975, -0.005454485770314932, -0.056527554988861084, 0.027824604883790016, -0.013730122707784176, 0.0005081318086013198, 0.01153672207146883, 0.03359616920351982, -0.0012027540942654014, -0.03920558840036392, 0.018635844811797142, 0.016369324177503586, 0.01640564575791359, -0.044065989553928375, 0.05047209933400154, -0.0017433034954592586, 0.0067632440477609634, -0.040135499089956284, 0.010914194397628307, -0.0027776500210165977, -0.01082627847790718, 0.03414163365960121, 0.0002617358695715666, -0.02363281138241291, 0.017509257420897484, -0.012187422253191471, 0.01795489713549614, -0.023270271718502045, 0.016985947266221046, 0.010296466760337353, -0.005748153664171696, -0.011751183308660984, 0.00440886989235878, -0.03396143019199371, -0.002537389285862446, -0.00412872014567256, 0.004568542819470167, 0.017964445054531097, -0.026699595153331757, 0.03027685172855854, 0.07924942672252655, -0.019659528508782387, 0.0063460092060267925, -0.05678523704409599, 0.0020370460115373135, 0.023836057633161545, -0.006188530009239912, 0.0150058688595891, 0.01658160425722599, 0.012379775755107403, 0.024467833340168, -0.0016463767969980836, 0.0006669873255304992, -0.022058479487895966, 0.046114757657051086, 0.0026398475747555494, -0.030010886490345, 0.022711381316184998, -0.007385540287941694, -0.009580333717167377, -0.006511279381811619, 0.012948501855134964, -0.02226998470723629, 0.0009723281254991889, 0.01507970504462719, 0.030894115567207336, 0.042320020496845245, -0.025859933346509933, -0.02969658188521862, -0.0035591418854892254, 0.03041519783437252, 0.028676031157374382, 0.030904127284884453, -0.00750764412805438, 0.012277154251933098, -0.008812247775495052, 0.042522188276052475, -0.008356521837413311, 0.014156188815832138, 0.013510899618268013, 0.006637446582317352, 0.02770908921957016, 0.010280936025083065, 0.020881596952676773, 0.020508164539933205, -0.05030370131134987, -0.01158513780683279, -0.05196692422032356, -0.03739466890692711, -0.02703390270471573, 0.00022725922462996095, -0.024060502648353577, -0.01413017138838768, 0.004758871626108885, 0.03969905152916908, 0.04604190215468407, 0.05162542685866356, 0.007609803229570389, 0.028774354606866837, -0.035148799419403076, -0.014349160715937614, -0.004366996698081493, -0.01131849829107523, -0.030763162299990654, -0.026934348046779633, 0.014852012507617474, -0.021491989493370056, 0.018528509885072708, 0.012983194552361965, 0.012152331881225109, -0.004434334114193916, 0.02805054746568203, 0.01076567079871893, 0.03343850374221802, -0.023046432062983513, 0.02403724566102028, -0.01788180135190487, -0.027470361441373825, -0.019807612523436546, 0.013402643613517284, 0.021238703280687332, -0.025382090359926224, -0.007949618622660637, -0.003197775222361088, -0.03465905785560608, -0.008697198703885078, 0.013086625374853611, -0.06013752892613411, -0.021080564707517624, 0.048595838248729706, 0.5746471881866455, -0.0021638397593051195, 0.03721684217453003, 0.014794094488024712, -0.03947456553578377, -0.022147804498672485, 0.034564051777124405, 0.04614078253507614, 0.008092629723250866, -0.03291228413581848, -0.018896853551268578, 0.029652366414666176, 0.010371068492531776, 0.012120302766561508, -0.04074671119451523, 0.005389431491494179, -0.006576988380402327, 0.010417456738650799, -0.04365498572587967, 0.01822865568101406, -0.03458253666758537, -0.005856449715793133, -0.03504599258303642, 0.01804318092763424, 0.05023808777332306, 0.005149743985384703, 0.011843160726130009, -0.016264401376247406, -0.02366838976740837, 0.05441836267709732, 0.010900384746491909, 0.010003735311329365, 0.04139552265405655, 0.04832203686237335, -0.0169498510658741, -0.05873308703303337, -0.0070053269155323505, -0.01019855123013258, 0.002311797346919775, -0.033183321356773376, -0.06478922814130783, 0.017169345170259476, -0.031544044613838196, 0.008034556172788143, 0.03132687509059906, 0.03483744338154793, 0.008136745542287827, 0.033466458320617676, 0.014695783145725727, 0.03261697292327881, 0.021151773631572723, -0.013828095048666, 0.004981762729585171, 0.008546554483473301, -0.00228811870329082, 0.012718578800559044, 0.017705481499433517, -0.005309815052896738, 0.017700104042887688, -0.014034923166036606, -0.043942879885435104, -0.008494568057358265, -0.06477055698633194, -0.061298176646232605, 0.04511111229658127, 0.025982793420553207, -0.013633434660732746, 0.011613614857196808, -0.0038702497258782387, 0.025722192600369453, -0.02156665176153183, -0.013843524269759655, -0.019195087254047394, 0.003909241408109665, -0.00284085632301867, 0.014906622469425201, -0.02294773794710636, 0.019121170043945312, 0.024145513772964478, -0.0018097271677106619, -0.010218561626970768, -0.006205220706760883, -0.003641583491116762, -0.05635327100753784, 0.01627141423523426, -0.025481052696704865, -0.020158687606453896, -0.019329208880662918, 0.009889199398458004, 0.004221049137413502, -0.011453350074589252, -0.006353664677590132, -0.014813502319157124, 0.0057856738567352295, -0.0151705676689744, -0.008650255389511585, 0.023376530036330223, -0.018713217228651047, -0.01538961473852396, -0.007501063868403435, 0.031248388811945915, -0.026506084948778152, 0.004908808972686529, 0.01712685450911522, 0.03482547402381897, 0.05687278136610985, -0.013556968420743942, 0.011842670850455761, -0.034033969044685364, 0.0014703335473313928, 0.030604325234889984, -0.002322182059288025, 0.03477587178349495, -0.017538288608193398, 0.0041997553780674934, 0.0047328881919384, 0.033893514424562454, 0.0323948860168457, -0.004277077503502369, 0.049560029059648514, 0.016485784202814102, -0.015047798864543438, 0.04932660609483719, 0.016953496262431145, 0.0033094591926783323, -0.014709779992699623, 0.04585457593202591, -0.06718432903289795, -0.01899970881640911, -0.016447851434350014, -0.03367644175887108, -0.011245781555771828, -0.01111549325287342, -0.04392039403319359, 0.008601372130215168, 0.03426558896899223, -0.011473428457975388, 0.03415505215525627, 0.027992060407996178, 0.0001421473571099341, -0.008254858665168285, -0.04086121916770935, 0.036229219287633896, 0.022941293194890022, 0.005215071141719818, 0.07886680960655212, 0.036939457058906555, 0.002673549111932516, -0.10249343514442444, -0.0017184969037771225, 0.01927855610847473, -0.002325860084965825, -0.03080853633582592, 0.012175123207271099, 0.0032206315081566572, -0.05238082632422447, 0.005457645282149315, -0.006848548538982868, -0.01715165004134178, -0.007060299161821604, -0.02219846285879612, 0.035803575068712234, 0.0066286600194871426, 0.03596895933151245, 0.046367935836315155, -0.017706269398331642, 0.012560607865452766, 0.045062821358442307, -0.0014402103843167424, -0.0211991835385561, -0.01800517365336418, -0.044337350875139236, -0.01188370119780302, 0.017905034124851227, -0.016492389142513275, 0.019060958176851273, 0.011913161724805832, -0.024502212181687355, 0.019072741270065308, 0.06189027801156044, -0.035829633474349976, 0.007750152610242367, -0.0019259901018813252, -0.022864658385515213, 0.03640781342983246, 0.008797001093626022, -0.004960386082530022, -0.004357203375548124, 0.009424425661563873, 0.03687794879078865, 0.01135780569165945, 0.009572775103151798, 0.009095140732824802, 0.02587467059493065, -0.012418880127370358, 0.0256170816719532, -0.04344342648983002, 0.011957569047808647, -0.012683081440627575, -0.022889496758580208, 0.007853744551539421, 0.0060508968308568, 0.007017918862402439, 0.019497089087963104, -0.027013670653104782, -0.00024802403640933335, 0.0010379462037235498, 0.012957148253917694, -0.012981301173567772, -0.010672777891159058, 0.013576127588748932, -0.0037575585301965475, -0.04549695551395416, 0.02176765911281109, -0.027545280754566193, -0.004687804263085127, -0.005261702463030815, -0.05454704910516739, 0.017903821542859077, -0.002932162955403328, -0.011676367372274399, 0.02053121104836464, 0.029936011880636215, -0.03966178745031357, -0.02292674593627453, -0.0011938008246943355, 0.04818391054868698, -0.0024696614127606153, -0.01035589911043644, 0.04181589186191559, -0.007086696103215218, -0.007971778512001038, 0.002740161493420601, -0.0029806047677993774, 0.004426533821970224, -0.041136786341667175, -0.0395740307867527, 0.007536514662206173, 0.00802573747932911, 0.0072229099459946156, 0.01623784378170967, -0.050235264003276825, 0.05245588347315788, -0.01324340421706438, -0.0754217728972435, -0.036805637180805206, 0.017956813797354698, -0.007900218479335308, 0.02120318077504635, -0.023420074954628944, 0.014294152148067951, -0.02609657123684883, -0.013298343867063522, -0.006681214086711407, -0.003594182664528489, -0.008464593440294266, 0.06334807723760605, -0.033813830465078354, -0.058477193117141724, 0.031723473221063614, -0.004404939245432615, 0.043316684663295746, -0.03351032733917236, 0.058384351432323456, 0.009602141566574574, 0.020903712138533592, 0.014359324239194393, 0.001979660475626588, 0.030598707497119904, -0.02570374496281147, 0.01658586785197258, 0.014222204685211182, -0.0029670479707419872, -0.02566392719745636, 0.01964760012924671, -0.012916591949760914, -0.001421055174432695, 0.005872574634850025, 0.038944993168115616, 0.03718577325344086, -0.037723708897829056, -0.014625650830566883, 0.019269216805696487, -0.0009177820757031441, -0.023626266047358513, -0.03260182589292526, 0.03893987834453583, 0.058539509773254395, 0.02068711817264557, -0.02304176799952984, 0.05308017507195473, -0.04276299849152565, 0.01550788152962923, 0.005202260334044695, 0.009175985120236874, -0.0026549464091658592, 0.009050455875694752, 0.0026646440383046865, 0.015612011775374413, -0.013862698338925838, -0.023312589153647423, 0.012424957007169724, -0.017054760828614235, -0.053136661648750305, 0.0054754591546952724, 0.015946844592690468, 0.07346971333026886, 0.019685646519064903, 0.03374848887324333, 0.033535730093717575, 0.006993355695158243, -0.01756281591951847, -0.017167454585433006, -0.00541924312710762, -0.018584920093417168, -0.04619469866156578, -0.01813378743827343, 0.007630879525095224, -0.03221938759088516, 0.0058021885342895985, -0.022397788241505623, 0.010480575263500214, 0.0005184513283893466, 0.003923013806343079, 0.2638297379016876, -0.010564211755990982, 0.015073212794959545, -0.017756134271621704, -0.003912237007170916, 0.04998406022787094, 0.004050089046359062, 0.0329083651304245, -0.008806223049759865, 0.0008881740504875779, -0.030628705397248268, 0.027756810188293457, -0.01794704981148243, 0.0077681164257228374, -0.022281719371676445, 0.0013125429395586252, -0.05108371004462242, 0.009379249997437, -0.005522694904357195, -0.00697331503033638, 0.012776514515280724, -0.0003251535235904157, -0.016247306019067764, -0.05993266776204109, -0.02596137672662735, -0.0022428559605032206, 0.030315948650240898, 0.027421506121754646, -0.020837198942899704, 0.00774557888507843, -0.018519321456551552, -0.007874926552176476, -0.015227328054606915, -0.02390533871948719, 0.0029032803140580654, -0.0615825392305851, 0.0009039205615408719], [-0.04733856022357941, 0.05632166936993599, -0.034485142678022385, 0.07577801495790482, 0.007138376589864492, 0.044992201030254364, 0.019897568970918655, -0.011995320208370686, -0.018586041405797005, -0.002242158865556121, -0.016495337709784508, -0.028054559603333473, 2.6621288270689547e-05, 0.019824495539069176, 0.0019987933337688446, 0.032489974051713943, -0.007294998969882727, 0.01101282611489296, -0.005731170531362295, -0.013397703878581524, 0.037639547139406204, 0.0052023474127054214, 0.02570444345474243, 0.022724652662873268, 0.022542482241988182, 0.02535773627460003, 0.011676459573209286, 0.045200519263744354, -0.016307514160871506, 0.03563443571329117, 0.03276569023728371, -0.02216680347919464, -0.10987267643213272, -0.05940626561641693, 0.02157638967037201, 0.0056659141555428505, -0.019018054008483887, 0.026344358921051025, 0.023613762110471725, -0.25534191727638245, -0.040846701711416245, -0.03387316316366196, 0.05789940059185028, 0.003966923803091049, 0.012444274500012398, -0.024966126307845116, 0.031105276197195053, -0.017776334658265114, -0.011773322708904743, 0.003434071084484458, -0.0282357856631279, 0.020115261897444725, -0.024474330246448517, -0.026127852499485016, 0.02996799908578396, -0.004036475904285908, -0.02864936925470829, -0.03328811749815941, 0.028301149606704712, -0.004891381599009037, -0.0069894022308290005, -0.019560176879167557, -0.00543972197920084, -0.02548883855342865, -0.008582662791013718, -0.0016567105194553733, -0.0306589026004076, 0.06880298256874084, -0.002364962361752987, 0.01997384987771511, -0.014397963881492615, 0.0001701113214949146, 0.040078386664390564, -0.0010141822276636958, 0.030686426907777786, -0.006124215666204691, -0.031123414635658264, -0.05854719504714012, -0.013880202546715736, 0.02198694460093975, -0.007474221289157867, -0.026269657537341118, 0.007131047546863556, 0.016693158075213432, -0.0267295241355896, -0.049232713878154755, 0.06463197618722916, 0.03602886199951172, 0.018134694546461105, 0.012320339679718018, 0.031566932797431946, -0.003171496093273163, 0.023919815197587013, 0.0015765577554702759, 0.03815528377890587, 0.01722554862499237, -0.0020130302291363478, 0.0023837967310100794, -0.02993028610944748, 0.028456496074795723, 0.014442721381783485, 0.017404519021511078, -0.02929696999490261, -0.05005495250225067, -0.047280099242925644, -0.03555307537317276, -0.02943366952240467, 0.00472129974514246, -0.015278997831046581, -0.0010044599184766412, 0.012864897027611732, -0.009396751411259174, -0.04031156376004219, 0.03439539670944214, 0.009891427122056484, 0.015143923461437225, -0.006729753687977791, -0.02219008468091488, -0.016215180978178978, -0.0005026006256230175, 0.002161267213523388, -0.01915981061756611, -0.020071446895599365, 0.014358666725456715, -0.021145803853869438, 0.037602413445711136, -0.018027573823928833, 0.026985403150320053, -0.032398052513599396, 0.011264568194746971, -0.0052625807002186775, -0.006048271898180246, 0.03962089866399765, 0.028963664546608925, -0.02944924496114254, -0.02227833867073059, 0.027708541601896286, -0.03101196512579918, 0.02375646121799946, -0.016123104840517044, -0.03883713483810425, 0.0415029302239418, 0.006410506088286638, -0.01818188466131687, -0.007808461785316467, -0.017648495733737946, 0.011605702340602875, 0.01679947040975094, -0.009783931076526642, -0.00018809533503372222, -0.03572619706392288, 0.0066689178347587585, 0.016335248947143555, -0.003779045306146145, 0.018768077716231346, 0.020494893193244934, 0.020279332995414734, 0.05450994893908501, 0.021140435710549355, -0.01153651438653469, -0.0035966443829238415, -0.014157513156533241, 0.0029762990307062864, 0.012173151597380638, -0.011608550325036049, -0.0018207188695669174, 0.02181183733046055, -0.009885445237159729, -0.012461337260901928, 0.03964407742023468, 0.014166767708957195, -0.011100955307483673, 0.018067102879285812, -0.004300225526094437, 0.049148306250572205, -0.02601434662938118, 0.027323096990585327, -0.0007440219633281231, 0.04991026967763901, -0.002440195530653, -0.03300369530916214, -0.008631319738924503, 0.01915764808654785, 0.027817722409963608, 0.0007927745464257896, 0.002607085509225726, -0.00023540595429949462, -0.05383957549929619, -0.0595882274210453, 0.04876113682985306, -0.02504078671336174, 0.0005654341657646, 0.013365648686885834, -0.032948754727840424, -0.0025310476776212454, -0.033530861139297485, -0.013049104250967503, -0.010279126465320587, -0.04254506528377533, 0.008521155454218388, -0.011892050504684448, -0.025022946298122406, -0.05610257387161255, -0.01146046631038189, -0.012196527794003487, 0.02668718807399273, 0.07179111242294312, 0.00600680336356163, -0.015467751771211624, 0.0006312562036328018, -0.02426164038479328, 0.006855532992631197, 0.01547444798052311, -0.018132394179701805, 0.034242190420627594, 0.03766747564077377, 0.05267464742064476, -0.004373061005026102, -0.039561882615089417, -0.012479420751333237, 0.027832182124257088, -0.030580883845686913, -0.04474364221096039, 0.04976446554064751, 0.016021467745304108, -0.0017514453502371907, -0.021111447364091873, 0.019022734835743904, -0.0059035527519881725, 0.037713050842285156, -0.002541624940931797, -0.007476466242223978, 0.03241024911403656, 0.005705877207219601, 0.0046160840429365635, -0.04723838344216347, -0.02918075956404209, 0.008712089620530605, -0.03658314049243927, -0.0002626681816764176, -0.009053226560354233, -0.030479269102215767, -0.015138690359890461, -0.046836599707603455, 0.0019604407716542482, 0.02613714337348938, 0.001860923133790493, -0.009364070370793343, -0.04572800546884537, -0.030651642009615898, -0.06838870048522949, -0.014729373157024384, 0.008441871032118797, -0.023304332047700882, -0.04174875095486641, -0.02274661883711815, 0.01961752213537693, -0.0050299386493861675, -0.017512992024421692, -0.017453454434871674, -0.0035334706772118807, 0.02101079374551773, -0.006394172552973032, -0.019298437982797623, -0.0030849601607769728, -0.002645415486767888, -0.01478171069175005, 0.03066692128777504, 0.009591118432581425, -0.02371651493012905, -0.02041110210120678, 0.013405345380306244, 0.004670938942581415, -0.009011665359139442, 0.010703129693865776, 0.033905722200870514, 0.020338131114840508, 0.0340101420879364, 0.04058002308011055, -0.024703864008188248, 0.04667387902736664, -0.026877539232373238, 0.03664633259177208, 0.02171309106051922, -0.011678147129714489, -0.014997909776866436, -0.04919927567243576, -0.03294754400849342, 0.02708672173321247, 0.05658084154129028, -0.0519774928689003, -0.03846505656838417, -0.003750043222680688, -0.009409235790371895, 0.04064071550965309, 0.0006120824837125838, -0.03304414078593254, -0.022654706612229347, -0.02114589884877205, 0.003949695732444525, -0.018845006823539734, -0.01594085991382599, -0.008918879553675652, -0.006119187455624342, -0.059585779905319214, -0.003904609940946102, -0.0281854048371315, -0.02613977901637554, 0.0663059651851654, 0.0035279891453683376, 0.02254825457930565, 0.020430637523531914, 0.003703635884448886, -0.009133104234933853, 0.05311432108283043, -0.012955847196280956, -0.02169201150536537, 0.03348657116293907, -0.06314805895090103, -0.010837449692189693, 0.02412356436252594, -0.00013706335448659956, 0.01126016117632389, -0.013137693516910076, 0.003776619676500559, -0.013377259485423565, -0.016419600695371628, -0.022227605804800987, 0.004189216066151857, -0.009892395697534084, -0.05721323564648628, -0.01259518414735794, 0.020605145022273064, -0.04678840935230255, 0.006445703562349081, -0.002510420512408018, -0.010635528713464737, -0.0038845117669552565, 0.022360000759363174, -0.010354399681091309, 0.022287821397185326, 0.03538858890533447, -0.04988262802362442, -0.01621738076210022, -0.011413645930588245, 0.06283839792013168, -0.016325736418366432, -0.04626711457967758, 0.0015348431188613176, 0.038164231926202774, -0.010215604677796364, 0.005022829864174128, 0.0520390160381794, 0.05121582746505737, -0.0026896728668361902, -0.00347928237169981, 0.0441715344786644, 0.01329854503273964, 0.0006163748330436647, -0.006722947582602501, 0.036787405610084534, -0.00692404992878437, -0.013189866207540035, 0.048936109989881516, -0.03495115041732788, -0.009329979307949543, -0.04149701073765755, -0.007053497713059187, 0.010377074591815472, -0.012961531057953835, -0.0311807319521904, 0.012602295726537704, 0.004332143347710371, 0.008209167048335075, -0.04667846858501434, 0.03264431655406952, 0.006353392731398344, 0.027412433177232742, 0.023277802392840385, -0.07667126506567001, -0.018392343074083328, 0.03120393492281437, -0.03382481634616852, -0.04225941747426987, -0.006935488432645798, -0.02255389839410782, -0.018963050097227097, 0.018765414133667946, -0.01846335455775261, 0.037049803882837296, 0.003955627791583538, 0.021563757210969925, -0.01662270911037922, -0.01427440159022808, -0.0022209661547094584, -0.022102665156126022, -0.00033413420896977186, -0.054883912205696106, -0.0028791679069399834, -0.03629471734166145, 0.02666889689862728, 0.008443791419267654, 0.016140516847372055, 0.02098393440246582, -0.03466309607028961, -0.015731828287243843, 0.010946898721158504, 0.5487937927246094, 0.007689827121794224, 0.024024002254009247, -0.02290819026529789, 0.004160651937127113, 0.02363581582903862, -0.046751488000154495, -0.004360450431704521, 0.050348907709121704, -0.03728916868567467, -0.006592592224478722, 0.026203488931059837, -0.017380958423018456, 0.04301954805850983, -0.0234213937073946, 0.003436734899878502, 0.04107823967933655, -0.021495338529348373, -0.0003013263922184706, 0.02845284901559353, -0.006841323338449001, 0.005404993426054716, 0.057400405406951904, -0.003933349158614874, 0.046318139880895615, 0.0315239243209362, 0.003518698737025261, -0.019566060975193977, -0.012923338450491428, 0.008416197262704372, 0.005980143789201975, -0.019223639741539955, -0.01940537430346012, 0.008462408557534218, -0.011148088611662388, 0.023971404880285263, 0.017886685207486153, 0.01095545757561922, -0.007613617926836014, -0.005415323190391064, 0.0290639940649271, -0.006773334927856922, -0.03743257746100426, 0.030659504234790802, -0.025527730584144592, -0.049527883529663086, -0.00685191061347723, 0.0004929990391246974, -0.03108610212802887, 0.04439308121800423, -0.048281993716955185, 0.015526814386248589, -0.010801980271935463, 0.033858299255371094, 0.027140652760863304, 0.005046084988862276, 0.0007494744495488703, -0.013745809905230999, -0.026411214843392372, 0.025167370215058327, -0.015352735295891762, -0.002342310268431902, -0.009895569644868374, -0.004397868178784847, -0.03719290345907211, -0.03929979354143143, -0.009362258017063141, 0.029522519558668137, -0.016720017418265343, 0.018473446369171143, -0.051554080098867416, -0.017423266544938087, -0.05851856246590614, -0.010875930078327656, 0.007700926624238491, 0.018709657713770866, -0.01848788373172283, 0.0405491478741169, 0.014018657617270947, 0.014642476104199886, -0.02787497267127037, 0.03327539190649986, 0.010157252661883831, 0.008241717703640461, -0.011080972850322723, -0.01095304824411869, -0.0027654748409986496, -0.012000499293208122, 0.005143006797879934, 0.04251698777079582, -0.027436764910817146, 0.02428615465760231, 0.04169971123337746, 0.05610739812254906, -0.008577548898756504, 0.015597639605402946, 0.02901860885322094, -0.029874131083488464, -0.0009767967276275158, -0.014335538260638714, 0.040790021419525146, -0.028521977365016937, 0.009493822231888771, 0.006047813221812248, 0.018970558419823647, 0.03070656582713127, -0.034320514649152756, 0.015218671411275864, 0.027212845161557198, 0.06291652470827103, 0.025179386138916016, 0.011495716869831085, 0.01385508943349123, -0.002086021238937974, 0.05520673468708992, -0.0014359962660819292, -0.007456119172275066, 0.0075558689422905445, -0.03285086527466774, 0.011794865131378174, -0.02209181897342205, 0.009225806221365929, -0.023586412891745567, -0.059796225279569626, 0.027049871161580086, 0.012661845423281193, -0.0521821565926075, -0.029367094859480858, -0.02536633424460888, 0.0015803248388692737, -0.029545865952968597, -0.022298527881503105, 0.0018592358101159334, -0.07949189096689224, 0.01747284270823002, -0.002320117549970746, -0.019698403775691986, 0.04495461657643318, 0.01078725140541792, 0.007403452415019274, 0.0208533126860857, -0.02628309279680252, -0.0011815950274467468, -0.0033268965780735016, -0.021733194589614868, -0.022784043103456497, 0.07542378455400467, 0.005238904617726803, -0.0033472797367721796, -0.0007948832935653627, 0.014734708704054356, -0.0011213214602321386, 0.03779522329568863, -0.005359348841011524, -0.056533779948949814, -0.0028865018393844366, 0.0038985353894531727, 0.02869703806936741, 0.036717336624860764, 0.05231814086437225, -0.018889345228672028, 0.02790830098092556, 0.007270338479429483, 0.039649270474910736, -0.001941413851454854, 0.032181642949581146, 0.00991637259721756, -0.010527066886425018, 0.046525806188583374, -0.016378991305828094, -0.03340582177042961, -0.013615918345749378, 0.03185795247554779, 0.05879386514425278, -0.0059484755620360374, 0.001517773256637156, -0.013257178477942944, 0.03273609280586243, 0.015976253896951675, -0.01185210607945919, 0.010631472803652287, 0.048417381942272186, -0.008164139464497566, 0.016300713643431664, 0.031860582530498505, 0.01445532776415348, 0.006931812036782503, 0.027462398633360863, -0.048923276364803314, -0.01908804476261139, -0.01189440954476595, -0.0402926430106163, -0.017829766497015953, 0.06392188370227814, 0.025101492181420326, 0.012461701408028603, 0.011342138051986694, -0.008282220922410488, -0.02305861935019493, 0.029016923159360886, -0.006843233946710825, 0.009841358289122581, -0.049876868724823, 0.02461123839020729, -0.015479272231459618, -0.024454856291413307, 0.0006529362290166318, 0.05699242278933525, -0.0338074192404747, 0.009967759251594543, -0.017728816717863083, -0.028825342655181885, 0.00210262811742723, -0.04359518364071846, 0.015398296527564526, -0.011291872709989548, -0.00016463108477182686, 0.014743008650839329, 0.0422930009663105, -0.03703897446393967, -0.056975752115249634, 0.032132796943187714, 0.014409146271646023, -0.0017305350629612803, -0.032943326979875565, 0.04176173359155655, -0.004334002733230591, 0.0008603095775470138, -0.00882288720458746, -0.028569184243679047, 0.018523022532463074, -0.009904109872877598, 0.031007839366793633, -0.02721254527568817, 0.018170373514294624, 0.012151651084423065, 0.00847067590802908, 0.030301570892333984, 0.03488847613334656, -0.002916107652708888, 0.04119332134723663, -0.013450524769723415, 0.033014096319675446, -0.04579661041498184, -0.013471814803779125, -0.04150956869125366, -0.024362614378333092, -0.015414584428071976, 0.017847374081611633, 0.026056138798594475, -0.00458153709769249, -0.036123327910900116, -0.005147910676896572, -0.00281236763112247, -0.08053063601255417, 0.015409755520522594, -0.022743528708815575, -0.037957075983285904, -0.03201806917786598, 0.007554385811090469, -0.01845269650220871, 0.0835302472114563, 0.011531748808920383, 0.00643592793494463, -0.0005088312900625169, -0.016374608501791954, 0.014818324707448483, -0.025191934779286385, -0.02091296762228012, 0.01764930970966816, 0.03335397318005562, 0.0071317292749881744, 0.018002426251769066, 0.0006941526080481708, 0.002450455678626895, -0.007543983403593302, 0.015906620770692825, -0.004000017885118723, -0.03129718452692032, -0.013092830777168274, 0.05806829780340195, 0.011687969788908958, 0.014796961098909378, 0.009271507151424885, 0.016892658546566963, -0.06405051052570343, 0.04962936043739319, 0.04645302891731262, 0.029477117583155632, -0.03222339227795601, 0.019763996824622154, 0.010189536027610302, 0.025536125525832176, 0.01914229989051819, 0.030429599806666374, 0.010904138907790184, 0.010231340304017067, -0.02333707921206951, -0.0007251431234180927, -0.002749625826254487, 0.027375569567084312, -0.044404227286577225, -0.021716348826885223, -0.02934306487441063, -0.024788254871964455, -0.004918346647173166, -0.0009685744880698621, -0.05653664842247963, 0.03212771937251091, 0.005499642342329025, -0.01682705245912075, 0.0038578095845878124, -0.03381246700882912, -0.01595030538737774, 0.02369961515069008, -0.02539604902267456, -0.04581836983561516, 0.03479164093732834, -0.005823908373713493, 0.00882470142096281, -0.012912201695144176, -0.03618080914020538, -0.010947407223284245, -0.003881186945363879, 0.20777712762355804, 0.03996175527572632, -0.006889685522764921, 0.018619496375322342, 0.022096332162618637, 0.06538017839193344, -0.018782339990139008, -0.008083407767117023, -0.019665822386741638, 0.005800278857350349, -0.07472991198301315, -0.08663196861743927, -0.013160294853150845, 0.021463854238390923, -0.010536298155784607, 0.03239307925105095, 0.023067183792591095, -0.006991229020059109, -0.03388296812772751, -0.0021192149724811316, -0.0004060578648932278, 0.008340765722095966, 0.024670200422406197, 0.023291707038879395, -0.02302461490035057, -0.0314331017434597, 0.034949302673339844, 0.023937804624438286, -0.025532377883791924, 0.03022584691643715, 0.013374283909797668, -0.030988290905952454, 0.0077906521037220955, -0.03173501044511795, -0.007838317193090916, -0.003946456592530012, 0.011939875781536102], [0.04150858893990517, 0.004914832301437855, -0.03761454299092293, 0.016169555485248566, -0.020314812660217285, -0.018375232815742493, 0.01866663061082363, -0.027693886309862137, 0.0006398302502930164, 0.01029431726783514, 0.007546691689640284, -0.015559854917228222, 0.0361643061041832, -0.009227591566741467, 0.02394246496260166, -0.023871557787060738, 0.004756588488817215, 0.031986407935619354, 0.012862849980592728, -0.06132832542061806, -0.009288158267736435, 0.009579331614077091, -0.003666023025289178, 0.058402255177497864, 0.0057745762169361115, 0.024341152980923653, -0.00044007020187564194, -0.012113450095057487, -0.0277949720621109, 0.03131897374987602, -0.031811680644750595, -0.0272675808519125, -0.05948779359459877, -0.08367034792900085, 0.01608015038073063, -0.007625661324709654, -0.013870874419808388, -0.010154152289032936, 0.0086685074493289, -0.30788350105285645, 0.01584000326693058, 0.027152622118592262, -0.015945876017212868, 0.021639332175254822, -0.02015790529549122, -0.018585408106446266, -0.02240835502743721, -0.01862785406410694, 0.022343851625919342, -0.002376039745286107, -0.04229104518890381, -0.0007658306858502328, 0.005132046993821859, -0.0234801284968853, -0.014439811930060387, -0.024723608046770096, -0.008425826206803322, 0.006910242140293121, 0.03081474080681801, 0.016150426119565964, -0.05174132063984871, -0.010044407099485397, 0.04101874679327011, 0.008644577115774155, 0.016744311898946762, -0.023824060335755348, 0.004810929298400879, 0.07901342958211899, 8.303246431751177e-05, 0.03661031275987625, 0.012889259494841099, 0.015099544078111649, -0.008447415195405483, 0.003714054822921753, 0.04146777093410492, -0.042776137590408325, 0.00345366052351892, -0.040462009608745575, -0.0026361015625298023, 0.006098772399127483, 0.005971113685518503, -0.04152672365307808, -0.03340965136885643, -0.029844481498003006, 0.01017356663942337, -0.01339219231158495, 0.00831554178148508, -0.007928759790956974, 0.010594340041279793, -0.0007693720399402082, 0.012734551914036274, -0.0009951313259080052, -0.006230172235518694, -0.042138949036598206, 0.002621913328766823, -0.01891845464706421, -0.020437512546777725, 0.013806942850351334, -0.03912029042840004, -0.009613814763724804, 0.0047763013280928135, 0.02593260072171688, -0.005805060267448425, -0.037135690450668335, -0.033792492002248764, -0.01965172030031681, 0.019213777035474777, -0.023750975728034973, -0.020028356462717056, 0.04921866953372955, 0.004619986284524202, -0.04200546070933342, -0.0417090579867363, 0.023605480790138245, 0.009116783738136292, -0.011961421929299831, -0.024278702214360237, -0.013205110095441341, -0.017005618661642075, 0.007148300297558308, 0.0038114325143396854, 0.024455657228827477, 0.02513892948627472, -0.026503579691052437, 0.05029807984828949, 0.011865832842886448, 0.0034278100356459618, 0.0018624403746798635, -0.013115111738443375, 0.022772280499339104, -0.008700569160282612, -0.03833737596869469, -0.021452661603689194, -0.03784708306193352, -0.02779897302389145, 0.00980608630925417, 0.022823963314294815, 0.005506552755832672, 0.034303195774555206, -0.002317027887329459, -0.05841997638344765, 0.01780613511800766, 0.01363921444863081, 0.007879588752985, -0.024056008085608482, 0.025729579851031303, -0.030141038820147514, 0.02501789852976799, 0.04815642908215523, -0.0027385640423744917, -0.022672424092888832, -0.020822839811444283, 0.027477357536554337, 0.004431967623531818, -0.037505704909563065, 0.021885333582758904, 0.011617832817137241, 0.011666099540889263, -0.022774947807192802, -0.011656464077532291, -0.008825873956084251, 0.02102239616215229, 0.0320478118956089, 0.012000538408756256, 0.023175012320280075, 0.006416071206331253, -0.019603721797466278, 0.028639493510127068, 0.01132506225258112, -0.010976975783705711, 0.03016362152993679, -0.00363113172352314, 0.04875163361430168, -0.01931590586900711, -0.011093075387179852, 0.015437161549925804, -0.023401983082294464, 0.007240699138492346, 0.006307536736130714, 0.017255648970603943, 0.027320433408021927, -0.005208518356084824, 0.01717892289161682, -0.007315592374652624, 0.02249879017472267, 0.018674053251743317, -0.018324755132198334, 0.040161680430173874, -0.0035076180938631296, 0.0004305739712435752, -0.0023677682038396597, -0.014420337975025177, 0.02990064024925232, 0.028163516893982887, 0.00020873775065410882, -0.004144505597651005, 0.0008242611074820161, -0.030149701982736588, 0.0397372730076313, 0.01735496148467064, -0.013789715245366096, 0.03105497546494007, -0.011799588799476624, -0.03682880848646164, -0.04451167210936546, 0.04606687277555466, 0.021350011229515076, -0.00927094928920269, 0.03755684569478035, -0.019792288541793823, 0.02683199942111969, -0.019779203459620476, 0.029360750690102577, -0.0021928998176008463, 0.0073910243809223175, -0.013043837621808052, 0.025238431990146637, -0.01109487097710371, -0.021847618743777275, -0.007962058298289776, 0.010028406046330929, -0.035728953778743744, -0.05223939195275307, -0.0011352449655532837, 0.012292461469769478, 0.003830597037449479, -0.017640717327594757, 0.004910395946353674, 0.025761298835277557, 0.009545587003231049, -0.029505323618650436, -8.62189190229401e-05, 0.011618047021329403, -0.026005705818533897, 0.017221724614501, -0.09323932975530624, -0.008853343315422535, 0.032482970505952835, -0.022355886176228523, 0.026311902329325676, -0.009766168892383575, -0.025553405284881592, 0.04147554188966751, -0.019486116245388985, -0.018205339089035988, -0.002807385753840208, -0.010310497134923935, 0.018793590366840363, 0.002316097728908062, -0.032913219183683395, -0.0505417175590992, -0.0008294430444948375, -0.007987883873283863, -0.020254071801900864, 0.0319257527589798, -0.045978326350450516, -0.0032692798413336277, 0.006395358592271805, -0.02253696136176586, 0.00627762358635664, -0.013661312870681286, 0.013595106080174446, 0.0166565403342247, -0.029049547389149666, 0.006594160571694374, -0.027035493403673172, -0.002512473613023758, -0.006100178230553865, -0.0255417563021183, 0.007058900315314531, -0.04596506059169769, 0.019202470779418945, -0.017702747136354446, -0.0145602747797966, -0.008388173766434193, 0.04465516284108162, 0.014872522093355656, 0.01608775183558464, 0.016489040106534958, -0.030600877478718758, 0.039084624499082565, 0.04032336547970772, -0.010258704423904419, -0.026077326387166977, 0.03263493999838829, -0.007763265166431665, 0.029718171805143356, -0.01709802635014057, 0.01757821999490261, -0.0008120835409499705, -0.03134230524301529, 0.007147349417209625, -0.02688588760793209, 0.03261283040046692, 0.0331464521586895, 0.004204703960567713, 0.02111043967306614, 0.026280127465724945, -0.025675807148218155, 0.002408380853012204, 0.00348762352950871, 0.06973587721586227, -0.03674781322479248, -0.022978248074650764, -0.03352253511548042, -0.026408813893795013, -0.022200267761945724, 0.03362922742962837, 0.023602990433573723, -0.0129607655107975, -0.0013821570901200175, -0.04183981567621231, -0.0004621637344826013, -0.007253550458699465, 0.013725824654102325, 0.0053141373209655285, -0.04574379324913025, 0.08478990942239761, -0.02810106985270977, -0.03187866136431694, -0.01255109440535307, -0.009131216444075108, 0.02424081601202488, 0.024407537654042244, 0.015934068709611893, -0.013850163668394089, -0.017669163644313812, -0.016602959483861923, 0.0012556348228827119, -0.028754616156220436, 0.016977820545434952, 0.0005568431806750596, -0.06912222504615784, -0.028415899723768234, 0.0008690467802807689, -0.02244495041668415, 0.012655475176870823, -0.016812415793538094, 0.05825779214501381, 0.0007006681407801807, 0.012626631185412407, -0.0006910263909958303, 0.041784144937992096, 0.018171755596995354, -0.008499838411808014, 0.017425308004021645, 0.016468185931444168, 0.014513806439936161, -0.0017350020352751017, 0.011325023137032986, -0.01265338808298111, -0.002916921628639102, -0.0024334718473255634, 0.06330535560846329, -0.029228325933218002, -0.018078666180372238, 0.043668460100889206, 0.008124240674078465, -0.022794105112552643, 0.011028284206986427, -0.023854322731494904, -0.026365108788013458, -0.0007602863479405642, 0.022619951516389847, 0.0068533592857420444, 0.02462172880768776, -0.01368445809930563, -0.018083008006215096, 0.011491870507597923, 0.025898613035678864, -0.008085633628070354, 0.026531711220741272, 0.001416489714756608, 0.007981469854712486, 0.033137429505586624, 0.030743414536118507, 0.007967904210090637, 0.034551139920949936, 0.0028160824440419674, -0.01710304245352745, 0.032751601189374924, 0.046769432723522186, 0.032893091440200806, -0.026649056002497673, -0.03181670978665352, 0.009884859435260296, -0.045166730880737305, -0.002259868197143078, -0.005415815860033035, 0.03140747547149658, -0.017926985397934914, 0.020486922934651375, -0.020133018493652344, -0.0007975318003445864, 0.007461459841579199, -0.01639515534043312, 0.0038233529776334763, 0.031430669128894806, -0.011641256511211395, 0.03804297745227814, -0.010392220690846443, 0.012501061893999577, 0.006829684134572744, -0.026600142940878868, -0.03518308699131012, -0.04710928723216057, 0.003108107252046466, 0.5968785285949707, 0.02139146253466606, -0.021015673875808716, 0.0014344153460115194, 0.0022061767522245646, 0.016949838027358055, -0.0012551987310871482, 0.00889113824814558, -0.023140080273151398, -0.024948101490736008, -0.015448817983269691, -0.012852360494434834, -0.03399711847305298, 0.02379189059138298, -0.0410657674074173, -0.04393891245126724, 0.01195333618670702, -0.01858539879322052, 0.018180852755904198, 0.024333197623491287, -0.01159137673676014, 0.00841202400624752, -0.0005618894356302917, 0.016695154830813408, 0.026667632162570953, 0.04411042109131813, -0.001081894850358367, 0.019969336688518524, -0.035061515867710114, -0.001804339699447155, -0.02249077893793583, 0.03184112533926964, -0.02268299274146557, 0.0358283668756485, 0.03732777386903763, -0.03276153653860092, 0.03835524246096611, 0.021974869072437286, -0.01216393243521452, -0.009733661077916622, -0.03289363160729408, 0.03369411826133728, -0.025363624095916748, -0.026144960895180702, 0.016286663711071014, -0.03298116847872734, 0.0038582319393754005, 0.009590795263648033, 0.013966916128993034, 0.007370802108198404, 0.00790910329669714, 0.012866667471826077, -0.05701141431927681, 0.018003210425376892, -0.03484675660729408, 0.015806257724761963, 0.01084140595048666, -0.04994392395019531, -0.00010806084173964337, 0.008785325102508068, -0.014469792135059834, 0.008858488872647285, -0.010959935374557972, -0.04370139539241791, -0.008640563115477562, -0.0001331617240794003, 0.0007743361056782305, -0.02060328982770443, -0.0008915188955143094, 0.00021375896176323295, -0.002464092569425702, 0.013365494087338448, -0.003350401995703578, 0.023607473820447922, 0.055106598883867264, -0.05262554809451103, -0.025865202769637108, -0.010243586264550686, -0.011000659316778183, 0.0026145202573388815, -0.017204971984028816, 0.006446472834795713, 0.012692691758275032, -0.026712872087955475, 0.009602740406990051, -0.007728394586592913, -0.004885121714323759, 2.1556295905611478e-05, -0.02470717392861843, -0.01783169060945511, -0.01891016587615013, 0.04805309697985649, -0.04525672271847725, 0.042144011706113815, 0.044046614319086075, -0.00914849154651165, 0.002596677280962467, -0.013967175036668777, -0.07510477304458618, -0.025067811831831932, -0.006081255152821541, 0.034183308482170105, 0.019524576142430305, -0.018871257081627846, 0.009647129103541374, 0.07556615024805069, 0.010677899233996868, 0.01468176580965519, 0.01628325879573822, 0.017289994284510612, -0.012836867943406105, 0.014971797354519367, 0.0076768966391682625, 0.014347746036946774, -0.0031677614897489548, 0.02412283420562744, 0.03612643480300903, 0.004274002276360989, -0.020444048568606377, -0.020289018750190735, 0.018283311277627945, -0.008959418162703514, -0.012669909745454788, 0.0011578720295801759, 0.07603244483470917, -0.013409525156021118, 0.027025604620575905, 0.002835645107552409, -0.018156275153160095, -0.02941174991428852, -0.0039371950551867485, 0.01308736577630043, 0.003617360722273588, -0.04996635392308235, 0.03518831357359886, 0.015079631470143795, 0.013819153420627117, 0.032222848385572433, -0.044977180659770966, 0.004914066754281521, 0.02411673218011856, 0.017867736518383026, 0.04078492149710655, 0.022802937775850296, -0.03572731837630272, -0.030514990910887718, 0.04864028841257095, -0.006690250709652901, -0.12249256670475006, -0.0024295486509799957, 0.030972931534051895, 0.03275404870510101, -0.038547333329916, -0.0071995314210653305, 0.0150013267993927, -0.019710876047611237, -0.007055988535284996, -0.004927315283566713, -0.009715864434838295, -0.006215418688952923, -0.0073796710930764675, 0.008069396950304508, 0.02278446964919567, 0.0007732578087598085, -0.00309800636023283, -0.008667433634400368, -0.002022342523559928, -0.013079901225864887, 0.04302360862493515, 0.017550265416502953, -0.00830526277422905, -0.0499478317797184, -0.0015743167605251074, 0.006871283520013094, 0.028185144066810608, 0.027191918343305588, -0.005378128960728645, -0.07546906173229218, 0.047183047980070114, 0.015487341210246086, -0.036819472908973694, -0.013066676445305347, -0.002517006127163768, -0.020799947902560234, 0.009024464525282383, -0.053677257150411606, -0.01464003138244152, 0.025737080723047256, 0.036762576550245285, -0.008806485682725906, 0.009800209663808346, 0.011255396530032158, 0.0120685501024127, -0.03149695694446564, 0.028497982770204544, -0.013414372690021992, 0.010717138648033142, 0.03739171102643013, 0.003582597244530916, 0.024728113785386086, -0.002340012462809682, 0.02961450070142746, -0.08504566550254822, -0.02115391381084919, 0.08022933453321457, 0.017565149813890457, -0.01800680346786976, 0.004383709281682968, 0.007454734295606613, 0.027440128847956657, -0.010621393099427223, -0.05142798274755478, 0.0026805440429598093, 0.018252933397889137, -0.017791299149394035, 0.0038616894744336605, -0.013852333649992943, -0.0045608337968587875, -0.013840045779943466, 0.013804689049720764, -0.061135292053222656, 0.03963663801550865, 0.003001066856086254, 0.009934318251907825, -0.015070803463459015, 0.03550693392753601, 0.00209896732121706, -0.010576506145298481, -0.0013683618744835258, -0.008857708424329758, -0.02603686973452568, 0.0030029986519366503, 0.019361626356840134, -0.02753523550927639, 0.018793579190969467, 0.030575571581721306, -0.006681131199002266, -0.003169027157127857, -0.006676424760371447, -0.020936114713549614, -0.0148440757766366, 0.01153882872313261, 0.05760888382792473, -0.04574333131313324, -0.025342252105474472, 0.0028846063651144505, 0.046057358384132385, -0.010596481151878834, 0.029271714389324188, -0.013027640990912914, 0.03964843973517418, -0.03814748674631119, -0.014489613473415375, -0.02761099301278591, -0.013634205795824528, 0.010307938791811466, -0.021008459851145744, -0.06040625646710396, -0.00813204888254404, -0.006397987250238657, -0.0033812220208346844, 0.005153832957148552, -0.006548183038830757, -0.017426472157239914, 0.015068495646119118, -0.019352402538061142, -0.006835784763097763, 0.01572006195783615, -0.005261251702904701, 0.008266369812190533, -0.00690017594024539, -0.016314677894115448, 0.0005170052172616124, -0.05486411601305008, -0.008156212978065014, -0.02931792661547661, -0.032794926315546036, 0.007108556106686592, -0.020500581711530685, -0.014477303251624107, -0.00497443089261651, 0.01526602078229189, -0.009131528437137604, 0.02921173721551895, 0.0040460508316755295, -0.025677740573883057, -0.001480037230066955, 0.03961740434169769, 0.002564458642154932, 0.0045325178653001785, 0.006892170757055283, 0.029066143557429314, -0.04337061196565628, -0.008352569304406643, 0.021063053980469704, 0.020366229116916656, 0.018697639927268028, 0.013019518926739693, 0.005684887990355492, 0.008359246887266636, 0.013508804142475128, 0.013403674587607384, -0.016791360452771187, 0.0440157875418663, -0.004066108725965023, 0.032830823212862015, -0.01639745384454727, -0.01786598190665245, 0.03365461528301239, 0.004225574433803558, -0.020469777286052704, 0.0058364770375192165, 0.044228918850421906, -0.013444455340504646, -0.008711847476661205, -0.016760986298322678, -0.02305297553539276, -0.012102734297513962, 0.017491746693849564, -0.012633119709789753, -0.019204208627343178, -0.020428547635674477, 0.014361418783664703, 0.0450429730117321, 0.21367745101451874, 0.015052849426865578, -0.027097828686237335, -0.02947433851659298, 0.014113147743046284, 0.025732140988111496, 0.0004819419700652361, -0.010868487879633904, -0.0033568411599844694, -0.02763376384973526, -0.0019905182998627424, -0.08116235584020615, 0.013707315549254417, 0.005476766731590033, 0.012167749926447868, -0.012108023278415203, -0.020619112998247147, -0.025299977511167526, 0.02252749539911747, 0.00445486418902874, -0.006559371016919613, 0.00276387482881546, -0.0059562865644693375, 0.014792208559811115, 0.008406493812799454, -0.009951948188245296, -0.007074355147778988, 0.009672587737441063, -0.03452044352889061, 0.012163014151155949, 0.040922556072473526, -0.0035614350344985723, 0.012458417564630508, -0.029401404783129692, -0.013196039013564587, 0.017450764775276184, 0.03017435222864151], [0.0460958406329155, -0.01457084622234106, -0.016272885724902153, 0.007017883472144604, 0.0002736650058068335, -0.017727483063936234, 0.012788927182555199, -0.03854664787650108, -0.010629785247147083, 0.013253954239189625, 0.014451577328145504, -0.03768035024404526, 0.03254793956875801, -0.002694502007216215, 0.021192407235503197, -0.005239701364189386, -0.006495155859738588, 0.025866009294986725, 0.01514085941016674, -0.03431756794452667, -0.017456592991948128, 0.027442455291748047, 0.0011805477552115917, 0.07725223898887634, 0.002332966076210141, 0.022675490006804466, 0.0051652672700583935, -0.020605655387043953, -0.022676479071378708, 0.010086898691952229, -0.025539759546518326, -0.015679888427257538, -0.07918959110975266, -0.07133210450410843, 0.018998628482222557, -0.0034838891588151455, -0.007719636894762516, -0.014034749940037727, -0.019861914217472076, -0.26678338646888733, 0.046429481357336044, 0.0380060039460659, 0.009539011865854263, 0.012259921990334988, -0.010141286067664623, -0.0029561668634414673, -0.03714728355407715, -0.019086718559265137, 0.0257415771484375, -0.013787801377475262, -0.03516209498047829, 0.007890074513852596, 0.0046735405921936035, -0.02988719940185547, -0.023116132244467735, -0.008088123053312302, -0.01608811318874359, -0.027191689237952232, 0.041114307940006256, 0.0131918266415596, -0.035553183406591415, -0.02224227972328663, 0.020175859332084656, 0.0052192010916769505, 0.009142562747001648, -0.006357505917549133, 0.0019083749502897263, 0.04883415624499321, -0.015459438785910606, 0.025488756597042084, 0.03246914967894554, 0.017458338290452957, -0.010569880716502666, 0.0049341521225869656, 0.03824065625667572, -0.040063876658678055, 0.01688014343380928, -0.0331559032201767, 0.005808087531477213, 0.02247011847794056, -0.037092775106430054, -0.031615424901247025, -0.02807902731001377, -0.031584013253450394, 0.01790057122707367, -0.020024996250867844, -0.02190709300339222, -0.02293938212096691, -0.002079842146486044, 0.0124851418659091, 0.021358780562877655, 0.004731452092528343, -0.016961969435214996, -0.029433811083436012, 0.00478268601000309, -0.028155991807579994, -0.026848413050174713, -0.0034989304840564728, -0.0553680919110775, -0.011867618188261986, 0.0023156290408223867, 0.024693621322512627, -0.02198617160320282, -0.055113550275564194, -0.021413378417491913, 0.011323103681206703, 0.0012943490874022245, -0.022598808631300926, -0.026400716975331306, 0.0536976158618927, 0.012624944560229778, -0.03466472402215004, -0.010724445804953575, 0.019297126680612564, 0.0025147648993879557, -0.011729676276445389, -0.02125799097120762, -0.0066391522996127605, -0.014863944612443447, 0.0065749273635447025, -0.0007009133696556091, 0.019327176734805107, 0.006498607806861401, -0.023141859099268913, 0.05161977931857109, 0.008939594961702824, 0.001111742458306253, -0.00284613692201674, -0.012559906579554081, 0.014349158853292465, 0.0020201895385980606, -0.032057344913482666, 0.013032096438109875, -0.06921786814928055, -0.022150440141558647, 0.021439403295516968, 0.016272129490971565, 0.005040227901190519, 0.027203653007745743, -0.02343452163040638, -0.032539356499910355, 0.03942902013659477, 0.009143013507127762, 0.017905758693814278, -0.013394060544669628, 0.034677401185035706, -0.018219908699393272, 0.025536900386214256, 0.07920907437801361, -0.012725180014967918, -0.013629589229822159, -0.039672695100307465, 0.017789283767342567, 0.007286076433956623, -0.0350392609834671, 0.013964132405817509, 0.01494472473859787, 0.029841117560863495, -0.02303522638976574, -0.029986457899212837, -0.022148307412862778, 0.028704669326543808, 0.05204539746046066, 0.015379301272332668, 0.039309464395046234, -0.009667700156569481, -0.03317367285490036, 0.012312375009059906, 0.01623990759253502, -0.015515701845288277, 0.008039192296564579, -0.0019519306952133775, 0.048693567514419556, -0.013478834182024002, 0.003849870292469859, 0.011676822789013386, -0.007157516200095415, -0.012562796473503113, 0.012898951768875122, 0.02377975359559059, 0.0009455170365981758, -0.008438152261078358, 0.01987951435148716, 0.010143738240003586, 0.02286725491285324, 0.031468573957681656, -0.02180670015513897, 0.03985176607966423, 0.0008185294573195279, -0.022912122309207916, -0.011455276049673557, -0.008127018809318542, 0.027853021398186684, 0.009955246932804585, 0.0055129146203398705, -0.003535947762429714, -0.01870706118643284, -0.009725170210003853, 0.045763321220874786, 0.003303066361695528, -0.004924325738102198, 0.03582076355814934, -0.015787869691848755, -0.037611693143844604, -0.041181255131959915, 0.021632764488458633, 0.033602286130189896, 0.005249035079032183, 0.04038059711456299, -0.030504826456308365, 0.03097872994840145, -0.011301188729703426, 0.042364880442619324, 0.010187401436269283, -0.0031547890976071358, -0.01217521633952856, 0.014690853655338287, -0.023133032023906708, -0.03173139691352844, -0.0052702114917337894, 0.010827461257576942, -0.0625268891453743, -0.04656132310628891, 0.01570112444460392, 0.010334251448512077, 0.004780035465955734, -0.014196326024830341, 0.012066861614584923, 0.028710274025797844, 0.019221819937229156, -0.011352489702403545, 0.010690816678106785, 0.021089857444167137, -0.028658650815486908, 0.012398796156048775, -0.07657960057258606, -0.0061258613131940365, 0.030258290469646454, -0.0313725583255291, 0.03185020387172699, -0.01907164789736271, -0.016142087057232857, 0.02643241360783577, 0.00355969974771142, -0.009138813242316246, -0.007791627664119005, -0.011359838768839836, 0.02766050025820732, -0.013265589252114296, -0.04178127273917198, -0.03559793904423714, -0.0026134925428777933, -0.011545435525476933, -0.023603128269314766, 0.02668185718357563, -0.05326230078935623, -0.013583142310380936, 0.001248458051122725, -0.026999233290553093, 0.018917009234428406, -0.006726691499352455, 0.01336456835269928, 0.02429460734128952, -0.026318076997995377, 0.006171050481498241, -0.03463316336274147, -0.009026850573718548, -0.019127044826745987, -0.02274479903280735, 0.03374398127198219, -0.04702422767877579, 0.0064981598407030106, -0.02596578560769558, 0.00935029610991478, -0.02840210683643818, 0.048138055950403214, 0.024083439260721207, 0.03004775568842888, 0.024996209889650345, -0.03473380208015442, 0.0550219863653183, 0.05905867740511894, -0.012575101107358932, -0.027875235304236412, 0.03575602173805237, 0.0013471655547618866, 0.02457505092024803, -0.017203325405716896, 0.023968538269400597, 0.006218785885721445, -0.050033312290906906, 0.0176912359893322, -0.03040899895131588, 0.03860419988632202, 0.018020935356616974, -0.004931723698973656, 0.026594065129756927, 0.003646946744993329, -0.021271664649248123, -0.0020435682963579893, 0.0025331813376396894, 0.08313173055648804, -0.04369829595088959, -0.025245383381843567, -0.02642841450870037, -0.03268551826477051, -0.011089634150266647, 0.05107346922159195, 0.03077324666082859, -0.00871854368597269, -0.001139312400482595, -0.04028928652405739, -0.0026886092964559793, -0.00709621561691165, 0.025234369561076164, 0.02140946313738823, -0.03723227232694626, 0.08542744070291519, -0.01616504043340683, -0.03434661403298378, -0.01653156243264675, -0.013069932349026203, 0.01919715851545334, 0.029628360643982887, 0.01634673960506916, 0.003811340080574155, -0.020140264183282852, -0.008376081474125385, 0.00987099390476942, -0.029748760163784027, 0.013183844275772572, 0.015991495922207832, -0.06295882165431976, -0.03490617498755455, -0.0041227880865335464, -0.028280990198254585, 0.009589936584234238, -0.021975088864564896, 0.07004279643297195, -0.011974523775279522, 0.00017309746181126684, -0.019798612222075462, 0.05806165933609009, 0.014675627462565899, -0.005605059210211039, 0.0014242137549445033, 0.02701599895954132, 0.007477234583348036, 0.0025382505264133215, -0.0025033392012119293, -0.002538040978834033, -0.005700005684047937, 0.005499155260622501, 0.08939895033836365, -0.02078489400446415, -0.011537330225110054, 0.035690151154994965, 0.010486660525202751, -0.028610479086637497, 0.011515993624925613, -0.033162590116262436, -0.017570920288562775, -0.05103999003767967, 0.013117386028170586, 0.0031175045296549797, 0.03797522559762001, 0.002374447649344802, -0.02511756680905819, -0.0018749359296634793, 0.025984466075897217, -0.007979734800755978, 0.028582779690623283, 0.009206047281622887, 0.0016620733076706529, 0.03263423964381218, 0.022631917148828506, 0.00609762454405427, 0.02407689392566681, 0.01534335222095251, -0.027641071006655693, 0.04264655336737633, 0.042886801064014435, 0.04392528906464577, -0.032408878207206726, -0.03018973022699356, 0.008747654967010021, -0.04728865996003151, 0.005304862279444933, 0.00011416243069106713, 0.08255814760923386, -0.01988873817026615, 0.004299628082662821, -0.005473125260323286, 0.003465827787294984, 0.0072737159207463264, -0.019051656126976013, 0.02583412453532219, 0.012744027189910412, -0.021756671369075775, 0.04212437570095062, -0.02071979083120823, 0.007951385341584682, 0.017283249646425247, -0.034547511488199234, -0.038653384894132614, -0.048811282962560654, 0.021995993331074715, 0.5874636769294739, 0.013156518340110779, -0.022017575800418854, -0.00029147727764211595, -0.0037362026050686836, 0.027351269498467445, 0.0034645134583115578, -0.00929927546530962, -0.014768517576158047, -0.04646207019686699, -0.009667162783443928, -0.021392058581113815, -0.03717749938368797, 0.01257842592895031, -0.02577289193868637, -0.04730594903230667, 0.017139334231615067, -0.0015822580317035317, 0.009685097262263298, 0.018610388040542603, -0.005729404743760824, 0.01245049573481083, 0.008658045902848244, 0.009716982953250408, 0.02003195695579052, 0.049358952790498734, 0.014237022027373314, 0.02398633398115635, -0.036181990057229996, 0.004110689740628004, -0.01989719457924366, 0.036252107471227646, -0.009801835753023624, 0.050185609608888626, 0.02644580602645874, -0.030493319034576416, 0.03908853605389595, 0.006843599025160074, -0.010838568210601807, -0.015878723934292793, -0.026148492470383644, 0.018640533089637756, -0.030420785769820213, -0.03785103186964989, 0.008673999458551407, 0.011845346540212631, 0.01231315266340971, 0.01595313474535942, 0.021026508882641792, 0.009084452874958515, 0.01599726267158985, 0.006081828847527504, -0.034021344035863876, 0.013176044449210167, -0.023875977843999863, 0.01669425703585148, 0.01825738325715065, -0.02781892567873001, -0.00208582216873765, -0.014242866076529026, -0.00959225744009018, 0.005375873297452927, 0.017118310555815697, -0.024596339091658592, -0.005969730671495199, -0.007134606130421162, 0.017626214772462845, -0.022629495710134506, 0.014473343268036842, 0.015982849523425102, -0.005202953238040209, 0.00805712677538395, 0.0027541574090719223, 0.01045249868184328, 0.0554368756711483, -0.052693258970975876, -0.030059311538934708, -0.013977224938571453, -0.0013725126627832651, -0.008217399008572102, -0.02148297242820263, 0.0036196617875248194, 0.00837627798318863, -0.011865509673953056, 0.010329095646739006, -0.005311485845595598, -0.003737501800060272, 0.004520323593169451, 0.003281323704868555, -0.03304535523056984, -0.012599034234881401, 0.02397691272199154, -0.03213099390268326, 0.041260454803705215, 0.013611146248877048, -0.005490227602422237, 0.007146832067519426, -0.029762985184788704, -0.02633840963244438, -0.01153630018234253, 0.0020868137944489717, 0.03648047894239426, 0.015932653099298477, -0.014576802961528301, 0.004997537471354008, 0.07977434247732162, 0.005367714911699295, 0.003582647768780589, 0.02029627561569214, 0.03156120702624321, -0.014073049649596214, 0.017962509766221046, 0.01718480885028839, 0.020877761766314507, -0.00296118319965899, 0.015912380069494247, 0.030279437080025673, 0.010053711012005806, -0.04042394086718559, 0.0029661317821592093, 0.018270740285515785, -0.023899251595139503, -0.0020163145381957293, 0.014246457256376743, 0.08171312510967255, -0.024016479030251503, 0.035824380815029144, 0.01416025124490261, -0.029592037200927734, -0.0343957394361496, 0.010669996961951256, 0.017416201531887054, -0.013824925757944584, -8.094064469332807e-06, 0.033150941133499146, 0.010480069555342197, 0.017109500244259834, 0.04021843522787094, -0.040237750858068466, 0.006945888511836529, 0.014776173047721386, 0.0010200743563473225, 0.030964238569140434, 0.018442662432789803, -0.029677188023924828, -0.03592982143163681, 0.05802859738469124, -0.02208634279668331, -0.10047074407339096, -0.026471488177776337, 0.022441916167736053, 0.03200717270374298, -0.056260425597429276, 0.008219527080655098, 0.03513290733098984, -0.0234869122505188, -0.0028844138141721487, -0.008777512237429619, -0.006543177645653486, -0.009075701236724854, -0.007302251644432545, 0.006325483787804842, 0.018063222989439964, -0.01601123996078968, 0.00841659028083086, 0.007636065594851971, 0.012661461718380451, -0.03583317995071411, 0.038512345403432846, 0.0020205960609018803, -0.011156678199768066, -0.012958467938005924, 0.003629039041697979, 0.0061180079355835915, 0.018479719758033752, 0.06327767670154572, -0.011008953675627708, -0.08156764507293701, 0.05209294334053993, 0.010179545730352402, -0.011507618241012096, -0.009985296986997128, -0.020909469574689865, -0.029768206179142, -0.002610532334074378, -0.042902011424303055, -0.008774121291935444, 0.009782307781279087, 0.04757608845829964, -0.05025259777903557, -0.0015347420703619719, 0.0111960768699646, 0.024338344112038612, -0.045791223645210266, 0.014050672762095928, 0.0005442916881293058, 0.008382999338209629, 0.02795209176838398, 0.011538623832166195, 0.02168074995279312, -0.002862460445612669, 0.03513277694582939, -0.09084823727607727, -0.004465661011636257, 0.06569744646549225, 0.016685152426362038, 0.002258989028632641, -0.008964446373283863, 0.01417151652276516, 0.030804112553596497, -0.022305389866232872, -0.0427301824092865, 0.014943506568670273, 0.012553179636597633, -0.026465343311429024, 0.0032348462846130133, -0.02726503275334835, -0.0014749933034181595, -0.0008444254635833204, -0.006169243715703487, -0.03347308933734894, 0.02900511957705021, 0.013128572143614292, -0.004155170172452927, -0.033774036914110184, 0.04135672375559807, -0.008437607437372208, -0.007086350116878748, 0.0009209215058945119, -0.010387350805103779, -0.026044871658086777, 0.008337166160345078, 0.01936720684170723, -0.037930041551589966, 0.015038784593343735, 0.01456491369754076, 0.006126962136477232, -0.010320240631699562, -0.006733482237905264, 0.00832312274724245, -0.0038308610673993826, 0.003033358370885253, 0.04895478114485741, -0.03339018300175667, -0.030180251225829124, -0.006990171503275633, 0.03648296371102333, -0.013950188644230366, 0.007731559220701456, -0.015151767060160637, 0.02111903205513954, -0.039457567036151886, -0.017767010256648064, -0.05291255563497543, -0.0113877197727561, 0.007964878343045712, -0.015778427943587303, -0.06147485971450806, -0.008538271300494671, -0.01584894210100174, 0.0038799301255494356, -0.01245446503162384, -0.01835627853870392, -0.015729311853647232, 0.010552236810326576, -0.02323237992823124, -0.0012521770549938083, 0.002942138584330678, -0.009661694057285786, 0.003454393008723855, -0.01568511687219143, -0.033256322145462036, -0.008085105568170547, -0.048854630440473557, 0.026612363755702972, -0.02261146530508995, -0.04481262341141701, 0.0018113182159140706, -0.013924732804298401, -0.044709835201501846, -0.0036856902297586203, 0.007672451436519623, -0.0014062520349398255, 0.02673741616308689, -0.0068328892812132835, -0.05732661113142967, -0.0102640800178051, 0.022425977513194084, 0.013883111998438835, 0.037370916455984116, 0.046121448278427124, 0.01929589919745922, -0.017782259732484818, -0.011654075235128403, 0.03352544829249382, 0.014962702058255672, 0.014335928484797478, 0.029365910217165947, -0.009793205186724663, -0.011908420361578465, 0.018492497503757477, 0.0068880775943398476, -0.005650999955832958, 0.019921626895666122, -0.001583580276928842, 0.06020527705550194, -0.027562828734517097, -0.0058937082067132, 0.03438764065504074, -0.002318018116056919, -0.005403704941272736, 0.0231021661311388, 0.03375973552465439, 0.002533142687752843, 0.004472375847399235, -0.02318430319428444, -0.031852055341005325, -0.013753457926213741, 0.021135874092578888, -0.0005739254993386567, -0.013418699614703655, -0.00984237715601921, 0.00548303360119462, 0.05239381641149521, 0.2164638787508011, 0.016468867659568787, -0.026577481999993324, -0.01708299107849598, 0.004780443385243416, 0.011005150154232979, -0.009431126527488232, -0.010364079847931862, 0.0013563252286985517, -0.005258910823613405, -0.015049629844725132, -0.09048061072826385, 0.026161443442106247, -0.007799750659614801, -0.0010401273611932993, 0.014448686502873898, -0.012277732603251934, -0.008596401661634445, -0.0014084333088248968, -0.0005153636448085308, -0.0008317864267155528, -0.012957234866917133, -0.01797523908317089, 0.01974075473845005, 0.02876526303589344, -0.0016051469137892127, 0.003168916329741478, 0.013691993430256844, -0.050808586180210114, -0.004108887165784836, 0.039933688938617706, 0.007829139940440655, 0.008563998155295849, -0.020381156355142593, -0.017212551087141037, 0.011904338374733925, 0.024187617003917694]], "endpoint_self_jitter": 0.0} \ No newline at end of file diff --git a/benchmarks/phase188_live_gate/task05_battery_index_off.txt b/benchmarks/phase188_live_gate/task05_battery_index_off.txt new file mode 100644 index 0000000..3bb398a --- /dev/null +++ b/benchmarks/phase188_live_gate/task05_battery_index_off.txt @@ -0,0 +1,65 @@ +eval: threshold=0.62 vector_candidates=100 lexical_candidates=30 rrf_k=60 recency_boost=0.0007 recency_half_life_days=365 name_hit_bonus=0.005 + +question: 'How do I deploy gitea?' + gate: best_cosine=0.8528 fts_hits=38 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml cosine=0.7640 fts=1 effective=0.03056 created=2026-10-17 (playbook) + 2. deploy/Deployments/reeseapps/gateway/README.md cosine=0.7736 fts=1 effective=0.02902 created=2026-10-17 (Proxy Server) + 3. homelab/active/container_gitea/gitea.md cosine=0.7612 fts=1 effective=0.02561 created=2026-10-17 (Gitea) + 4. homelab/active/kubernetes_gitea/gitea-values.yaml cosine=0.7232 fts=1 effective=0.02512 created=2026-10-17 (gitea-values) + 5. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 cosine=0.7159 fts=1 effective=0.02495 created=2026-10-17 (gitea-postgres.env) + suggested: 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml 2. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 3. deploy/Deployments/reeseapps/gateway/README.md 4. deploy/Deployments/reeseapps/gitea/README.md 5. deploy/Deployments/reeseapps/gitea/app/gitea-web.env.j2 + +question: 'How do I deploy forgejo?' + gate: best_cosine=0.7876 fts_hits=37 -> HIGH (answer) + 1. homelab/templates/container/foobar.md cosine=0.7876 fts=0 effective=0.01709 created=2026-10-17 (Podman foobar) + 2. deploy/Deployments/reeseapps/forgejo/README.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Reeseapps Forgejo) + 3. deploy/Deployments/reeseapps/README.md cosine=0.7857 fts=0 effective=0.01683 created=2026-10-17 (Deployments) + 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (forgejo-app) + 5. homelab/active/container_bifrost/bifrost.md cosine=0.7672 fts=0 effective=0.01657 created=2026-10-17 (Podman bifrost) + suggested: 1. deploy/Deployments/reeseapps/forgejo/README.md 2. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container 3. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.env.j2 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.container 5. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.env.j2 + +question: 'How do I set up the gateway?' + gate: best_cosine=0.7816 fts_hits=37 -> HIGH (answer) + 1. homelab/active/device_shelly/shelly.md cosine=0.6940 fts=1 effective=0.02509 created=2026-10-17 (Shelly Devices) + 2. deploy/Deployments/reeseapps/forgejo/app/forgejo.network cosine=0.6727 fts=1 effective=0.02050 created=2026-10-17 (forgejo) + 3. homelab/active/device_framework_16/framework_16.md cosine=0.7816 fts=0 effective=0.01709 created=2026-10-17 (Framework Laptop 16) + 4. deploy/Deployments/delta/istio/gateway-metallb.yaml cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (gateway-metallb) + 5. homelab/active/device_home_assistant/home_assistant.md cosine=0.7752 fts=0 effective=0.01683 created=2026-10-17 (Home Assistant) + suggested: 1. homelab/active/device_shelly/shelly.md 2. deploy/Deployments/delta/istio/gateway-metallb.yaml 3. deploy/Deployments/delta/istio/test-gateway.yaml 4. deploy/Deployments/reeseapps/gateway/README.md 5. deploy/Deployments/reeseapps/gateway/certbot/playbook.yaml + +question: 'Do you have notes on the qwen 3.8 27b model?' + gate: best_cosine=0.7483 fts_hits=31 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container cosine=0.7402 fts=1 effective=0.03322 created=2026-10-17 (qwen3.8-27b-epic-vulkan) + 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container cosine=0.7142 fts=1 effective=0.03058 created=2026-10-17 (qwen3.8-27b-juggernaut-vulkan) + 3. homelab/active/container_localai/localai.md cosine=0.6837 fts=1 effective=0.02570 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/software_ai_stack/ai_stack.md cosine=0.7029 fts=1 effective=0.02545 created=2026-10-17 (Self Hosted AI Stack) + 5. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container cosine=0.6675 fts=1 effective=0.02491 created=2026-10-17 (qwen3.8-flash-next) + suggested: 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container 3. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container 4. homelab/active/container_localai/localai.md 5. homelab/active/software_ai_stack/ai_stack.md + +question: 'What is the capital of Mongolia?' + gate: best_cosine=0.6070 fts_hits=5 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/README.md cosine=0.5588 fts=1 effective=0.02382 created=2026-10-17 (Deployments) + 2. homelab/active/container_nginx/nginx.md cosine=0.6070 fts=0 effective=0.01709 created=2026-10-17 (Ngnix) + 3. homelab/active/container_localai/localai.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/os_arch/arch-workstation.md cosine=0.6059 fts=0 effective=0.01683 created=2026-10-17 (Workstation) + 5. homelab/active/software_k3s/k3s.md cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (K3S) + suggested: 1. deploy/Deployments/reeseapps/README.md 2. homelab/active/container_nginx/nginx.md 3. homelab/active/container_localai/localai.md 4. homelab/active/os_arch/arch-workstation.md 5. homelab/active/software_k3s/k3s.md + +question: 'How do I build a Besta?' + gate: best_cosine=0.7078 fts_hits=33 -> HIGH (answer) + 1. homelab/active/container_bifrost/bifrost.md cosine=0.7078 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 2. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 3. vibe-bot/README.md cosine=0.6962 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 4. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + 5. vibe-bot/build.sh cosine=0.0000 fts=1 effective=0.01657 created=2026-10-17 (build) + suggested: 1. homelab/active/software_distoolbox/arch-build.sh 2. homepage/build.sh 3. vibe-bot/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md + +question: 'How do I build the BESTÅ?' + gate: best_cosine=0.6969 fts_hits=33 -> HIGH (answer) + 1. vibe-bot/build.sh cosine=0.6267 fts=1 effective=0.02519 created=2026-10-17 (build) + 2. homelab/active/container_bifrost/bifrost.md cosine=0.6969 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 3. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 4. vibe-bot/README.md cosine=0.6887 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 5. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + suggested: 1. vibe-bot/build.sh 2. homelab/active/software_distoolbox/arch-build.sh 3. homepage/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md +EXIT=0 diff --git a/benchmarks/phase188_live_gate/task05_battery_index_on.txt b/benchmarks/phase188_live_gate/task05_battery_index_on.txt new file mode 100644 index 0000000..3bb398a --- /dev/null +++ b/benchmarks/phase188_live_gate/task05_battery_index_on.txt @@ -0,0 +1,65 @@ +eval: threshold=0.62 vector_candidates=100 lexical_candidates=30 rrf_k=60 recency_boost=0.0007 recency_half_life_days=365 name_hit_bonus=0.005 + +question: 'How do I deploy gitea?' + gate: best_cosine=0.8528 fts_hits=38 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml cosine=0.7640 fts=1 effective=0.03056 created=2026-10-17 (playbook) + 2. deploy/Deployments/reeseapps/gateway/README.md cosine=0.7736 fts=1 effective=0.02902 created=2026-10-17 (Proxy Server) + 3. homelab/active/container_gitea/gitea.md cosine=0.7612 fts=1 effective=0.02561 created=2026-10-17 (Gitea) + 4. homelab/active/kubernetes_gitea/gitea-values.yaml cosine=0.7232 fts=1 effective=0.02512 created=2026-10-17 (gitea-values) + 5. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 cosine=0.7159 fts=1 effective=0.02495 created=2026-10-17 (gitea-postgres.env) + suggested: 1. deploy/Deployments/reeseapps/gitea/app/playbook.yaml 2. deploy/Deployments/reeseapps/gitea/app/gitea-postgres.env.j2 3. deploy/Deployments/reeseapps/gateway/README.md 4. deploy/Deployments/reeseapps/gitea/README.md 5. deploy/Deployments/reeseapps/gitea/app/gitea-web.env.j2 + +question: 'How do I deploy forgejo?' + gate: best_cosine=0.7876 fts_hits=37 -> HIGH (answer) + 1. homelab/templates/container/foobar.md cosine=0.7876 fts=0 effective=0.01709 created=2026-10-17 (Podman foobar) + 2. deploy/Deployments/reeseapps/forgejo/README.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Reeseapps Forgejo) + 3. deploy/Deployments/reeseapps/README.md cosine=0.7857 fts=0 effective=0.01683 created=2026-10-17 (Deployments) + 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (forgejo-app) + 5. homelab/active/container_bifrost/bifrost.md cosine=0.7672 fts=0 effective=0.01657 created=2026-10-17 (Podman bifrost) + suggested: 1. deploy/Deployments/reeseapps/forgejo/README.md 2. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.container 3. deploy/Deployments/reeseapps/forgejo/app/forgejo-app.env.j2 4. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.container 5. deploy/Deployments/reeseapps/forgejo/app/forgejo-postgres.env.j2 + +question: 'How do I set up the gateway?' + gate: best_cosine=0.7816 fts_hits=37 -> HIGH (answer) + 1. homelab/active/device_shelly/shelly.md cosine=0.6940 fts=1 effective=0.02509 created=2026-10-17 (Shelly Devices) + 2. deploy/Deployments/reeseapps/forgejo/app/forgejo.network cosine=0.6727 fts=1 effective=0.02050 created=2026-10-17 (forgejo) + 3. homelab/active/device_framework_16/framework_16.md cosine=0.7816 fts=0 effective=0.01709 created=2026-10-17 (Framework Laptop 16) + 4. deploy/Deployments/delta/istio/gateway-metallb.yaml cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (gateway-metallb) + 5. homelab/active/device_home_assistant/home_assistant.md cosine=0.7752 fts=0 effective=0.01683 created=2026-10-17 (Home Assistant) + suggested: 1. homelab/active/device_shelly/shelly.md 2. deploy/Deployments/delta/istio/gateway-metallb.yaml 3. deploy/Deployments/delta/istio/test-gateway.yaml 4. deploy/Deployments/reeseapps/gateway/README.md 5. deploy/Deployments/reeseapps/gateway/certbot/playbook.yaml + +question: 'Do you have notes on the qwen 3.8 27b model?' + gate: best_cosine=0.7483 fts_hits=31 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container cosine=0.7402 fts=1 effective=0.03322 created=2026-10-17 (qwen3.8-27b-epic-vulkan) + 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container cosine=0.7142 fts=1 effective=0.03058 created=2026-10-17 (qwen3.8-27b-juggernaut-vulkan) + 3. homelab/active/container_localai/localai.md cosine=0.6837 fts=1 effective=0.02570 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/software_ai_stack/ai_stack.md cosine=0.7029 fts=1 effective=0.02545 created=2026-10-17 (Self Hosted AI Stack) + 5. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container cosine=0.6675 fts=1 effective=0.02491 created=2026-10-17 (qwen3.8-flash-next) + suggested: 1. deploy/Deployments/reeseapps/ai/deployments/epic/quadlets/qwen3.8-27b-epic-vulkan.container 2. deploy/Deployments/reeseapps/ai/deployments/juggernaut/quadlets/qwen3.8-27b-juggernaut-vulkan.container 3. deploy/Deployments/reeseapps/ai/deployments/turbo/quadlets/qwen3.8-flash-next.container 4. homelab/active/container_localai/localai.md 5. homelab/active/software_ai_stack/ai_stack.md + +question: 'What is the capital of Mongolia?' + gate: best_cosine=0.6070 fts_hits=5 -> HIGH (answer) + 1. deploy/Deployments/reeseapps/README.md cosine=0.5588 fts=1 effective=0.02382 created=2026-10-17 (Deployments) + 2. homelab/active/container_nginx/nginx.md cosine=0.6070 fts=0 effective=0.01709 created=2026-10-17 (Ngnix) + 3. homelab/active/container_localai/localai.md cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (Local AI with Anything LLM) + 4. homelab/active/os_arch/arch-workstation.md cosine=0.6059 fts=0 effective=0.01683 created=2026-10-17 (Workstation) + 5. homelab/active/software_k3s/k3s.md cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (K3S) + suggested: 1. deploy/Deployments/reeseapps/README.md 2. homelab/active/container_nginx/nginx.md 3. homelab/active/container_localai/localai.md 4. homelab/active/os_arch/arch-workstation.md 5. homelab/active/software_k3s/k3s.md + +question: 'How do I build a Besta?' + gate: best_cosine=0.7078 fts_hits=33 -> HIGH (answer) + 1. homelab/active/container_bifrost/bifrost.md cosine=0.7078 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 2. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 3. vibe-bot/README.md cosine=0.6962 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 4. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + 5. vibe-bot/build.sh cosine=0.0000 fts=1 effective=0.01657 created=2026-10-17 (build) + suggested: 1. homelab/active/software_distoolbox/arch-build.sh 2. homepage/build.sh 3. vibe-bot/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md + +question: 'How do I build the BESTÅ?' + gate: best_cosine=0.6969 fts_hits=33 -> HIGH (answer) + 1. vibe-bot/build.sh cosine=0.6267 fts=1 effective=0.02519 created=2026-10-17 (build) + 2. homelab/active/container_bifrost/bifrost.md cosine=0.6969 fts=0 effective=0.01709 created=2026-10-17 (Podman bifrost) + 3. homelab/active/software_distoolbox/arch-build.sh cosine=0.0000 fts=1 effective=0.01709 created=2026-10-17 (arch-build) + 4. vibe-bot/README.md cosine=0.6887 fts=0 effective=0.01683 created=2026-10-17 (Vibe Discord Bot with RAG Chat History) + 5. homepage/build.sh cosine=0.0000 fts=1 effective=0.01683 created=2026-10-17 (build) + suggested: 1. vibe-bot/build.sh 2. homelab/active/software_distoolbox/arch-build.sh 3. homepage/build.sh 4. homelab/active/container_bifrost/bifrost.md 5. vibe-bot/README.md +EXIT=0 diff --git a/tests/conftest.py b/tests/conftest.py index 5156356..136639f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,6 +109,15 @@ os.environ["BOR_ANSWER_TEXT"] = str(_Settings.model_fields["answer_text"].defaul os.environ["BOR_SHOW_TEXT_MAX_CHARS"] = str( _Settings.model_fields["show_text_max_chars"].default ) + +# Phase 188: the same leak class for the HNSW ``ef_search`` knob — an +# operator's shell may legitimately carry ``BOR_RETRIEVAL_EF_SEARCH`` +# raised after the recall battery, and the default / clamp / statement-text +# pins (unit + integration) must see the code default (100 = the A7 +# candidate count, so a default run never clamps). +os.environ["BOR_RETRIEVAL_EF_SEARCH"] = str( + _Settings.model_fields["retrieval_ef_search"].default +) os.environ["BOR_SHOW_TEXT_MAX_BLOCKS"] = str( _Settings.model_fields["show_text_max_blocks"].default ) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 9648dbe..2a2eaa7 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -156,6 +156,14 @@ def app_server(mock_llm: int) -> Iterator[str]: env["BOR_SHOW_TEXT_MAX_BLOCKS"] = str( _Settings.model_fields["show_text_max_blocks"].default ) + # Phase 188: the same leak guard for the HNSW ``ef_search`` knob — an + # operator's shell may legitimately carry ``BOR_RETRIEVAL_EF_SEARCH`` + # raised after the recall battery, and the story suites' retrieval pins + # must see the CODE default (100 = the A7 candidate count, so a default + # run never clamps). + env["BOR_RETRIEVAL_EF_SEARCH"] = str( + _Settings.model_fields["retrieval_ef_search"].default + ) env.setdefault("BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese") # Phase 16: admin auth must be set or create_app() refuses to boot. env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD diff --git a/tests/e2e/test_vector_index_recall.py b/tests/e2e/test_vector_index_recall.py new file mode 100644 index 0000000..cb843c8 --- /dev/null +++ b/tests/e2e/test_vector_index_recall.py @@ -0,0 +1,889 @@ +"""Phase 188 (task 05) E2E (Playwright, mock-only): the ``chunks.embedding`` +HNSW index reaches the owner's chat, changes nothing the owner sees, and the +one retrieval knob the phase adds is honoured on a real turn. + +Source: owner incident 2026-10-15. ``pg_indexes`` for ``chunks`` listed the +PK, ``ix_chunks_document_id`` and ``ix_chunks_tsv`` and NOTHING on +``embedding``, so every turn's A7 cosine tier (``ORDER BY embedding <=> :vec +LIMIT 100``) full-scanned a ~3 GB, 414,220-row relation on the interactive +path. Migration ``0029`` adds ``ix_chunks_embedding_hnsw`` (HNSW, +``vector_cosine_ops``, ``m = 16``, ``ef_construction = 64``) and task 02 adds +``BOR_RETRIEVAL_EF_SEARCH`` — shipped at **400** on task 04's measured +decision — applied as ``SET LOCAL hnsw.ef_search`` inside the turn's own +transaction. + +**What this module owns, and what it does not.** The plan-level proof (the +tier's captured statement really names the index, and a turn's ``SET LOCAL`` +reaches the connection) lives in +``tests/integration/test_chunks_hnsw_recall.py`` (tasks 03/04) and the DDL +round trip in ``tests/integration/test_chunks_hnsw_index.py`` (task 01). +THIS file is the last mile: one question, in a real browser, over a real +import, with the index DROPPED and RESTORED around it — an A/B the owner +could run on their own box. The corpus is 11 small documents, so Postgres' +cost model may well serve BOTH arms from a sequential scan; that is stated, +not hidden. An index at this size is not a speed claim, it is a +**non-regression** claim, and the only non-regression that matters is the +answer. So what is compared between the arms is what reached the model: the +durable retrieval record (``query_log.sources`` / ``top_score`` / +``chunk_hits`` / ``fts_hits`` — LOCKED A3's witness), the ``done`` frame's +tiers, and the rendered "Nearby docs, in case" row. The DDL state is read +back from the catalog INSIDE each arm, so "OFF" can never quietly mean "ON". + +Both sentinels are calibrated against the deterministic mock embedder (the +``tests/e2e/test_retrieval_quality.py`` convention, not invented) and the +margins are quoted in the units the honesty gate reads: + +* ANSWERED — ``"How do I raise the tilecache eviction watermark so the hot + zoom levels stay pinned?"`` → best cosine **0.6187**, lexical (FTS) hits + **10**, leading document ``hnswkb/infra/tilecache.md``. ``0.6187 >= 0.30`` + is A8's HIGH quadrant by a margin of 0.32, so neither arm can flip on a + rounding difference. +* DEFLECTED — ``"Which bow should I draw for a baroque viola sonata?"`` (no + note in this KB can answer it) → best cosine **0.1706**, lexical hits **0**. + A8 deflects only on weak cosine AND zero lexical hits, and both halves hold + with room: 0.13 BELOW the 0.30 threshold, and a lexical side that is empty + rather than merely low. + +Run in isolation (DB up: ``podman compose up -d db``, schema at head so +``0029`` has run — the module fails loudly, never vacuously, if it has not): + + uv run pytest tests/e2e/test_vector_index_recall.py -v --no-cov + +MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the cosine and lexical +numbers above are properties of the deterministic mock, and the parity pins +compare them exactly between the two arms. + +Isolation (the house shape): ONE module app on its own port (the conftest +session app owns 8123 in a combined run), the mock LLM on the conftest port, +a dedicated ``BOR_SOURCES_DIR`` / ``BOR_PDF_PAGE_DIR`` so nothing writes into +the operator's, the shared E2E Postgres truncated BEFORE **and** AFTER every +test, and the KB re-imported through the REAL importer for every test — so +the HNSW insert path runs in every arm, including the ones that then drop the +index. The admin enters through the real form login. + +Test → observable mapping (Playwright Mapping Rule — the rendered DOM is the +observable, the durable row is the witness, and the two are read as one +comparison): + +1. ``test_answered_question_cites_the_same_document_indexed_or_not`` — the answered + sentinel, twice: with ``ix_chunks_embedding_hnsw`` present, then after the + real ``0029`` downgrade drops it (restored in ``finally``, and again by the + module guard). Both arms ground (no ``.is-deflected``, the mock marker + renders), both cite the SAME document — ``hnswkb/infra/tilecache.md`` first + in ``query_log.sources`` and first in the rendered row — and agree on + ``top_score`` to within ``1e-9``, on ``chunk_hits`` and ``fts_hits``, and + on the byte-identical answer. The DDL really moved; the answer did not. +2. ``test_index_exists_and_is_valid_after_import`` — the import that just ran + left the index present, VALID in ``pg_index``, HNSW over + ``vector_cosine_ops`` per ``pg_get_indexdef``, and carrying pages. An + INVALID index is silent (the failure ``0029`` exists to catch), so the + post-import state is pinned instead of assumed. +3. ``test_near_miss_question_still_deflects`` — the out-of-KB sentinel in + BOTH DDL arms: the amber deflection, the "Maybe try" chips, the ``done`` + frame citing nothing, and the row's two gate inputs read in the units A8 + defines. An index must never manufacture a high ``top_score``. +4. ``test_ef_search_setting_is_honoured`` — two extra apps on their own ports + with stdout captured (the surface the operator greps): ``BOR_RETRIEVAL_EF_SEARCH=150`` + answers the sentinel normally and logs NO clamp line; ``=50`` answers it + just as well — same leading doc, same ``top_score`` — and logs the + operator-visible ``retrieval: ef_search=50 raised to 100 (A7 candidate + count)`` WARNING. The clamp keeps its promise in the browser, not only in + a unit test. +""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +import re +import subprocess +import sys +import time +from collections.abc import Iterator +from pathlib import Path +from threading import Thread +from typing import Any + +import pytest +from alembic.config import Config as AlembicConfig +from playwright.sync_api import Page, expect +from sqlalchemy import func, select, text + +from alembic import command +from app.config import Settings as _Settings +from app.db import SessionLocal +from app.models import QueryLog +from app.rag.importer import ImportSummary, import_sources +from app.rag.llm import LLMClient +from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + MOCK_PORT, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) + +REPO = Path(__file__).resolve().parents[2] + +#: The tracked fixture KB — 11 markdown notes over 6 folders. Tracked rather +#: than generated: the cosine/lexical calibration in the module docstring +#: belongs to THESE bytes, so a re-run ranks the same way on any day. +FIXTURES = REPO / "tests" / "fixtures" / "hnswkb" + +#: The ``source`` the importer records for a walked root (``Path(path).name``). +SOURCE_NAME = "hnswkb" +N_DOCS = 11 + +#: Dedicated module ports (the house env-overridable pattern): the conftest +#: session app owns 8123, phase 187's modules own 8203/8204. +APP_PORT = int(os.environ.get("E2E_APP_PORT_VECIDX", "8205")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" +EF_OK_PORT = int(os.environ.get("E2E_APP_PORT_VECIDX_EF150", "8206")) +EF_OK_URL = f"http://127.0.0.1:{EF_OK_PORT}" +EF_CLAMP_PORT = int(os.environ.get("E2E_APP_PORT_VECIDX_EF50", "8207")) +EF_CLAMP_URL = f"http://127.0.0.1:{EF_CLAMP_PORT}" + +#: Migration 0029 and the revision it sits on — the pair the drop/restore +#: walks, moved in the migration's OWN directions. This file contains no +#: inline ``CREATE INDEX``/``DROP INDEX``: 0029 is the single definition of +#: the index these pins name. +REV_HNSW = "0029" +REV_BEFORE = "0028" + +#: The index 0029 builds. +INDEX_NAME = "ix_chunks_embedding_hnsw" + +#: The answered sentinel (measured, module docstring): cosine 0.6187, 10 +#: lexical hits, ``infra/tilecache.md`` first in the fused record. +ANSWER_QUESTION = ( + "How do I raise the tilecache eviction watermark so the hot zoom levels " + "stay pinned?" +) +#: The document the owner asked about — the one both arms must cite. +CITED_DOC = f"{SOURCE_NAME}/infra/tilecache.md" + +#: The near-miss sentinel (measured): cosine 0.1706, 0 lexical hits. +DEFLECT_QUESTION = "Which bow should I draw for a baroque viola sonata?" + +#: The mock's grounded answer marker (``tests/e2e/mock_llm.py``). +MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" + +#: The honest-deflection phrase the mock answers with (the house constant — +#: ``test_honest_deflection.py`` et al., phase 52's copy as the deterministic +#: mock speaks it). +DEFLECT_PHRASE = r"haven't done anything like that" + +#: The app's honesty gate — the values the module app boots with, named once +#: here so the pins quote the gate they are measured against. +GATE_THRESHOLD = 0.30 +GATE_LEXICAL_FLOOR = 0.15 + +#: The A7 cosine tier's candidate count (``hybrid_vector_candidates``) — the +#: floor the ``ef_search`` clamp raises to. Read from the code default instead +#: of restated, so a change to the knob moves the pin with it. +A7_CANDIDATES = int(_Settings.model_fields["hybrid_vector_candidates"].default) + +#: The suggested tier's size (``suggested_docs``) — the split between what the +#: prompt is seeded with and what renders as "Nearby docs, in case". +SUGGESTED_DOCS = int(_Settings.model_fields["suggested_docs"].default) + +#: The clamp line ``app/rag/retriever.py`` logs (AGENTS rule 10: a knob that +#: would degrade recall is never silent), matched on the app's captured +#: stdout: ``… WARNING app.rag.retriever :: retrieval: ef_search=50 raised +#: to 100 (A7 candidate count)``. +CLAMP_LINE = re.compile( + r"WARNING\s+app\.rag\.retriever\s+::\s+retrieval: ef_search=(\d+) " + r"raised to (\d+) \(A7 candidate count\)" +) + +#: Numeric slack between the arms' ``top_score``. Both arms run the same exact +#: cosine tier over this corpus, so the true slack is zero; the epsilon only +#: absorbs the float's travel through the row. +SCORE_SLACK = 1e-9 + +#: Turn budget: a mock turn lands in well under a second; 30 s is the house +#: head-room with no client-side hard timeout. +SETTLE_TIMEOUT_MS = 30_000 + + +# --------------------------------------------------------------------------- +# The module app + the two ef_search apps +# --------------------------------------------------------------------------- + + +def _app_env(scratch: Path, ef_search: str | None = None) -> dict[str, str]: + """App env: the mock LLM, the mock-calibrated honesty gate (the conftest + convention — both floors stay ``<=`` the threshold or the app refuses to + boot), no git-source fallback, dedicated directories so nothing writes + into the operator's. + + ``ef_search=None`` pins the knob to the CODE default (the phase 188 leak + guard: an operator's ``.env`` value never reaches a test); any other value + is what the app boots with. + """ + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = f"http://127.0.0.1:{MOCK_PORT}/v1" + env["BOR_RELEVANCE_THRESHOLD"] = str(GATE_THRESHOLD) + env["BOR_LEXICAL_SUPPORT_FLOOR"] = str(GATE_LEXICAL_FLOOR) + env["BOR_SOURCE_USEFULNESS_FLOOR"] = str(GATE_LEXICAL_FLOOR) + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + env["BOR_GIT_SOURCES"] = "" + env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") + env["BOR_PDF_PAGE_DIR"] = str(scratch / "pdf-pages") + env["BOR_RETRIEVAL_EF_SEARCH"] = ( + ef_search + if ef_search is not None + else str(_Settings.model_fields["retrieval_ef_search"].default) + ) + return env + + +@contextlib.contextmanager +def _running_app( + url: str, port: int, env: dict[str, str], log_path: Path | None = None +) -> Iterator[tuple[str, Path | None]]: + """Boot the app on *port*, wait for ``/api/health``, tear it down. + + With *log_path* the app's stdout+stderr go to that file — the same surface + the operator greps (``grep ' :: ' server.log``), so the clamp pin reads + what a human would read rather than a test-private logger. + """ + args = [ + sys.executable, + "-m", + "uvicorn", + "app.main:app", + "--host", + "127.0.0.1", + "--port", + str(port), + "--log-level", + "warning", + ] + if log_path is None: + proc = subprocess.Popen(args, cwd=REPO, env=env) # noqa: S603 + try: + _wait_http(f"{url}/api/health") + yield url, None + finally: + proc.terminate() + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=10) + return + with open(log_path, "wb") as log: + proc = subprocess.Popen( # noqa: S603 + args, cwd=REPO, env=env, stdout=log, stderr=subprocess.STDOUT + ) + try: + _wait_http(f"{url}/api/health") + yield url, log_path + finally: + proc.terminate() + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=10) + + +@pytest.fixture(scope="module") +def app_server( + mock_llm: int, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[str]: + """The app under tests 1-3: the SHIPPED default ``retrieval_ef_search`` + (400 — task 04's measured decision) over the fixture KB.""" + if USE_REAL_LLM: + pytest.skip( + "mock-only suite — the cosine/lexical calibration and the two-arm " + "parity pins belong to the deterministic mock " + "(E2E_REAL_LLM unsupported)" + ) + assert mock_llm + scratch = tmp_path_factory.mktemp("bor_vector_index_app") + with _running_app(APP_URL, APP_PORT, _app_env(scratch)) as (url, _log): + yield url + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +# --------------------------------------------------------------------------- +# The index state, read from the catalog (never inferred) +# --------------------------------------------------------------------------- + + +def _index_state(db: Any) -> dict[str, Any] | None: + """The ``pg_index`` row for :data:`INDEX_NAME`, or ``None`` when absent. + + ``None`` is the OFF arm. ``valid=False`` is a state nothing here may treat + as ON: an INVALID index exists, is maintained on every insert, and serves + no queries — silently, which is the failure ``0029`` was written to catch. + """ + row = db.execute( + text( + "SELECT i.indisvalid, pg_table_size(c.oid) AS size_bytes, " + "pg_get_indexdef(c.oid) AS indexdef " + "FROM pg_index i " + "JOIN pg_class c ON c.oid = i.indexrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relname = :name AND n.nspname = 'public'" + ), + {"name": INDEX_NAME}, + ).first() + if row is None: + return None + return { + "valid": bool(row.indisvalid), + "size_bytes": int(row.size_bytes), + "indexdef": str(row.indexdef), + } + + +def _alembic_config() -> AlembicConfig: + cfg = AlembicConfig(str(REPO / "alembic.ini")) + cfg.set_main_option("script_location", str(REPO / "alembic")) + cfg.attributes["configure_logger"] = False + return cfg + + +def _alembic_version() -> str: + with SessionLocal() as db: + version = db.execute(text("SELECT version_num FROM alembic_version")).scalar() + db.rollback() + return str(version) + + +def _set_index_present(present: bool) -> None: + """Move the schema to the index OFF / ON state through 0029's own + ``downgrade`` / ``upgrade``, and PROVE the move landed. + + Both directions run in the migration's ``autocommit_block`` (a + ``CONCURRENTLY`` DDL cannot run inside a transaction). This process's own + session is rolled back FIRST — an open snapshot is what parks a + ``CONCURRENTLY`` build behind the test session forever, the failure task 01 + documented. + """ + with SessionLocal() as probe: + probe.rollback() + cfg = _alembic_config() + if present: + command.upgrade(cfg, REV_HNSW) + else: + command.downgrade(cfg, REV_BEFORE) + with SessionLocal() as db: + state = _index_state(db) + version = db.execute(text("SELECT version_num FROM alembic_version")).scalar() + db.rollback() + if present: + assert state is not None and state["valid"], ( + f"restoring {REV_HNSW} left no VALID {INDEX_NAME}: {state}" + ) + assert str(version) == REV_HNSW, f"alembic_version did not return: {version}" + else: + assert state is None, f"{INDEX_NAME} survived the {REV_BEFORE} downgrade: {state}" + assert str(version) == REV_BEFORE, f"alembic_version did not move: {version}" + + +@pytest.fixture(scope="module", autouse=True) +def _index_at_head() -> Iterator[None]: + """Module guard: the schema STARTS and ENDS at 0029 with a valid index. + + A suite that silently ran with the index missing would pass tests 1 and 3 + for the wrong reason, so entering is an assertion rather than a hope and + the teardown repairs whatever a failed test left behind. + """ + version = _alembic_version() + if version != REV_HNSW: + pytest.fail( + f"phase 188's task-05 suite needs the schema at {REV_HNSW} " + f"(found {version!r}) — run `uv run alembic upgrade head`" + ) + with SessionLocal() as db: + state = _index_state(db) + db.rollback() + if state is None or not state["valid"]: + _set_index_present(True) + yield + _set_index_present(True) + + +# --------------------------------------------------------------------------- +# The KB, imported through the REAL importer with the index in place +# --------------------------------------------------------------------------- + + +def _mock_llm_client() -> LLMClient: + """An embedder aimed at the mock (no ``.env``: the test process must not + inherit the operator's live endpoint).""" + kwargs: dict[str, Any] = { + "_env_file": None, + "llm_base_url": f"http://127.0.0.1:{MOCK_PORT}/v1", + } + settings = _Settings(**kwargs) # pyright: ignore[reportCallIssue] + return LLMClient(settings) + + +async def _import_kb() -> ImportSummary: + return await import_sources([FIXTURES], _mock_llm_client()) + + +def _run_in_thread(coro: Any) -> Any: + """Run a coroutine on a worker thread (Playwright owns the test loop).""" + box: dict[str, Any] = {} + + def runner() -> None: + try: + box["value"] = asyncio.run(coro) + except BaseException as e: # noqa: BLE001 — re-raised on the test thread + box["error"] = e + + thread = Thread(target=runner) + thread.start() + thread.join() + if "error" in box: + raise box["error"] + return box["value"] + + +def _truncate_all() -> None: + """Fresh KB + registry + query log (the phase-184 table list).""" + with SessionLocal() as db: + db.execute( + text( + "TRUNCATE chunks, documents, query_log, kb_overview, " + "git_sources, sources_meta, folder_summaries" + ) + ) + db.commit() # without the commit the TRUNCATE rolls back + + +@pytest.fixture(autouse=True) +def kb(mock_llm: int) -> Iterator[ImportSummary]: + """A fresh KB of the tracked fixture tree, imported with the HNSW index + PRESENT, so every arm — including the ones that then drop it — put its + embeddings into the graph. + + One fixture owns the whole boundary so the order cannot drift: truncate → + assert the index is live → import → test → truncate. The index assertion + runs AFTER the truncate because a TRUNCATE of an HNSW-indexed table is + exactly the insert path this suite is about. + """ + _truncate_all() + with SessionLocal() as db: + state = _index_state(db) + db.rollback() + assert state is not None and state["valid"], ( + f"{INDEX_NAME} must be present and valid BEFORE the import — the " + f"point of this suite is inserts into a live graph: {state}" + ) + + summary = _run_in_thread(_import_kb()) + assert summary.errors == 0, f"the fixture KB failed to import: {summary}" + assert summary.added == N_DOCS, f"expected {N_DOCS} fixture notes: {summary}" + assert summary.summaries == N_DOCS, f"every note needs a summary: {summary}" + yield summary + _truncate_all() + + +# --------------------------------------------------------------------------- +# Helpers: the wire, the DOM, the durable row +# --------------------------------------------------------------------------- + +#: Captures the raw SSE ``data:`` payloads of the /api/chat stream (a response +#: clone read in the background) — the wire-level witness, independent of the +#: rendering (the ``test_agent_document_tools.py`` hook). +SSE_HOOK = """ +() => { + if (window.__sseInstalled) return; + window.__sseInstalled = true; + window.__sseFrames = []; + const origFetch = window.fetch; + window.fetch = async function (...args) { + const res = await origFetch.apply(this, args); + try { + const url = typeof args[0] === 'string' ? args[0] : args[0].url; + if (url.includes('/api/chat')) { + res.clone().text().then((bodyText) => { + for (const block of bodyText.split('\\n\\n')) { + const line = block.trim(); + if (line.startsWith('data: ')) { + window.__sseFrames.push(line.slice(6)); + } + } + }); + } + } catch (e) { /* non-clonable response: ignored */ } + return res; + }; +} +""" + + +def _frames(page: Page, since: int) -> list[dict[str, Any]]: + """This turn's SSE frames, once the hook's background read settles.""" + deadline = time.monotonic() + 10.0 + while True: + raw = page.evaluate("() => window.__sseFrames || []") + parsed = [json.loads(line) for line in raw[since:] if line] + if any(f.get("type") == "done" for f in parsed): + return parsed + if time.monotonic() > deadline: + raise AssertionError( + f"SSE hook captured no `done` frame ({len(parsed)} frames) — " + "was the hook installed before the turn?" + ) + time.sleep(0.05) + + +def _done(frames: list[dict[str, Any]]) -> dict[str, Any]: + """The turn's ONE ``done`` frame — the shape task 03 pins.""" + dones = [f for f in frames if f.get("type") == "done"] + assert len(dones) == 1, f"expected exactly one done frame, got {dones}" + return dones[0] + + +def _new_chat(page: Page) -> None: + """Clear the conversation so the NEXT turn is a first turn (no history + from the previous arm rides the prompt). The control the owner uses.""" + page.click("#new-chat-btn") + expect(page.locator(".msg")).to_have_count(0) + + +def _submit(page: Page, question: str) -> int: + """Send *question* as the turn's ONLY message (phase 162's rule: a trailing + phrase changes what retrieval sees). Returns the frame count so far.""" + seen = int(page.evaluate("() => (window.__sseFrames || []).length")) + page.fill("#message-input", question) + page.click("#send-btn") + expect(page.locator(".msg.user .bubble").last).to_contain_text(question) + return seen + + +def _wait_settled(page: Page) -> None: + """The settle sync: the client re-labels Send on the ``done`` frame, and + the server writes the ``query_log`` row just before yielding it.""" + expect(page.locator("#send-label")).to_have_text("Send", timeout=SETTLE_TIMEOUT_MS) + + +def _row(question: str) -> dict[str, Any]: + """The durable record for *question* as a plain snapshot — exactly one row + (the caller clears the log between arms that ask the same question). + + Read as values, not as an ORM instance: the row outlives its session here + (the arm compares it against another arm's), and a detached instance + raises the moment an attribute is touched. + """ + with SessionLocal() as db: + rows = db.execute( + select( + QueryLog.top_score, + QueryLog.fts_hits, + QueryLog.chunk_hits, + QueryLog.deflected, + QueryLog.sources, + ).where(QueryLog.question == question) + ).all() + assert len(rows) == 1, f"expected one query_log row for {question!r}, got {len(rows)}" + row = rows[0] + return { + "top_score": float(row.top_score), + "fts_hits": row.fts_hits, + "chunk_hits": int(row.chunk_hits), + "deflected": bool(row.deflected), + "sources": str(row.sources), + } + + +def _recorded(row: dict[str, Any]) -> list[str]: + """``query_log.sources`` as the ordered ``source/path`` list the server + wrote: the retrieval tier that reached the model (LOCKED A3).""" + return [part.strip() for part in row["sources"].split(",") if part.strip()] + + +def _arm(page: Page, question: str, *, expect_index: bool) -> dict[str, Any]: + """Ask *question* once and collect everything that must not move between + the arms — the DOM, the wire, the durable row, and the DDL state the arm + ACTUALLY ran under.""" + with SessionLocal() as db: + state = _index_state(db) + db.rollback() + if expect_index: + assert state is not None and state["valid"], ( + f"the ON arm ran WITHOUT a valid {INDEX_NAME}: {state}" + ) + else: + assert state is None, f"the OFF arm still had {INDEX_NAME}: {state}" + + since = _submit(page, question) + _wait_settled(page) + done = _done(_frames(page, since)) + + brain = page.locator(".msg.brain").last + row = _row(question) + recorded = _recorded(row) + return { + "answer": brain.locator(".bubble").first.inner_text(), + "nearby": brain.locator(".related-docs .related-doc").all_inner_texts(), + "chips": brain.locator(".source-chip").count(), + "deflected_bubbles": page.locator(".msg.brain.is-deflected").count(), + "done": done, + "index_state": state, + "row_deflected": row["deflected"], + "top_score": float(row["top_score"]), + "fts_hits": row["fts_hits"], + "chunk_hits": row["chunk_hits"], + "recorded": recorded, + "suggested": recorded[:SUGGESTED_DOCS], + "nearby_recorded": recorded[SUGGESTED_DOCS:], + } + + +# --------------------------------------------------------------------------- +# 1 — the answered sentinel: the SAME document, index ON and index OFF +# --------------------------------------------------------------------------- + + +def test_answered_question_cites_the_same_document_indexed_or_not( + page: Page, app_url: str +) -> None: + """The A7/A8 contract in the UI: an index is an ACCESS PATH, so the same + question must reach the same document whether or not the graph exists.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + login(page, app_url, next="/") # phase 79: the chat is login-gated + page.evaluate(SSE_HOOK) + + # --- ON: the shipped state (0029 applied, index valid) ------------------ + on = _arm(page, ANSWER_QUESTION, expect_index=True) + assert on["row_deflected"] is False + assert on["deflected_bubbles"] == 0 + assert MOCK_ANSWER_MARKER in on["answer"] + # The measured margin, not just the direction. + assert on["top_score"] >= GATE_THRESHOLD + assert on["recorded"][0] == CITED_DOC, on["recorded"] + assert on["chunk_hits"] > 0 + # Phase 119 (LOCKED A1): a zero-read grounded turn chips nothing — the + # index changes nothing about that either. + assert on["chips"] == 0 + # The tier that reached the model, and the tier the page rendered. + assert CITED_DOC in on["suggested"] + assert on["nearby"], "the fixture KB must overflow into the rendered row" + assert on["nearby"] == on["nearby_recorded"] + assert on["done"]["sources"] == [] + assert [f"{s['source']}/{s['path']}" for s in on["done"]["related"]] == ( + on["nearby_recorded"] + ) + + # --- OFF: the real 0029 downgrade, restored in `finally` ---------------- + with SessionLocal() as db: + db.execute(text("TRUNCATE query_log")) + db.commit() + _new_chat(page) + _set_index_present(False) + try: + off = _arm(page, ANSWER_QUESTION, expect_index=False) + finally: + _set_index_present(True) + + # Same verdict, same document, same numbers: the DDL moved, the answer + # did not. + assert off["row_deflected"] is False + assert off["deflected_bubbles"] == 0 + assert MOCK_ANSWER_MARKER in off["answer"] + assert off["recorded"] == on["recorded"], ( + f"retrieval order moved with the index:\n ON={on['recorded']}\n" + f" OFF={off['recorded']}" + ) + assert off["recorded"][0] == CITED_DOC + assert off["suggested"] == on["suggested"] + assert off["nearby"] == on["nearby"] + assert abs(off["top_score"] - on["top_score"]) <= SCORE_SLACK, ( + f"top_score moved with the index: {on['top_score']} vs " + f"{off['top_score']}" + ) + assert off["chunk_hits"] == on["chunk_hits"] + assert off["fts_hits"] == on["fts_hits"] + assert off["answer"] == on["answer"] + for key in ("deflected", "sources", "related", "suggestions"): + assert off["done"].get(key) == on["done"].get(key), ( + f"the done frame's {key} moved with the index: " + f"{on['done'].get(key)!r} vs {off['done'].get(key)!r}" + ) + + +# --------------------------------------------------------------------------- +# 2 — the index exists and is VALID after a real import +# --------------------------------------------------------------------------- + + +def test_index_exists_and_is_valid_after_import(kb: ImportSummary) -> None: + """The import that just ran left a USABLE graph, not a silent INVALID one. + + ``0029`` post-validates its own build because a ``CONCURRENTLY`` build that + dies leaves the index INVALID — costing insert time on every sync and + serving nothing, with no error anywhere. The same check belongs after a + real import: chunks go into the graph on every sync, and an index that + drifted INVALID is invisible to every other surface the app has. + """ + with SessionLocal() as db: + state = _index_state(db) + counts = db.execute( + text("SELECT count(*) AS n, count(embedding) AS with_vec FROM chunks") + ).one() + stored = db.scalar(select(func.count()).select_from(QueryLog)) + db.rollback() + assert stored == 0, "this pin reads the catalog only — no turn has run" + + assert state is not None, f"{INDEX_NAME} is MISSING after the import" + assert state["valid"] is True, ( + f"{INDEX_NAME} exists but is INVALID after the import — every sync now " + f"pays for a graph that answers nothing: {state['indexdef']}" + ) + definition = state["indexdef"] + assert INDEX_NAME in definition, definition + assert "hnsw" in definition, definition + assert "vector_cosine_ops" in definition, definition + assert "embedding" in definition, definition + + # Not an empty shell over an empty table: the import put real embeddings + # in, and the graph carries pages for them. + assert int(counts.n) > 0, "the fixture import produced no chunks" + assert int(counts.with_vec) == int(counts.n), ( + f"chunks without an embedding would be invisible to the tier: {counts}" + ) + # Content chunks + one summary chunk per document (phase 30/118) — the + # importer's own two counts, so a row that never reached the graph shows + # up here instead of as an invisible hole. + assert int(counts.n) == kb.chunks + kb.summaries, ( + f"the graph holds {counts.n} rows, the import reported " + f"{kb.chunks} content + {kb.summaries} summary chunks" + ) + assert state["size_bytes"] > 0, f"{INDEX_NAME} carries no pages: {state}" + + +# --------------------------------------------------------------------------- +# 3 — the near-miss question still deflects (both arms) +# --------------------------------------------------------------------------- + + +def test_near_miss_question_still_deflects(page: Page, app_url: str) -> None: + """A8 in the browser, index ON and OFF: weak cosine AND zero lexical hits + still deflects, and an index never manufactures a high ``top_score``.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + login(page, app_url, next="/") + page.evaluate(SSE_HOOK) + + on = _arm(page, DEFLECT_QUESTION, expect_index=True) + assert on["row_deflected"] is True + assert on["deflected_bubbles"] == 1 + # The gate's two inputs, in the units A8 defines them. + assert 0.0 < on["top_score"] < GATE_THRESHOLD + assert on["fts_hits"] == 0, "deflection is only reached with zero hits" + # The owner-visible shape of the deflection (phase 52/118). + expect(page.locator(".msg.brain.is-deflected .bubble")).to_contain_text( + DEFLECT_PHRASE, timeout=SETTLE_TIMEOUT_MS + ) + chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") + expect(chips.first).to_be_visible(timeout=SETTLE_TIMEOUT_MS) + assert chips.count() >= 2, "a deflection must offer 2-3 alternatives" + assert all(text.strip() for text in chips.all_inner_texts()) + assert on["done"]["deflected"] is True + assert on["done"]["sources"] == [], "a deflected answer cites nothing" + + # OFF arm: the same verdict, for the same measured reason. + with SessionLocal() as db: + db.execute(text("TRUNCATE query_log")) + db.commit() + _new_chat(page) + _set_index_present(False) + try: + off = _arm(page, DEFLECT_QUESTION, expect_index=False) + finally: + _set_index_present(True) + + assert off["row_deflected"] is True + assert off["deflected_bubbles"] == 1 + assert 0.0 < off["top_score"] < GATE_THRESHOLD + assert off["fts_hits"] == 0 + assert abs(off["top_score"] - on["top_score"]) <= SCORE_SLACK + assert off["recorded"] == on["recorded"] + assert MOCK_ANSWER_MARKER not in off["answer"] + assert off["done"]["deflected"] is True + + +# --------------------------------------------------------------------------- +# 4 — the ef_search knob on a real turn, and the clamp it cannot bypass +# --------------------------------------------------------------------------- + + +def _log_lines(log_path: Path, pattern: re.Pattern[str]) -> list[str]: + """Every clamp line the app logged — read off the operator's surface.""" + return [ + line + for line in log_path.read_text(encoding="utf-8", errors="replace").splitlines() + if pattern.search(line) + ] + + +def test_ef_search_setting_is_honoured( + page: Page, tmp_path_factory: pytest.TempPathFactory +) -> None: + """``BOR_RETRIEVAL_EF_SEARCH`` reaches the turn, and a value BELOW the A7 + candidate count is neither silent nor harmful.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + + # --- 150 (>= the 100-candidate tier): a normal turn, no clamp line ----- + ok_dir = tmp_path_factory.mktemp("bor_ef_ok") + with _running_app( + EF_OK_URL, EF_OK_PORT, _app_env(ok_dir, "150"), ok_dir / "ef150.log" + ) as (ok_url, ok_log): + assert ok_log is not None + login(page, ok_url, next="/") + page.evaluate(SSE_HOOK) + on_ok = _arm(page, ANSWER_QUESTION, expect_index=True) + assert on_ok["row_deflected"] is False + assert MOCK_ANSWER_MARKER in on_ok["answer"] + assert on_ok["recorded"][0] == CITED_DOC + assert _log_lines(ok_log, CLAMP_LINE) == [], ( + "ef_search=150 sits ABOVE the A7 candidate count and must not " + f"clamp: {_log_lines(ok_log, CLAMP_LINE)}" + ) + + # --- 50 (< the tier): the clamp fires and the turn is UNCHANGED -------- + clamp_dir = tmp_path_factory.mktemp("bor_ef_clamp") + with _running_app( + EF_CLAMP_URL, EF_CLAMP_PORT, _app_env(clamp_dir, "50"), clamp_dir / "ef50.log" + ) as (clamp_url, clamp_log): + assert clamp_log is not None + with SessionLocal() as db: + db.execute(text("TRUNCATE query_log")) + db.commit() + login(page, clamp_url, next="/") + page.evaluate(SSE_HOOK) + clamped = _arm(page, ANSWER_QUESTION, expect_index=True) + assert clamped["row_deflected"] is False + assert MOCK_ANSWER_MARKER in clamped["answer"] + assert clamped["recorded"][0] == CITED_DOC + assert clamped["recorded"] == on_ok["recorded"] + assert abs(clamped["top_score"] - on_ok["top_score"]) <= SCORE_SLACK + + lines = _log_lines(clamp_log, CLAMP_LINE) + assert lines, ( + "ef_search=50 is BELOW the A7 candidate count and MUST log the " + "clamp (AGENTS rule 10 — a knob that would degrade recall is never " + "silent). Tail of the captured log:\n" + f"{clamp_log.read_text(encoding='utf-8', errors='replace')[-3000:]}" + ) + for line in lines: + match = CLAMP_LINE.search(line) + assert match is not None, line + assert int(match.group(1)) == 50, line + assert int(match.group(2)) == A7_CANDIDATES, line diff --git a/tests/fixtures/hnswkb/garage/shop-vacuum.md b/tests/fixtures/hnswkb/garage/shop-vacuum.md new file mode 100644 index 0000000..ad41db9 --- /dev/null +++ b/tests/fixtures/hnswkb/garage/shop-vacuum.md @@ -0,0 +1,12 @@ +# Shop vacuum + +Dust collection layout for the garage bench and the tools it serves. + +- The cyclone bin empties at the driveway door, twice a week. +- Replace the final filter each season or the airflow drops noticeably. +- The hose is the flexible 4 inch line, never the rigid conduit. +- The saw runs on its own circuit so the motor start does not brown out + the bench outlets. + +Airflow is measured at the hood with a cheap gauge; anything under the +marker means the cyclone needs emptying before anything else is checked. diff --git a/tests/fixtures/hnswkb/home/coffee-grinder.md b/tests/fixtures/hnswkb/home/coffee-grinder.md new file mode 100644 index 0000000..bfe7fbf --- /dev/null +++ b/tests/fixtures/hnswkb/home/coffee-grinder.md @@ -0,0 +1,12 @@ +# Coffee grinder notes + +Burr alignment and dose calibration for the counter grinder. Nothing here +applies to the commercial machine at the shop. + +- Grind finer when the shot runs short and sour. +- Clean the chute weekly; static holds grounds in the bend. +- Recalibrate the dose after any burr change with a blind basket test. +- Replace the basket screen when the base of the puck turns muddy. + +The dial clicks are not linear near the fine end, so two clicks of +adjustment there behave like five clicks in the middle of the range. diff --git a/tests/fixtures/hnswkb/home/garden-beds.md b/tests/fixtures/hnswkb/home/garden-beds.md new file mode 100644 index 0000000..eb2d6cf --- /dev/null +++ b/tests/fixtures/hnswkb/home/garden-beds.md @@ -0,0 +1,12 @@ +# Garden beds + +Crop rotation for the four raised beds behind the shed, following the +legume, brassica, root, fruit order so the beds rest every fourth year. + +- Bed one: legumes after the last frost date clears the soil. +- Bed two: heavy feeders get a top dressing of compost in spring. +- Bed three: roots only, no manure or the carrots fork. +- Bed four: fruiting crops staked and watered at the base. + +The irrigation line is one drip tape per bed on a fifteen minute timer, +and the mulch goes back on after every harvest rather than once a season. diff --git a/tests/fixtures/hnswkb/homelab/kubernetes-cluster.md b/tests/fixtures/hnswkb/homelab/kubernetes-cluster.md new file mode 100644 index 0000000..cdc7a49 --- /dev/null +++ b/tests/fixtures/hnswkb/homelab/kubernetes-cluster.md @@ -0,0 +1,15 @@ +# Kubernetes cluster + +The cluster is three nodes with a control plane on `k8s-master-1` and two +workers. It runs the shelf services that want scheduling rather than a +static container. + +- The ingress controller listens on 8443 and forwards to cluster services. +- etcd snapshots run nightly to the backup host and are kept for a month. +- Worker nodes join with a short-lived token minted for one hour. +- Storage is a local-path provisioner pinned to the nvme mount. +- Pod security labels are set per namespace, not cluster wide. + +Upgrades are one control plane replica at a time, then the workers with a +cordon and drain. The ingress is the last thing to move because the +services behind it are checked by the shelf watchdog. diff --git a/tests/fixtures/hnswkb/homelab/proxmox-nodes.md b/tests/fixtures/hnswkb/homelab/proxmox-nodes.md new file mode 100644 index 0000000..75b039f --- /dev/null +++ b/tests/fixtures/hnswkb/homelab/proxmox-nodes.md @@ -0,0 +1,13 @@ +# Proxmox nodes + +Three Proxmox hosts form the shelf cluster: node one is the hypervisor and +gateway, node two runs the always-on containers, node three is the GPU box. + +- Storage is an LVM-thin pool on the nvme, over-commit watched weekly. +- Live migration uses a dedicated bridge, not the management iface. +- The watchdog restarts a container that misses three health checks. +- Corosync traffic rides the management bridge with multicast. +- Backups of the VM configs are taken before every host upgrade. + +Node three is pinned for the inference workloads, so a migration off it is +manual and only happens for a scheduled host reboot. diff --git a/tests/fixtures/hnswkb/homelab/restic-backups.md b/tests/fixtures/hnswkb/homelab/restic-backups.md new file mode 100644 index 0000000..e4a07cc --- /dev/null +++ b/tests/fixtures/hnswkb/homelab/restic-backups.md @@ -0,0 +1,14 @@ +# Restic backups + +Restic backs up the media volume to the offsite bucket every night at 01:20 +with a lock timeout of five minutes. + +- The repository password lives in the vault, never in the env file. +- Prune keeps fourteen daily and eight weekly snapshots. +- Verification runs one read pass every Sunday, sampled rather than full. +- A failed lock means the previous run is still going; check before + forcing anything. +- Restores are tested quarterly against a scratch directory. + +The media volume changes slowly, so a nightly run is usually small and the +sampled verification is enough to catch a corrupt pack file early. diff --git a/tests/fixtures/hnswkb/infra/ntfy-relay.md b/tests/fixtures/hnswkb/infra/ntfy-relay.md new file mode 100644 index 0000000..ff3cf0d --- /dev/null +++ b/tests/fixtures/hnswkb/infra/ntfy-relay.md @@ -0,0 +1,15 @@ +# Ntfy relay + +The ntfy relay runs on `notify-host-1` and fans alert payloads out to the +phones and the desktop notifier. + +- Topic namespaces are per host, so a host can only publish into its own. +- Retention is twelve hours; anything older is dropped on the sweep. +- The access log lives at `/var/log/ntfy/access.log` and rotates weekly. +- SMTP fallback is disabled on purpose: a failed push is a signal, not + something to paper over with an email. +- The relay is fronted by the proxy on 443 with mutual TLS for publishers. + +Publishing is a plain POST with the topic in the path. The relay stamps a +monotonic sequence number so a subscriber can detect a gap after a +reconnect, and the gap detection is what the shelf watchdog polls. diff --git a/tests/fixtures/hnswkb/infra/tilecache.md b/tests/fixtures/hnswkb/infra/tilecache.md new file mode 100644 index 0000000..c4a8db2 --- /dev/null +++ b/tests/fixtures/hnswkb/infra/tilecache.md @@ -0,0 +1,35 @@ +# Tilecache + +Tilecache serves the vector tiles for the map boxes from `map-host-2`, +listening on port 8077 behind the reverse proxy. Everything below is the +shape of that cache on the shelf, not a general tile server guide. + +## Cache budget and eviction + +The cache budget is 40 GiB on the nvme. The eviction watermark sits at 88 +percent of that budget: once the cache crosses the watermark, tilecache +evicts the coldest tiles until it drops back under it. + +Raise the eviction watermark so the hot zoom levels stay pinned through a +bulk re-render: + +1. `tilecache config set eviction-watermark 92` +2. `tilecache config set pin-zoom-floor 12` +3. `systemctl restart tilecache` + +Zoom levels 12 to 15 are the hot set for the atlas layers, and the pin +floor keeps them resident while the watermark holds. Zoom 16 and above is +rendered on demand and is always evictable. + +## Origin fetches + +Origin fetches go to `origin.maps.internal`, and a slow origin reply is +treated as a miss after 900 ms. A stale origin reply keeps the previous +tile pinned rather than evicting the hot zoom set, so a watermark raise is +safe during a bulk re-render. + +## Watermark maths + +At a 40 GiB budget the 88 percent watermark means eviction starts at 35.2 +GiB. The 92 percent setting moves that to 36.8 GiB, which buys roughly one +extra render pass of zoom 14 before anything cold is dropped. diff --git a/tests/fixtures/hnswkb/infra/wireguard-hub.md b/tests/fixtures/hnswkb/infra/wireguard-hub.md new file mode 100644 index 0000000..c1cd35f --- /dev/null +++ b/tests/fixtures/hnswkb/infra/wireguard-hub.md @@ -0,0 +1,15 @@ +# Wireguard hub + +The Wireguard hub on `edge-host-1` is the only way into the shelf from a +laptop. Each device gets one peer entry and one allowed-IPs line. + +- Listen port 51820/udp, one peer per device. +- Allowed IPs are per-device /32s; the hub does not advertise the LAN. +- Persistent keepalive is 25 seconds for the roaming peers. +- Key rotation is a manual ceremony: generate, swap the peer file, reload. +- The hub hands out addresses from 10.66.0.0/24, one /32 per peer. + +A peer that roams between the mobile network and wifi must keep the same +endpoint or the hub drops the stale mapping after the keepalive window. +The routing table is re-read on a reload, so a swapped peer file needs no +reboot of the host. diff --git a/tests/fixtures/hnswkb/kitchen/sourdough-timetable.md b/tests/fixtures/hnswkb/kitchen/sourdough-timetable.md new file mode 100644 index 0000000..f01b33f --- /dev/null +++ b/tests/fixtures/hnswkb/kitchen/sourdough-timetable.md @@ -0,0 +1,11 @@ +# Sourdough timetable + +Bulk fermentation schedule for the weekend bakes at the kitchen bench. + +- Stretch and fold every forty minutes for the first two hours. +- Bulk ends when the dough has grown by half and jiggles, not by clock. +- Cold retard overnight in the fridge at four degrees. +- Bake from cold with steam for the first twenty minutes. + +The starter is fed one to five in the morning so it peaks mid afternoon, +which is when the mix actually happens. diff --git a/tests/fixtures/hnswkb/travel/packing-list.md b/tests/fixtures/hnswkb/travel/packing-list.md new file mode 100644 index 0000000..dd6dfa8 --- /dev/null +++ b/tests/fixtures/hnswkb/travel/packing-list.md @@ -0,0 +1,12 @@ +# Packing list + +Carry-on checklist for a week of trips with one bag and a laptop. + +- One shirt per two days, always folded not rolled. +- Spare battery in the personal item, never the checked bag. +- Toiletries under the hundred millilitre rule in one clear pouch. +- A single pair of shoes that survives both rain and a dinner. + +The laptop bag holds the cables, the drive and the paper documents, and it +is the thing that gets opened at every security checkpoint, so it stays +boring and easy to repack. diff --git a/tests/integration/test_chunks_hnsw_index.py b/tests/integration/test_chunks_hnsw_index.py new file mode 100644 index 0000000..1a7ad64 --- /dev/null +++ b/tests/integration/test_chunks_hnsw_index.py @@ -0,0 +1,354 @@ +"""Integration: migration 0029 (chunks.embedding HNSW index) — phase 188, task 01. + +Drives the **real Alembic engine** against the live dev database +(``podman compose up -d db``), mirroring the house pattern of +``test_migration_0028.py`` (fixture upgrades to head at setup and +teardown so the dev DB is never left below head; tests target the +0028 → 0029 step explicitly so later migrations cannot break them): + +* upgrade 0028 → 0029 → ``ix_chunks_embedding_hnsw`` exists, + ``pg_index.indisvalid`` is TRUE, and ``pg_get_indexdef`` names + ``USING hnsw`` + ``vector_cosine_ops`` + ``m=16`` + ``ef_construction=64`` + (the D1 shape the recall pins of task 03 depend on); +* downgrade to 0028 → the index is GONE (A13 — reversible, ``CONCURRENTLY``), + the ``chunks`` table and its PK survive; +* upgrade back to head → the index is back and valid (round-trip); +* the D2 self-heal: with an INVALID index of that name deliberately + present, ``upgrade()`` drops it and rebuilds it VALID — no + ``relation already exists`` error and no leftover INVALID index. + The INVALID state is forced with the documented superuser catalog + trick ``UPDATE pg_index SET indisvalid = false WHERE + indexrelid = CAST(:name AS regclass)`` — the same failure a killed + ``CREATE INDEX CONCURRENTLY`` leaves behind (pgvector cannot fail a + build on data, and the dev/test DB role is a superuser; a role + without catalog UPDATE rights gets a clean skip). + +Pure-helper matrix tests (no DB): ``should_drop_invalid`` and +``is_valid`` over ``None`` / ``True`` / ``False``. + +**Why every ``command.*`` call here is preceded by ``db.rollback()`` +when the session was used.** psycopg3 keeps its transaction snapshot +alive while the session sits idle-in-transaction (measured: ``pg_stat +_activity.backend_xmin`` stays set where psql's clears), and ``CREATE +INDEX CONCURRENTLY`` phase 3 waits for every older-snapshot +transaction to finish — an unreleased session parks the build FOREVER +(observed on the dev box). The test session must therefore hold no +open transaction while the concurrent DDL runs; the rollback is what +makes the file deterministic rather than flaky. +**Task 02 adds the ``ef_search`` half:** the knob the index needs is +applied with ``SET LOCAL`` inside the turn's own transaction, so the file +pins the SCOPING against the real server — after a ``SET LOCAL`` the same +transaction reports the (clamped) value and a NEW transaction reports the +server's own default again (LOCKED A4: no global GUC, nothing surviving +the request or a pool recycle). +""" +from __future__ import annotations + +import importlib.util +import logging +import types +from collections.abc import Iterator +from pathlib import Path + +import pytest +from alembic.config import Config +from sqlalchemy import text +from sqlalchemy.exc import ProgrammingError +from sqlalchemy.orm import Session + +from alembic import command +from app.config import Settings +from app.db import db_available +from app.rag import retriever +from app.rag.retriever import _vector_candidates + +MIGRATION_PATH = Path("alembic/versions/0029_chunks_embedding_hnsw.py") +INDEX_NAME = "ix_chunks_embedding_hnsw" + + +def _load_migration() -> types.ModuleType: + """Import the 0029 module by file path (``alembic/versions`` is not a + package — the house way to reach a migration's own helpers).""" + spec = importlib.util.spec_from_file_location("m0029_chunks_embedding_hnsw", MIGRATION_PATH) + assert spec is not None and spec.loader is not None, f"missing {MIGRATION_PATH}" + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture() +def alembic(db: Session) -> Iterator[Config]: + """Real Alembic config bound to the dev DB (URL from app settings). + + Starts at head (repairs an interrupted earlier run — 0029's own + INVALID self-heal makes that repair safe when the interrupted + migration was 0029 itself); teardown upgrades to head no matter + what happened, so the dev DB is never left below head. + """ + if not db_available(): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + cfg = Config() # no alembic.ini file — env.py gets the URL from app config + cfg.set_main_option("script_location", "alembic") + command.upgrade(cfg, "head") + try: + yield cfg + finally: + # Release the test session's open transaction BEFORE the repair + # DDL (the 0018 house pattern): a still-running snapshot would + # park the CONCURRENTLY build, and teardown is exactly where a + # failed test leaves one open. + db.rollback() + command.upgrade(cfg, "head") + + +def _version(db: Session) -> str | None: + return db.execute(text("SELECT version_num FROM alembic_version")).scalar() + + +def _index_present(db: Session) -> bool: + return ( + db.execute( + text("SELECT 1 FROM pg_indexes WHERE indexname = :n"), {"n": INDEX_NAME} + ).scalar() + is not None + ) + + +def _index_valid(db: Session) -> bool | None: + """``pg_index.indisvalid`` for the index, ``None`` when it is absent.""" + return db.execute( + text( + "SELECT i.indisvalid FROM pg_index i" + " JOIN pg_class c ON c.oid = i.indexrelid" + " WHERE c.relname = :n" + ), + {"n": INDEX_NAME}, + ).scalar() + + +def _index_def(db: Session) -> str | None: + return db.execute( + text("SELECT pg_get_indexdef(c.oid) FROM pg_class c WHERE c.relname = :n"), + {"n": INDEX_NAME}, + ).scalar() + + +# --------------------------------------------------------------------------- +# Pure helpers (no DB, no I/O) +# --------------------------------------------------------------------------- + + +def test_should_drop_invalid_matrix() -> None: + """``should_drop_invalid``: absent → False, valid → False, + INVALID → True (the only state the pre-drop fires for).""" + mod = _load_migration() + assert mod.should_drop_invalid(None) is False, "absent index needs no drop" + assert mod.should_drop_invalid(True) is False, "a valid index must be left alone" + assert mod.should_drop_invalid(False) is True, "an INVALID remnant must be dropped" + + +def test_is_valid_matrix() -> None: + """``is_valid``: only a present, validated index counts as valid.""" + mod = _load_migration() + assert mod.is_valid(None) is False, "absent is not valid" + assert mod.is_valid(True) is True, "validated is valid" + assert mod.is_valid(False) is False, "INVALID is not valid" + + +def test_migration_module_contract() -> None: + """The module's identity: revision 0029 on top of 0028, and the one + index name the whole phase pins on.""" + mod = _load_migration() + assert mod.revision == "0029" + assert mod.down_revision == "0028" + assert mod.INDEX_NAME == INDEX_NAME + + +# --------------------------------------------------------------------------- +# Migration round-trip against the real Alembic engine +# --------------------------------------------------------------------------- + + +def test_upgrade_creates_a_valid_hnsw_index(db: Session, alembic: Config) -> None: + """Upgrade 0028 → 0029: the index exists, VALIDATED, with the D1 + shape — HNSW, ``vector_cosine_ops`` (the operator class A7's ``<=>`` + needs), ``m=16``, ``ef_construction=64`` — and the ``chunks`` table + itself survives the additive step untouched.""" + command.downgrade(alembic, "0028") # start from the pre-0029 state + assert _version(db) == "0028" + assert not _index_present(db), f"{INDEX_NAME} must be absent at 0028" + + db.rollback() # no open snapshot while the CONCURRENTLY build runs + command.upgrade(alembic, "0029") + assert _version(db) == "0029", "alembic_version must be at 0029" + assert _index_present(db), "the index must exist at 0029" + assert _index_valid(db) is True, "a fresh build must be VALID (D2 post-check)" + + raw = _index_def(db) + assert raw is not None + definition = raw.replace("'", "") # pg renders WITH options quoted + assert "USING hnsw" in definition, f"not an HNSW index: {definition}" + assert "vector_cosine_ops" in definition, ( + f"cosine operator class required for <=>: {definition}" + ) + assert "m=16" in definition, f"m=16 expected: {definition}" + assert "ef_construction=64" in definition, f"ef_construction=64 expected: {definition}" + + +def test_downgrade_removes_it(db: Session, alembic: Config) -> None: + """Downgrade 0029 → 0028: the index is gone (A13 — reversible), and + the ``chunks`` relation with its PK survives (the drop touches no + data).""" + command.upgrade(alembic, "0029") + assert _index_present(db) + + db.rollback() # no open snapshot while the CONCURRENTLY drop runs + command.downgrade(alembic, "0028") + assert _version(db) == "0028" + assert not _index_present(db), f"{INDEX_NAME} must be dropped by the downgrade" + assert _index_valid(db) is None, "no pg_index row may survive the downgrade" + + pk = db.execute( + text("SELECT 1 FROM pg_indexes WHERE indexname = 'chunks_pkey'") + ).scalar() + assert pk is not None, "chunks_pkey must survive the downgrade" + + +def test_upgrade_head_again(db: Session, alembic: Config) -> None: + """A13 round-trip: downgrade to 0028, upgrade back to HEAD — the + index returns, valid, with the same definition.""" + command.downgrade(alembic, "0028") + command.upgrade(alembic, "head") + assert _index_present(db), "the index must be back after re-upgrade" + assert _index_valid(db) is True, "the rebuilt index must be VALID" + raw = _index_def(db) + assert raw is not None and "USING hnsw" in raw + + +def test_invalid_index_is_dropped_and_rebuilt(db: Session, alembic: Config) -> None: + """D2 self-heal: an INVALID index of this name (what a killed + ``CREATE INDEX CONCURRENTLY`` leaves behind) is dropped and rebuilt + VALID on the next ``upgrade()``. + + The INVALID state is forced with the documented superuser catalog + update ``UPDATE pg_index SET indisvalid = false WHERE indexrelid = + CAST(:name AS regclass)`` — pgvector cannot fail an HNSW build on + data, and timing a real interruption is nondeterministic; the dev/ + test DB role is a superuser, and a role without catalog UPDATE + rights skips cleanly instead of leaving the DB inconsistent (the + stamp that follows the forced state, never before it).""" + command.upgrade(alembic, "head") + assert _index_valid(db) is True, "precondition: a valid index to corrupt" + + try: + db.execute( + text( + "UPDATE pg_index SET indisvalid = false" + " WHERE indexrelid = CAST(:n AS regclass)" + ), + {"n": INDEX_NAME}, + ) + db.commit() + except ProgrammingError as exc: # non-superuser role + db.rollback() + pytest.skip(f"forcing INVALID needs catalog UPDATE rights: {exc}") + + # The stamp moves the version table WITHOUT touching the index, so + # the next upgrade() re-runs 0029 against the corrupted state. + command.stamp(alembic, "0028") + assert _version(db) == "0028" + assert _index_valid(db) is False, "the forced INVALID state must be visible" + + db.rollback() # no open snapshot of ours while the CONCURRENTLY run rebuilds + command.upgrade(alembic, "0029") + assert _index_present(db), "no 'relation already exists' — it was pre-dropped" + assert _index_valid(db) is True, "the rebuilt index must be VALID (no leftover INVALID)" + raw = _index_def(db) + assert raw is not None and "USING hnsw" in raw, "the rebuild is the HNSW index itself" + + +# --------------------------------------------------------------------------- +# Task 02: SET LOCAL hnsw.ef_search — applied, clamped, transaction-scoped +# --------------------------------------------------------------------------- + + +def _show_ef_search(db: Session) -> str: + return str(db.execute(text("SHOW hnsw.ef_search")).scalar()) + + +def _server_default_ef_search(db: Session) -> str: + """The server's own reset value for the GUC (the value a NEW + transaction must fall back to). Read from ``pg_settings`` rather than + hardcoded: pgvector renders its default, and the pin is about the + FALLBACK, not the number.""" + value = db.execute( + text("SELECT reset_val FROM pg_settings WHERE name = 'hnsw.ef_search'") + ).scalar() + assert value is not None, "pgvector must register hnsw.ef_search in this session" + return str(value) + + +def _stub_settings(**kwargs: object) -> Settings: + return Settings(_env_file=None, **kwargs) # pyright: ignore[reportCallIssue] + + +def test_set_local_ef_search_reaches_the_server_in_the_same_transaction( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """The D3 rule against a real server: the value the tier sends is the + value Postgres runs the search with. The setting here (10) sits BELOW + the limit (250), so the server must report the CLAMPED value — the + clamp is not a Python-side fiction.""" + monkeypatch.setattr( + retriever, "get_settings", lambda: _stub_settings(retrieval_ef_search=10) + ) + db.rollback() # a clean transaction boundary for the SET LOCAL + _vector_candidates(db, [0.0] * 768, limit=250) # pyright: ignore[reportArgumentType] + assert _show_ef_search(db) == "250", "the clamped value must be live in this tx" + db.rollback() + + +def test_set_local_ef_search_does_not_survive_the_transaction( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """LOCKED A4 pinned on the wire: once the turn's transaction ends, the + server is back on ITS default — no global GUC mutation, no state a + pooled connection carries into the next request.""" + monkeypatch.setattr( + retriever, "get_settings", lambda: _stub_settings(retrieval_ef_search=317) + ) + db.rollback() + _vector_candidates(db, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + assert _show_ef_search(db) == "317" + + db.rollback() # the end of the turn's transaction + after = _show_ef_search(db) + default = _server_default_ef_search(db) + assert after == default, f"a new tx must see the server default, got {after}" + assert after != "317" + db.rollback() + + +def test_ef_search_clamp_logs_one_warning_line( + db: Session, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """The clamp leaves exactly one operator-visible line in the server log + (AGENTS rule 10), and the pass-through case leaves none.""" + monkeypatch.setattr( + retriever, "get_settings", lambda: _stub_settings(retrieval_ef_search=5) + ) + db.rollback() + with caplog.at_level(logging.WARNING, logger="app.rag.retriever"): + _vector_candidates(db, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + db.rollback() + lines = [r.getMessage() for r in caplog.records if "ef_search" in r.getMessage()] + assert lines == ["retrieval: ef_search=5 raised to 100 (A7 candidate count)"] + + monkeypatch.setattr( + retriever, "get_settings", lambda: _stub_settings(retrieval_ef_search=100) + ) + caplog.clear() + with caplog.at_level(logging.WARNING, logger="app.rag.retriever"): + _vector_candidates(db, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + db.rollback() + assert [r.getMessage() for r in caplog.records if "ef_search" in r.getMessage()] == [] diff --git a/tests/integration/test_chunks_hnsw_recall.py b/tests/integration/test_chunks_hnsw_recall.py new file mode 100644 index 0000000..f02f84e --- /dev/null +++ b/tests/integration/test_chunks_hnsw_recall.py @@ -0,0 +1,612 @@ +"""Integration: the cosine tier USES ``ix_chunks_embedding_hnsw`` and the +top-100 did not move (phase 188, task 03 — the deterministic half of D4). + +**The production numbers this pin stands in for** (read 2026-10-15, read-only): +414,220 chunks — all with a non-null ``embedding`` — a ``chunks`` relation of +≈ **3,000 MB** (database 3,147 MB), ``shared_buffers=128MB``, ``work_mem=4MB``, +a 3,904 MB host with **no swap**, pgvector **0.8.0**, and ``pg_indexes`` for +``chunks`` listing only ``chunks_pkey``, ``ix_chunks_document_id`` and +``ix_chunks_tsv`` (GIN): **no index on ``embedding``**. Every chat turn's A7 +cosine tier (``ORDER BY embedding <=> :vec LIMIT 100``) was therefore a +sequential scan of that 3 GB relation. + +Two CI pins, both deterministic and LLM-free: + +1. **The plan pin** — ``EXPLAIN (FORMAT JSON)`` of the tier's own SELECT (the + statement object captured from + :func:`app.rag.retriever._vector_candidates`, compiled with literal binds so + the planner sees a real constant) names ``ix_chunks_embedding_hnsw`` and + contains no ``Seq Scan on chunks``. Its vacuity guard runs the SAME + statement with index scans refused and MUST see the seq scan, so the pin + cannot pass on a plan the index never influenced. +2. **The recall pin** — the index-backed top-100 against the EXACT cosine + top-100 of the fixture corpus: ``|overlap| / 100 >= 0.99`` for EVERY query + at the SHIPPED ``ef_search`` (400 since task 04's live battery; 100, the + A7 candidate count, was the task-02 default and the battery retired it), + each returned row's ``1 − distance`` equal to the exact cosine within + **1e-6**, and the mean one step DEEPER never below the mean at the shipped + value. Every recall run also re-reads the plan it just ran, because an + ``ef_search`` the planner declines turns the "index" pin into a second + measurement of the sequential scan — a vacuous pass, and the one failure + mode this file exists to catch. + +**Measured on this corpus** (5,000 chunks, ``TOP_K = 100``, 30 fixed-seed +queries, ``m = 16`` / ``ef_construction = 64`` from migration 0029): +``ef_search=100`` → mean top-100 overlap **0.9993**, per-query minimum **0.99** +(two of the thirty return 99/100), zero queries below the floor; +``ef_search=200`` and the shipped **400** → mean **1.0000**. Worst +``|cosine − exact cosine|`` over every +returned row: **5.7e-07**, and 1e-6 is the tightest tolerance that can hold — +``_vector_candidates`` publishes ``round(1 − distance, 6)``, which alone can move +a value by 5e-7, on top of the float4 rounding of the distance itself (~6e-8 at +this magnitude). The exact reference is computed over the vectors as the +database returned them, so the comparison measures the index. Setup stays inside the runtime guard: +generate ≈ 0.5 s, seed ≈ 2 s, ``CREATE INDEX CONCURRENTLY`` ≈ 1.4 s, read-back + +exact top-100 ≈ 3.5 s — ≈ 8 s against the ~60 s target. + +**What 5,000 rows proves and what it does not.** It is enough for a PLAN pin +(the planner needs a populated relation to prefer an ordered index scan over a +seq scan + top-N sort) and enough for a deterministic index-vs-exact +regression; it is not a proxy for 414k rows, because recall is a function of the +corpus shape and of graph density at 83× the size. The prod-scale gate is task +04 — the live retrieval battery, index off vs on, over the 1,000-document +replica KB, where every ``top_score``, top-5 set and A8 verdict must match. + +**Corpus shape, and why it is not uniform noise.** One +``random.Random(20261015)`` stream in a fixed order: a 16-direction principal +subspace (one random unit direction per disjoint 48-dim block of the 768 — +disjoint blocks are orthonormal by construction, so no Gram–Schmidt and no +numerical drift), 40 unit centroids inside it, 5,000 chunks of +``normalise(centroid + 2.0 × a random subspace direction)`` (125 per cluster), +then 30 queries drawn uniformly from the same subspace. Low EFFECTIVE rank (16 +of the 768 stored dimensions carry the structure; every dimension is still +non-zero) is deliberate: it is how trained text embeddings behave (their +spectrum concentrates far below the nominal dimension), and it is the shape +under which ``ef_search == LIMIT`` can be expected to return the exact top-100. + +The counter-case is recorded here because measuring it is a finding of this +task: the same 5,000 rows with the cluster bias thinned to a whisper (centroid +weight 0.1 over per-dimension Gaussian noise across all 768 dimensions — +effectively iid, the hardest shape a graph index can be handed, and the one +this file's sweep ran at centroid weights 0.1, 0.3 and 0.5 alike) measures mean +recall@100 ≈ **0.63** at ``ef_search=100``, ≈ **0.85** at 200 and **1.00** from +400 up, whatever ``m`` / ``ef_construction`` migration 0029 ships. ``ef_search`` +at the candidate count is therefore NOT a free win on high-effective-rank data; +if the live battery moves a ``top_score`` or a top-5 set, LOCKED A3's answer is +to raise ``BOR_RETRIEVAL_EF_SEARCH`` (towards 400), never to relax an assertion +here. + +The index is built by migration **0029** and never by an inline ``CREATE +INDEX`` — the migration is the single definition of the index these pins name +(``0029 → 0028`` → seed unindexed → ``upgrade 0029``). As in +``test_chunks_hnsw_index.py``, the seeding session holds NO open transaction +while the ``CONCURRENTLY`` DDL runs (an idle-in-transaction snapshot parks the +build), and teardown leaves the dev DB at head, index valid, fixture rows gone. +""" +from __future__ import annotations + +import math +import operator +import random +import struct +import uuid +from collections.abc import Iterator +from typing import Any + +import pytest +from alembic.config import Config +from sqlalchemy import select, text +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import Session + +from alembic import command +from app.config import Settings +from app.db import SessionLocal, db_available +from app.models import Chunk, Document +from app.rag import retriever +from app.rag.retriever import _vector_candidates + +# --------------------------------------------------------------------------- +# The corpus (D4's deterministic half) — one fixed stream, no wall clock +# --------------------------------------------------------------------------- + +DIM = 768 # A6: the embedding dimension is fixed at table creation +CENTROIDS = 40 +SUBSPACE = 16 # effective rank of the corpus (see the module docstring) +BLOCK = DIM // SUBSPACE # 48 dims per basis direction, disjoint by construction +NOISE = 2.0 # |noise| / |centroid| — cos(chunk, own centroid) ~= 0.45 +CORPUS_CHUNKS = 5000 +CHUNKS_PER_DOCUMENT = 10 +QUERIES = 30 +SEED = 20261015 # the date of the incident that started phase 188 +FIXTURE_SOURCE = "phase-188-hnsw-recall" + +TOP_K = 100 # A7's cosine candidate count — the LIMIT the tier promises +EF_PIN = 400 # the shipped default (Settings.retrieval_ef_search) +EF_RAISED = 600 # the monotonicity probe +RECALL_FLOOR = 0.99 # LOCKED A3 — a raised ef_search is the only remedy +COSINE_TOLERANCE = 1e-6 # "exact to 6 decimals" + +#: The pins need a small relation: the exact reference is a Python scan over +#: every row. A DB holding more is a restored replica rather than the dev +#: corpus — that scale is task 04's live battery, and 30 × N dot products here +#: would only make the suite crawl. +MAX_RELATION_ROWS = 20_000 + +INDEX_NAME = "ix_chunks_embedding_hnsw" +RELATION = "chunks" + + +def _f32(value: float) -> float: + """The ``float4`` the database holds for a query component. + + A question vector goes to Postgres as ``str(float)`` text and comes back as + ``float4`` (``strtof`` rounds to nearest, the same value ``struct.pack`` + produces), so the exact reference rounds the QUERY the way the server does. + The corpus needs no modelling — its components are read back from the + database — which is what makes the comparison measure the INDEX rather than + a precision artefact of the fixture. + """ + return struct.unpack("f", struct.pack("f", value))[0] + + +def _norm(vec: list[float]) -> float: + return math.sqrt(sum(v * v for v in vec)) + + +def _unit(vec: list[float]) -> list[float]: + n = _norm(vec) + return [v / n for v in vec] if n else vec + + +def _generate() -> tuple[list[list[float]], list[list[float]]]: + """Fixed-seed ``(corpus, queries)``, all L2-normalised. + + Normalising makes cosine the dot product, which is what keeps the exact + reference cheap enough to run in CI. One stream in a fixed order (basis → + centroids → chunks → queries) is what turns the recall floor into a + regression pin instead of a measurement: byte-identical input, therefore a + byte-identical graph, therefore a byte-identical overlap. + """ + rng = random.Random(SEED) + basis = [_unit([rng.gauss(0.0, 1.0) for _ in range(BLOCK)]) for _ in range(SUBSPACE)] + + def subspace_vector() -> list[float]: + """A random unit vector inside the 16-direction subspace.""" + coefs = [rng.gauss(0.0, 1.0) for _ in range(SUBSPACE)] + n = _norm(coefs) or 1.0 + out = [0.0] * DIM + for k, direction in enumerate(basis): + weight = coefs[k] / n + start = k * BLOCK + for j in range(BLOCK): + out[start + j] = weight * direction[j] + return out + + centroids = [subspace_vector() for _ in range(CENTROIDS)] + corpus: list[list[float]] = [] + for i in range(CORPUS_CHUNKS): + centroid = centroids[i % CENTROIDS] + noise = subspace_vector() + corpus.append(_unit([c + NOISE * n for c, n in zip(centroid, noise, strict=True)])) + queries = [subspace_vector() for _ in range(QUERIES)] + return corpus, queries + + +def _seed(session: Session, corpus: list[list[float]]) -> None: + """Insert the corpus — 500 fixture documents × 10 chunks, one commit. + + Ordinary content chunks (``is_summary`` False, embedding set): the + NULL-embedding skip is pinned in ``tests/unit/test_retriever.py``, and a + NULL row would sit in neither the exact set nor the index. + """ + documents = [ + Document( + id=uuid.uuid4(), + source=FIXTURE_SOURCE, + path=f"{FIXTURE_SOURCE}/{d}.md", + full_path=f"/tmp/{FIXTURE_SOURCE}/{d}.md", + title=f"hnsw recall fixture {d}", + content="fixture content", + content_hash="0" * 64, + ) + for d in range(CORPUS_CHUNKS // CHUNKS_PER_DOCUMENT) + ] + session.add_all(documents) + session.flush() + session.add_all( + [ + Chunk( + id=uuid.uuid4(), + document_id=documents[i // CHUNKS_PER_DOCUMENT].id, + position=i % CHUNKS_PER_DOCUMENT, + content=f"fixture chunk {i}", + embedding=vector, + is_summary=False, + ) + for i, vector in enumerate(corpus) + ] + ) + session.commit() + + +# --------------------------------------------------------------------------- +# Exact reference (pure Python, float4-modelled inputs) +# --------------------------------------------------------------------------- + + +def _exact_top(stored: dict[Any, list[float]], query: list[float]) -> list[tuple[float, Any]]: + """The EXACT cosine top-:data:`TOP_K` of *stored* for *query*. + + Mirrors pgvector's ``vector_cosine_distance``: three float8 accumulators + (``dot``, ``norm_a``, ``norm_b``) over the components left to right, then + ``dot / sqrt(norm_a * norm_b)``. The norms are recomputed from the stored + components rather than assumed to be 1, because the stored component is the + float4 of a normalised double. + + Pure Python (no numpy in this project — A3 keeps the dependency list + closed): ``sum(map(mul, …))`` is ~25 µs per 768-dim dot product, so 30 + queries over 5,000 rows cost ~3 s. The one residual inaccuracy is the + read-back itself: pgvector prints each float4 in shortest-round-trip form, + so a component comes back within ~1 ulp of the float4 (measured max + 7.4e-09 absolute over the fixture corpus) — three orders below the 1e-6 + this file pins. + """ + q = [_f32(v) for v in query] + q_norm = math.sqrt(sum(map(operator.mul, q, q))) + scored = [ + ( + sum(map(operator.mul, vec, q)) / (math.sqrt(sum(map(operator.mul, vec, vec))) * q_norm), + chunk_id, + ) + for chunk_id, vec in stored.items() + ] + scored.sort(key=lambda item: -item[0]) + return scored[:TOP_K] + + +# --------------------------------------------------------------------------- +# Capturing the tier's own statement (the plan pin must explain THAT SQL) +# --------------------------------------------------------------------------- + + +class _StatementRecorder: + """Records what :func:`_vector_candidates` sends, answering with no rows.""" + + def __init__(self) -> None: + self.statements: list[Any] = [] + + def execute(self, statement: Any, params: Any = None) -> _StatementRecorder: + self.statements.append(statement) + return self + + def all(self) -> list[Any]: + return [] + + +def _vector_statement_sql() -> str: + """The tier's cosine SELECT as one literal SQL string, ready for EXPLAIN. + + A hand-written copy would drift the moment the retriever changes and would + then pin SQL no request ever runs, so the statement OBJECT is captured. + ``literal_binds`` gives the planner a real vector constant + (``pgvector.sqlalchemy.VECTOR`` ships a literal processor, and the resulting + unknown string literal coerces to ``vector`` through the ``<=>`` operator's + own signature). The ``SET LOCAL`` line is deliberately not part of it: + ``EXPLAIN`` takes one statement, and that line tunes the search depth, not + the shape of the plan. + """ + recorder = _StatementRecorder() + _vector_candidates(recorder, [0.0] * DIM, TOP_K) # pyright: ignore[reportArgumentType] + set_local, select_statement = recorder.statements + assert str(set_local) == f"SET LOCAL hnsw.ef_search = {EF_PIN}", ( + f"the tier's ef_search line changed ({set_local}) — the capture must be the SELECT" + ) + sql = str( + select_statement.compile( + dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True} + ) + ) + assert "<=>" in sql, f"the cosine operator is gone from the tier: {sql[:200]}" + return sql + + +def _plan(session: Session, sql: str) -> list[dict[str, Any]]: + """Every node of ``EXPLAIN (FORMAT JSON)`` for *sql*, flattened.""" + document = session.execute(text(f"EXPLAIN (FORMAT JSON) {sql}")).scalar() + assert isinstance(document, list) and document, f"no plan for {sql[:120]}: {document!r}" + nodes: list[dict[str, Any]] = [] + + def walk(node: dict[str, Any]) -> None: + nodes.append(node) + for child in node.get("Plans", []): + walk(child) + + walk(document[0]["Plan"]) + return nodes + + +def _describe(nodes: list[dict[str, Any]]) -> str: + return ", ".join( + f"{node.get('Node Type')}[{node.get('Index Name') or node.get('Relation Name') or '?'}]" + for node in nodes + ) + + +def _uses_index(nodes: list[dict[str, Any]], index_name: str) -> bool: + return any( + "Index Scan" in str(node.get("Node Type", "")) and node.get("Index Name") == index_name + for node in nodes + ) + + +def _seq_scans(nodes: list[dict[str, Any]], relation: str) -> bool: + return any( + node.get("Node Type") == "Seq Scan" and node.get("Relation Name") == relation + for node in nodes + ) + + +# --------------------------------------------------------------------------- +# Fixture: seed unindexed, build through 0029, read the corpus back once +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def hnsw_corpus() -> Iterator[dict[str, Any]]: + """Seed the corpus, build the index with migration 0029, precompute exact. + + ``0029 → 0028`` → seed → ``0029``: seeding with the index present would pay + a graph insert per row, and building after the load is what the operator + does on the production box anyway. Teardown deletes ONLY this module's rows + (``source = FIXTURE_SOURCE``; the chunk rows go with them by the DB-level + cascade — AGENTS rule 12, no chunk hydration) and leaves the dev DB at head + with the index valid, exactly as it was found. + """ + if not db_available(): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + session = SessionLocal() + config = Config() # no alembic.ini — env.py reads the URL from app config + config.set_main_option("script_location", "alembic") + try: + preexisting = session.execute(text(f"SELECT count(*) FROM {RELATION}")).scalar() or 0 + if preexisting > MAX_RELATION_ROWS: + pytest.skip( + f"{RELATION} holds {preexisting} rows: the exact reference is a Python " + "scan, so these pins need the small dev corpus. Production scale " + "is task 04's live battery." + ) + corpus, queries = _generate() + session.rollback() # no open snapshot while the CONCURRENTLY DDL runs + command.downgrade(config, "0028") + session.rollback() + _seed(session, corpus) + session.rollback() + command.upgrade(config, "0029") # the ONE definition of the index + session.rollback() + # Planner statistics for the plan pin: a freshly loaded relation has + # none, and a row estimate of 0 makes "seq scan + top-N sort" look free. + session.execute(text(f"ANALYZE {RELATION}")) + session.commit() + rows = session.execute(select(Chunk.id, Chunk.embedding)).all() + stored: dict[Any, list[float]] = {row[0]: row[1] or [] for row in rows} + assert len(stored) >= preexisting + CORPUS_CHUNKS, ( + f"expected {CORPUS_CHUNKS} fixture rows on top of {preexisting}, " + f"the relation holds {len(stored)}" + ) + # The exact sets are the expensive half (30 × N dot products); the two + # recall pins differ only in the ef_search they ask the index for. + yield { + "queries": queries, + "exact": [_exact_top(stored, query) for query in queries], + "chunk_ids": set(stored), + "plan_sql": _vector_statement_sql(), + } + finally: + session.rollback() + session.execute( + text("DELETE FROM documents WHERE source = :src"), {"src": FIXTURE_SOURCE} + ) + session.commit() + session.rollback() + command.upgrade(config, "head") # the dev DB never stays below head + session.close() + + +def _stub_settings(**kwargs: object) -> Settings: + """A :class:`Settings` off the env file (the ``test_chunks_hnsw_index.py`` + helper's shape — an operator's shell must not tune these pins).""" + return Settings(_env_file=None, **kwargs) # pyright: ignore[reportCallIssue] + + +def _force_index_scan(session: Session) -> None: + """Put the planner on the index, because a recall pin has to measure the + index and not the planner's preference. + + Postgres costs an HNSW scan partly on ``ef_search``, so on a relation this + small it can legitimately prefer a seq scan + top-N sort — and a recall run + that did that would score the EXACT scan twice and report a perfect 1.0000 + without visiting a single graph node. That is not hypothetical: the live + gate (``benchmarks/phase188_live_gate/``) watched exactly this happen on the + 8,866-chunk replica, where the planner used the index at ``ef_search = 40`` + and refused it at 100. + + ``enable_seqscan = off`` is Postgres' own "take any other plan if one + exists" lever. It is set at SESSION scope because ``_vector_candidates`` + opens its own transaction, so a ``SET LOCAL`` issued here would die with the + caller's transaction instead of reaching the tier. Every caller resets it in + a ``finally`` — a leaked ``off`` would slow every later test on this pooled + connection — and the assertion beside it proves the lever took effect. + """ + session.rollback() + session.execute(text("SET enable_seqscan = off")) + + +def _release_index_scan(session: Session) -> None: + session.rollback() + session.execute(text("RESET enable_seqscan")) + session.rollback() + + +def _set_ef_search(monkeypatch: pytest.MonkeyPatch, value: int) -> None: + """Drive the tier at *value* the way an operator does — the setting.""" + settings = _stub_settings(retrieval_ef_search=value) + monkeypatch.setattr(retriever, "get_settings", lambda: settings) + + +# --------------------------------------------------------------------------- +# Pin 1 — the planner uses the index (with its vacuity guard) +# --------------------------------------------------------------------------- + + +def test_plan_uses_the_hnsw_index(db: Session, hnsw_corpus: dict[str, Any]) -> None: + """The 2026-10-15 incident WAS a plan, so the pin is a plan: the tier's own + ``ORDER BY embedding <=> LIMIT 100`` runs through + ``ix_chunks_embedding_hnsw`` and never reads ``chunks`` sequentially. + + Explaining the captured statement is also what proves D1's "no query + rewrite": the ``<=>`` the retriever already emits is exactly the operator + ``vector_cosine_ops`` serves. + """ + nodes = _plan(db, hnsw_corpus["plan_sql"]) + assert _uses_index(nodes, INDEX_NAME), ( + f"{INDEX_NAME} is not in the plan — the incident is back: [{_describe(nodes)}]" + ) + assert not _seq_scans(nodes, RELATION), ( + f"a sequential scan of {RELATION} survived the index: [{_describe(nodes)}]" + ) + db.rollback() + + +def test_plan_without_the_index_seq_scans_the_relation( + db: Session, hnsw_corpus: dict[str, Any] +) -> None: + """Vacuity guard: the same statement, same session, same statistics, with + index scans refused, MUST fall back to ``Seq Scan on chunks``. + + ``SET LOCAL enable_indexscan = off`` rather than dropping the index: a + ``DROP INDEX CONCURRENTLY`` + rebuild would cost a second graph build and + leave the corpus unindexed if the rebuild failed, while this GUC is + transaction-scoped (it dies with the rollback) and removes exactly the + access path under test. Without this test the plan pin would pass vacuously + on any plan that merely mentions some index. + """ + sql = hnsw_corpus["plan_sql"] + db.rollback() + db.execute(text("SET LOCAL enable_indexscan = off")) + db.execute(text("SET LOCAL enable_indexonlyscan = off")) + nodes = _plan(db, sql) + assert _seq_scans(nodes, RELATION), ( + f"the vacuity guard itself did not seq-scan {RELATION}: [{_describe(nodes)}]" + ) + assert not _uses_index(nodes, INDEX_NAME), ( + f"{INDEX_NAME} was used with index scans disabled: [{_describe(nodes)}]" + ) + db.rollback() + + +# --------------------------------------------------------------------------- +# Pin 2 — the approximate top-100 IS the exact top-100 +# --------------------------------------------------------------------------- + + +def test_top100_recall_at_the_shipped_ef_search( + db: Session, + hnsw_corpus: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """At the shipped ``ef_search`` the index returns the exact top-100 for + every query, exact to 1e-6 on the score the honesty gate reads. + + Two different failures surface here: a ``top_score`` that moved — A8's + deflect/answer verdict flips on that one number, so it is pinned per row, + not per query — and an overlap below the floor, i.e. a candidate that never + reaches RRF. The failure message carries every query's overlap so a future + ``ef_search`` raise has its evidence in the report instead of in a re-run. + """ + queries: list[list[float]] = hnsw_corpus["queries"] + exact: list[list[tuple[float, Any]]] = hnsw_corpus["exact"] + chunk_ids: set[Any] = hnsw_corpus["chunk_ids"] + _set_ef_search(monkeypatch, EF_PIN) + + _force_index_scan(db) + try: + depth_nodes = _plan(db, hnsw_corpus["plan_sql"]) + assert _uses_index(depth_nodes, INDEX_NAME), ( + f"this recall run at ef_search={EF_PIN} did not use {INDEX_NAME} even " + "with sequential scans refused — the pin would be measuring a scan " + f"and calling it parity: [{_describe(depth_nodes)}]" + ) + + overlaps: list[float] = [] + worst_delta = 0.0 + for index, (query, exact_rows) in enumerate(zip(queries, exact, strict=True)): + exact_ids = {chunk_id for _, chunk_id in exact_rows} + exact_scores = {chunk_id: cosine for cosine, chunk_id in exact_rows} + candidates = _vector_candidates(db, query, TOP_K) + db.rollback() # end the turn's transaction, SET LOCAL included + assert len(candidates) == TOP_K, ( + f"query {index}: the tier returned {len(candidates)} candidates, " + f"not the {TOP_K} the A7 cosine tier promises" + ) + got_ids = {candidate.chunk_id for candidate in candidates} + assert got_ids <= chunk_ids, ( + f"query {index}: a candidate is outside the corpus the exact " + "reference was computed over" + ) + overlaps.append(len(exact_ids & got_ids) / TOP_K) + for candidate in candidates: + exact_cosine = exact_scores.get(candidate.chunk_id) + if exact_cosine is not None: + worst_delta = max(worst_delta, abs(candidate.cosine - exact_cosine)) + finally: + _release_index_scan(db) + + mean = sum(overlaps) / len(overlaps) + below = [(i, overlap) for i, overlap in enumerate(overlaps) if overlap < RECALL_FLOOR] + assert not below, ( + f"top-{TOP_K} recall below {RECALL_FLOOR} at ef_search={EF_PIN} for " + f"{len(below)}/{len(overlaps)} queries: {below}; per-query overlap=" + f"{[round(overlap, 4) for overlap in overlaps]} mean={mean:.4f} — the " + "remedy is a higher BOR_RETRIEVAL_EF_SEARCH, never this floor (LOCKED A3)" + ) + assert worst_delta < COSINE_TOLERANCE, ( + f"1 - distance drifted from the exact cosine by {worst_delta:.3e} " + f"(tolerance {COSINE_TOLERANCE:.0e}) — top_score would not be exact" + ) + + +def test_recall_never_below_the_pinned_floor_when_ef_search_raised( + db: Session, + hnsw_corpus: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Monotonicity sanity: a deeper search cannot recover FEWER of the exact + top-100 than a shallower one. + + This is the property the operator leans on when the live battery (task 04) + says "raise it" — ``ef_search`` is a recall dial, not a lottery. A + regression here means the value never reaches the server (the ``SET LOCAL`` + of task 02), which is exactly what the shipped 400 would then silently not + do. ``EF_RAISED`` sits ABOVE the shipped value, so the probe still measures + a deeper search after task 04 moved the default. + """ + exact: list[list[tuple[float, Any]]] = hnsw_corpus["exact"] + queries: list[list[float]] = hnsw_corpus["queries"] + + def mean_overlap(ef_search: int) -> float: + _set_ef_search(monkeypatch, ef_search) + total = 0.0 + for query, exact_rows in zip(queries, exact, strict=True): + exact_ids = {chunk_id for _, chunk_id in exact_rows} + candidates = _vector_candidates(db, query, TOP_K) + db.rollback() + total += len(exact_ids & {candidate.chunk_id for candidate in candidates}) / TOP_K + return total / len(exact) + + pinned = mean_overlap(EF_PIN) + raised = mean_overlap(EF_RAISED) + assert raised >= pinned, ( + f"ef_search={EF_RAISED} recovered less of the exact top-{TOP_K} " + f"({raised:.4f}) than ef_search={EF_PIN} ({pinned:.4f})" + ) diff --git a/tests/integration/test_migration_0004.py b/tests/integration/test_migration_0004.py index 929c664..f684de6 100644 --- a/tests/integration/test_migration_0004.py +++ b/tests/integration/test_migration_0004.py @@ -46,6 +46,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0005.py b/tests/integration/test_migration_0005.py index 2951e56..e858b99 100644 --- a/tests/integration/test_migration_0005.py +++ b/tests/integration/test_migration_0005.py @@ -48,6 +48,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0006.py b/tests/integration/test_migration_0006.py index 81d7e93..28768a1 100644 --- a/tests/integration/test_migration_0006.py +++ b/tests/integration/test_migration_0006.py @@ -50,6 +50,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0007.py b/tests/integration/test_migration_0007.py index 68a4cec..d071df5 100644 --- a/tests/integration/test_migration_0007.py +++ b/tests/integration/test_migration_0007.py @@ -57,6 +57,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0008.py b/tests/integration/test_migration_0008.py index 4804d47..64553b1 100644 --- a/tests/integration/test_migration_0008.py +++ b/tests/integration/test_migration_0008.py @@ -60,6 +60,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0009.py b/tests/integration/test_migration_0009.py index a35d754..d8617b5 100644 --- a/tests/integration/test_migration_0009.py +++ b/tests/integration/test_migration_0009.py @@ -52,6 +52,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0010.py b/tests/integration/test_migration_0010.py index 841a614..3ec5609 100644 --- a/tests/integration/test_migration_0010.py +++ b/tests/integration/test_migration_0010.py @@ -52,6 +52,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0011.py b/tests/integration/test_migration_0011.py index a36e463..eaae6e5 100644 --- a/tests/integration/test_migration_0011.py +++ b/tests/integration/test_migration_0011.py @@ -58,6 +58,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0012.py b/tests/integration/test_migration_0012.py index 8a9bdbb..f86ef89 100644 --- a/tests/integration/test_migration_0012.py +++ b/tests/integration/test_migration_0012.py @@ -68,6 +68,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0014.py b/tests/integration/test_migration_0014.py index debdf32..2cc1481 100644 --- a/tests/integration/test_migration_0014.py +++ b/tests/integration/test_migration_0014.py @@ -48,6 +48,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0016.py b/tests/integration/test_migration_0016.py index 2ae048f..cb26700 100644 --- a/tests/integration/test_migration_0016.py +++ b/tests/integration/test_migration_0016.py @@ -58,6 +58,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0017.py b/tests/integration/test_migration_0017.py index 98b012d..b602553 100644 --- a/tests/integration/test_migration_0017.py +++ b/tests/integration/test_migration_0017.py @@ -53,6 +53,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/integration/test_migration_0028.py b/tests/integration/test_migration_0028.py index e31963c..4d67cd7 100644 --- a/tests/integration/test_migration_0028.py +++ b/tests/integration/test_migration_0028.py @@ -59,6 +59,12 @@ def alembic(db: Session) -> Iterator[Config]: try: yield cfg finally: + # Release the test session's open transaction BEFORE the repair + # DDL: psycopg3 keeps its snapshot alive while idle-in-transaction, + # and 0029's CREATE INDEX CONCURRENTLY phase 3 waits for every + # older-snapshot transaction — an unreleased session would park + # the build forever. + db.rollback() command.upgrade(cfg, "head") diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index dc28d18..075f1b0 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -553,6 +553,39 @@ def test_name_hit_bonus_rejects_negative(monkeypatch: pytest.MonkeyPatch) -> Non _settings() +def test_retrieval_ef_search_default_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Phase 188 (D3, raised by task 04's live battery): the HNSW + ``ef_search`` for the cosine tier defaults to 400 — FOUR times the A7 + candidate count, because the battery measured ``ef_search = 100`` + (``== LIMIT``) recovering only mean 0.9814 / min 0.94 of the exact + top-100 on the 1,000-document live replica's real embeddings, under this + phase's own 0.99 floor — and 400 recovered 1.0000 with a max ``top_score`` + delta of exactly 0. It stays env-tunable (``BOR_RETRIEVAL_EF_SEARCH``) in + either direction, and the retriever applies it per transaction with + ``SET LOCAL``.""" + monkeypatch.delenv("BOR_RETRIEVAL_EF_SEARCH", raising=False) + assert _settings().retrieval_ef_search == 400 + monkeypatch.setenv("BOR_RETRIEVAL_EF_SEARCH", "100") + assert _settings().retrieval_ef_search == 100 + + +def test_retrieval_ef_search_refuses_zero_and_negative( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """There is no ``ef_search=0`` to switch the index off: the retriever + clamps anything below the A7 candidate count UP to it, so a ``0`` an + operator might read as "off" would silently become 100. It fails at + startup naming the field instead (phase 188, D3; the + ``show_text_max_chars`` fail-loud family — with ``0`` refused the way + ``recency_half_life_days`` refuses it).""" + for bad in ("0", "-1"): + monkeypatch.setenv("BOR_RETRIEVAL_EF_SEARCH", bad) + with pytest.raises(ValidationError, match="retrieval_ef_search"): + _settings() + + def test_recency_half_life_rejects_non_positive( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_retriever.py b/tests/unit/test_retriever.py index 03822ad..b35e9cb 100644 --- a/tests/unit/test_retriever.py +++ b/tests/unit/test_retriever.py @@ -697,6 +697,155 @@ def test_vector_candidates_default_is_summary_false_for_legacy_chunks() -> None: assert out[0].is_summary is False +# --------------------------------------------------------------------------- +# Phase 188 (task 02): the HNSW ``ef_search`` knob — SET LOCAL, clamped +# --------------------------------------------------------------------------- + + +def _one_vector_row_session() -> _FakeSession: + """A stub session whose one rowset is a single ordinary chunk.""" + doc = _doc("ef.md", "EF_BODY") + chunk = Chunk( + id=uuid.uuid4(), + document_id=doc.id, + position=0, + content="ef content", + is_summary=False, + ) + return _FakeSession([(chunk, 0.25, doc)]) + + +def _sql(session: _FakeSession) -> list[str]: + """The recorded statements as SQL text, in call order.""" + return [str(stmt) for stmt, _params in session.statements] + + +def _ef_statement(session: _FakeSession) -> str | None: + for sql in _sql(session): + if sql.startswith("SET LOCAL hnsw.ef_search"): + return sql + return None + + +def test_vector_candidates_sets_ef_search_first( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Phase 188 (D3/LOCKED A4): the tier opens with + ``SET LOCAL hnsw.ef_search = `` BEFORE the ``<=>`` SELECT (a + ``SET`` that arrived after the SELECT would tune the next turn + instead of this one), and it is LOCAL — never a session/global ``SET``. + The default (400 — the depth task 04's live battery reached exact + top-100 parity at) is what goes out when the operator set nothing.""" + settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue] + assert settings.retrieval_ef_search == 400 # the production default + monkeypatch.setattr(retriever, "get_settings", lambda: settings) + session = _one_vector_row_session() + out = _vector_candidates(session, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + assert len(out) == 1, "the candidates still come back" + statements = _sql(session) + assert statements[0] == "SET LOCAL hnsw.ef_search = 400" + assert len(statements) == 2, "exactly one SET plus the candidate SELECT" + assert "<=>" in statements[1] + + +def test_vector_candidates_clamps_ef_search_below_the_candidate_count( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """``ef_search`` BELOW the candidate count cannot be honoured — it + returns FEWER candidates than the A7 tier promises — so it is CLAMPED + up to the count with one WARNING line (the phase-188 D3 rule), and the + SELECT still asks for the full ``LIMIT``.""" + settings = Settings(_env_file=None, retrieval_ef_search=10) # pyright: ignore[reportCallIssue] + monkeypatch.setattr(retriever, "get_settings", lambda: settings) + session = _one_vector_row_session() + with caplog.at_level("WARNING", logger="app.rag.retriever"): + _vector_candidates(session, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + assert _ef_statement(session) == "SET LOCAL hnsw.ef_search = 100" + messages = [r.getMessage() for r in caplog.records] + assert "retrieval: ef_search=10 raised to 100 (A7 candidate count)" in messages + select_sql = _sql(session)[1] + # the LIMIT stays the tier's own (parameterised) count — the clamp + # raises the SEARCH depth, it never shrinks the promise + assert "LIMIT :param_1" in select_sql + + +def test_vector_candidates_ef_search_equal_to_the_limit_does_not_warn( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """The clamp is STRICTLY below: ``ef_search == limit`` (the default + 100/100 shape) is the healthy state and logs nothing.""" + settings = Settings(_env_file=None, retrieval_ef_search=100) # pyright: ignore[reportCallIssue] + monkeypatch.setattr(retriever, "get_settings", lambda: settings) + session = _one_vector_row_session() + with caplog.at_level("WARNING", logger="app.rag.retriever"): + _vector_candidates(session, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + assert _ef_statement(session) == "SET LOCAL hnsw.ef_search = 100" + assert [r.getMessage() for r in caplog.records if "ef_search" in r.getMessage()] == [] + + +def test_vector_candidates_passes_ef_search_above_the_limit( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Above the candidate count the operator's number goes through + untouched (the recall dial the phase-188 battery may raise it to) and + nothing warns.""" + settings = Settings(_env_file=None, retrieval_ef_search=200) # pyright: ignore[reportCallIssue] + monkeypatch.setattr(retriever, "get_settings", lambda: settings) + session = _one_vector_row_session() + with caplog.at_level("WARNING", logger="app.rag.retriever"): + _vector_candidates(session, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + assert _ef_statement(session) == "SET LOCAL hnsw.ef_search = 200" + assert [r.getMessage() for r in caplog.records if "ef_search" in r.getMessage()] == [] + + +def test_vector_candidates_ef_search_statement_is_int_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The interpolated fragment is ``int()``-coerced: the SET line carries + digits only — no quote, no semicolon, nothing a string could smuggle + into the statement (the value is a ``Settings`` int, but the coercion + is the belt beside the suspenders).""" + settings = Settings(_env_file=None, retrieval_ef_search=137) # pyright: ignore[reportCallIssue] + monkeypatch.setattr(retriever, "get_settings", lambda: settings) + session = _one_vector_row_session() + _vector_candidates(session, [0.0] * 768, limit=100) # pyright: ignore[reportArgumentType] + line = _ef_statement(session) + assert line is not None + value = line.split("=", 1)[1].strip() + assert value == "137" and value.isdigit() + + +def test_ef_search_below_one_refuses_startup_the_clamp_has_no_off_state() -> None: + """The clamp means ``ef_search=0`` could never mean "off" — it would be + silently rewritten to the candidate count — so ``0`` and negatives are + refused at settings construction instead (phase 188, D3; the + ``show_text_max_*`` family's fail-loud pattern).""" + from pydantic import ValidationError + + for bad in (0, -1): + with pytest.raises(ValidationError, match="retrieval_ef_search"): + Settings(_env_file=None, retrieval_ef_search=bad) # pyright: ignore[reportCallIssue] + + +def test_ef_search_default_clears_the_a7_candidate_count() -> None: + """The two numbers the clamp rule reads: the shipped default sits ABOVE + ``hybrid_vector_candidates`` (so a default install never warns), and it + is not merely equal to it. + + Equality was the task-02 default and the task-04 live battery retired it: + ``ef_search == LIMIT`` is the weakest depth an HNSW graph search can be + handed, and on the 1,000-document live replica's real embeddings it + recovered mean 0.9814 / min 0.94 of the exact top-100 — under this + phase's 0.99 floor. 400 recovered 1.0000 with every A8 verdict, top-5 set + and ``top_score`` unchanged, and it costs the tier nothing measurable + (12-14 ms against 12-13 ms at 100, on 416k chunks, versus the ~550 ms + sequential scan this index replaces). + """ + settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue] + assert settings.hybrid_vector_candidates == 100 # the A7 clamp floor + assert settings.retrieval_ef_search == 400 > settings.hybrid_vector_candidates + + def _lexical_row(is_summary: bool, doc_path: str) -> object: """One row of ``_LEXICAL_SQL`` (attribute access, as SQLAlchemy returns).""" doc = _doc(doc_path, "DOC_BODY") diff --git a/tests/unit/test_settings_catalog.py b/tests/unit/test_settings_catalog.py index 6907c0e..8f995de 100644 --- a/tests/unit/test_settings_catalog.py +++ b/tests/unit/test_settings_catalog.py @@ -73,6 +73,36 @@ def test_non_overridable_is_exactly_locked_a3() -> None: assert set(NON_OVERRIDABLE) == {"database_url", "embedding_dim"} +def test_ef_search_row_is_a_plain_live_overridable_row() -> None: + """Phase 188 (task 02, D3): the HNSW ``ef_search`` row sits in + ``Retrieval & honesty`` (beside ``hybrid_vector_candidates`` — the two + numbers the clamp rule reads together) with the ``Vector ef_search`` + label and is plain — not ``secret`` (an int), not ``restart_required`` + (``_vector_candidates`` reads ``get_settings()`` per turn, so a saved + override applies on the next question), overridable (the overlay is + generic over ``int`` fields). ``0``/negative fails startup, not the + panel. + + The default this row ships with is 400, not the A7 candidate count it + launched at: task 04's live gate measured the graph recovering the exact + top-100 for 0.9814 of rows on average (0.94 at worst) at 100 against 1.0000 + at 400, on the restored live KB with the vectors pinned + (``benchmarks/phase188_live_gate/``). It stays live-overridable in either + direction, which is what makes that a tuning decision and not a redeploy. + """ + meta = CATALOG["retrieval_ef_search"] + assert meta.group == "Retrieval & honesty" + assert meta.label == "Vector ef_search" + assert meta.secret is False + assert meta.restart_required is False + assert meta.overridable is True + assert "retrieval_ef_search" not in SECRETS + assert "retrieval_ef_search" not in RESTART_REQUIRED + assert "retrieval_ef_search" not in NON_OVERRIDABLE + assert Settings.model_fields["retrieval_ef_search"].annotation is int + assert Settings.model_fields["retrieval_ef_search"].default == 400 + + def test_restart_required_is_the_boot_wired_set() -> None: # LOCKED A4: logging, the gzip middleware (compress_min_bytes), the # static mount (static_dir), create_engine (db_pool_*), and