feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

app/config.py

@@ -8,8 +8,15 @@ from __future__ import annotations
 import os
 from functools import lru_cache
 
+from pydantic import field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+#: The A9 import formats (PLAN anchor A9, revised 2026-08-21).
+#: ``BOR_IMPORT_EXTENSIONS`` may narrow — but never widen — this set.
+_ALLOWED_IMPORT_EXTENSIONS: frozenset[str] = frozenset(
+    {"md", "markdown", "txt", "yaml", "yml", "json", "py"}
+)
+
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(
@@ -37,14 +44,58 @@ class Settings(BaseSettings):
 
     # --- RAG tuning ---
     embedding_dim: int = 768  # verified against aipi /v1 (embed model)
-    top_k_chunks: int = 4
     top_n_docs: int = 2
-    relevance_threshold: float = 0.30
+    # Honesty gate (A8, re-tuned 2026-08-21): the ``embed`` model's cosine
+    # scores compress into 0.41–0.84 on the real corpus, so the old 0.30
+    # default never discriminated. LOW only fires when best cosine < this
+    # AND no candidate chunk matches the question lexically (see A8).
+    relevance_threshold: float = 0.62
     max_context_chars: int = 24_000
     chunk_target_chars: int = 2_000
     chunk_overlap_chars: int = 200
     embed_batch_size: int = 16
 
+    # --- Hybrid retrieval (A7, revised 2026-08-21) ---
+    # cosine top-N ∪ Postgres FTS top-N, fused with Reciprocal Rank Fusion
+    # (score = Σ 1/(rrf_k + rank) over the lists a chunk appears in).
+    #
+    # The vector window is deliberately wider than the lexical one: a
+    # name-your-tool question's best *lexical* chunk (e.g. the "Install"
+    # section of gitlab.md) can sit far down the vector ranking because the
+    # question embeds close to generic templates. A 100-wide window is what
+    # lets such chunks double-hit (one RRF term per list) and outrank a
+    # template that owns vector rank 1 — measured 2026-08-22 against the
+    # live 2774-chunk KB for "How did I install gitlab?" (gitlab.md:1 at
+    # vrank 100 / lrank 3 → fused 0.0221 vs the template's 0.0164).
+    hybrid_vector_candidates: int = 100
+    hybrid_lexical_candidates: int = 30
+    rrf_k: int = 60
+
+    # --- Import scope (A9, revised 2026-08-21) ---
+    # Comma-separated list of lowercased file extensions (no dot) imported
+    # by ``scripts/import_docs.py``. Hidden (dot) path components are always
+    # skipped, plus the importer's exclusion list.
+    # Stored as a raw CSV string (env-native — no JSON) and parsed on demand
+    # via :py:meth:`import_extension_set`. ``mode="after"`` validation runs
+    # against the raw string so a typo fails loudly at startup.
+    import_extensions: str = "md,markdown,txt,yaml,yml,json,py"
+
+    @field_validator("import_extensions")
+    @classmethod
+    def _import_extensions_known(cls, v: str) -> str:
+        """Reject unknown/empty formats loudly instead of silently importing
+        nothing (a typo like ``md,jsonn`` would otherwise walk zero files)."""
+        exts = {part.strip().lstrip(".").lower() for part in v.split(",") if part.strip()}
+        if not exts:
+            raise ValueError("import_extensions must name at least one format")
+        unknown = exts - _ALLOWED_IMPORT_EXTENSIONS
+        if unknown:
+            raise ValueError(
+                f"unknown import extension(s): {', '.join(sorted(unknown))} — "
+                f"allowed: {', '.join(sorted(_ALLOWED_IMPORT_EXTENSIONS))}"
+            )
+        return v
+
     # Suggested questions (onboarding + empty state).
     suggestions: list[str] = [
         "How is my Kubernetes cluster set up?",
@@ -53,6 +104,15 @@ class Settings(BaseSettings):
         "What's currently running in the homelab?",
     ]
 
+    @property
+    def import_extension_set(self) -> frozenset[str]:
+        """Lowercased, dotted extension set (``.md``) for path filtering."""
+        return frozenset(
+            f".{part.strip().lstrip('.').lower()}"
+            for part in self.import_extensions.split(",")
+            if part.strip()
+        )
+
     @property
     def effective_api_key(self) -> str:
         """API key for aipi: explicit setting, then $AIPI_KEY, then a placeholder."""