feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

2026-08-22 01:27:02 -04:00
parent 2f738a7f19
commit 7e8d14702e
36 changed files with 2018 additions and 290 deletions
@@ -0,0 +1,40 @@
+"""hybrid retrieval: generated FTS column on chunks + query_log.fts_hits
+
+Revision ID: 0002
+Revises: 0001
+Create Date: 2026-08-21
+
+A7/A8 (revised 2026-08-21, owner permission): retrieval becomes hybrid
+(cosine top-N + Postgres full-text top-N, RRF-fused). This adds:
+
+* ``chunks.tsv`` — generated ``TSVECTOR`` (``to_tsvector('english',
+  content) STORED``) + GIN index for the lexical candidate list.
+* ``query_log.fts_hits`` — INT, nullable (pre-existing rows stay NULL:
+  the column only carries meaning from hybrid retrieval onward).
+"""
+from __future__ import annotations
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "0002"
+down_revision = "0001"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TABLE chunks "
+        "ADD COLUMN tsv tsvector "
+        "GENERATED ALWAYS AS (to_tsvector('english', content)) STORED"
+    )
+    op.execute("CREATE INDEX ix_chunks_tsv ON chunks USING gin (tsv)")
+    op.add_column("query_log", sa.Column("fts_hits", sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("query_log", "fts_hits")
+    op.execute("DROP INDEX IF EXISTS ix_chunks_tsv")
+    op.execute("ALTER TABLE chunks DROP COLUMN IF EXISTS tsv")