phase: 97_kb_tree_catalog
All completion criteria verified — everything is green, no defects found. Final report: ## Phase 97 final verification pass — ALL GREEN **Verified (no code changes needed):** - `GET /api/docs/tree` (admin), `build_kb_tree` pure builder, `PATCH /api/folders/summary`, migration 0018 (`manually_edited`, head confirmed), generator skip/keep + `kept_manual` stat, RAG tree UI + edit affordance in `sources.js`/`index.html`/`styles.css` - `tests/e2e/test_kb_tree.py`: 8 passed — top level, drill source/folder, edit round-trip, clear, manual-desc-survives-sync, reload fallback, anonymous gate - Integration: tree shape/order/403/empty/indexed-only + PATCH update/create/root/clear/404/403/no-LLM + stat-walk equivalence (in `test_docs_api.py`); 3-field `folder_summaries=` import token preserved **Gates (exact commands):** - `uv run pytest --cov=app --cov-report=term-missing` → **2053 passed**, TOTAL coverage **99%** (>90% ✓) - `uv run ruff check . && uv run pyright` → **All checks passed / 0 errors** - `uv run pytest tests/e2e/test_kb_tree.py -v --no-cov` → **8 passed** in isolation - 30 story/RAG-view E2E suites run **one per process**: all passed, incl. `test_ls_tree_drilldown` (agent `ls` byte-identical ✓), `test_import_documents`, `test_edit_summaries`, `test_admin_auth`, `test_kb_overview` **Completion criteria:** tree view ✓ · edit round-trip + clear ✓ · manual persists/clear resets ✓ · `ls` unchanged ✓ · pytest/coverage/lint ✓ · E2E isolation ✓ · commit — left to harness per protocol (working tree untouched, `git add/commit` not run) **Deviations:** none. **Next pending phase:** none — `todo/` contains only 97 (96 already committed).
This commit is contained in:
@@ -1,15 +1,68 @@
|
||||
"""Integration tests: GET /api/docs — empty shape + populated shape.
|
||||
"""Integration tests: GET /api/docs — empty shape + populated shape —
|
||||
and GET /api/docs/tree (phase 97, task 02 — the full recursive KB tree
|
||||
the RAG view renders: shape, ordering, counts, summaries, the 403 gate,
|
||||
the superset rule, and the stat-walk equivalence with ``GET /api/docs``).
|
||||
|
||||
Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import itertools
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import text
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import func, select, text
|
||||
|
||||
from app.models import Chunk, Document
|
||||
import app.api.docs as docs_api
|
||||
from app.config import Settings
|
||||
from app.core import tokens as token_service
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Chunk, Document, FolderSummary, GitSource
|
||||
from app.rag import git_sources as rag_git_sources
|
||||
|
||||
_TREE_TABLES = "chunks, documents, folder_summaries, git_sources"
|
||||
|
||||
|
||||
def _truncate_tree_tables(db) -> None:
|
||||
db.execute(text(f"TRUNCATE {_TREE_TABLES}"))
|
||||
db.commit()
|
||||
|
||||
|
||||
_counter = itertools.count()
|
||||
|
||||
|
||||
def _seed_doc(db, source: str, path: str, title: str, n_chunks: int, indexed_at: datetime) -> None:
|
||||
"""One indexed document with *n_chunks* content chunks (unique hash)."""
|
||||
doc = Document(
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/tmp/{source}/{path}",
|
||||
title=title,
|
||||
content=f"# {title}\n\nBody.",
|
||||
content_hash=f"{next(_counter):064d}",
|
||||
indexed_at=indexed_at,
|
||||
)
|
||||
db.add(doc)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
Chunk(document_id=doc.id, position=i, content=f"chunk {i}", embedding=[0.01] * 768)
|
||||
for i in range(n_chunks)
|
||||
)
|
||||
|
||||
|
||||
def _tree_file_nodes(sources) -> list[dict]:
|
||||
"""Every file node of a tree response, walked recursively."""
|
||||
files: list[dict] = []
|
||||
for source in sources:
|
||||
for child in source["children"]:
|
||||
if child["kind"] == "file":
|
||||
files.append(child)
|
||||
else:
|
||||
files.extend(_tree_file_nodes([child]))
|
||||
return files
|
||||
|
||||
|
||||
def test_docs_empty_shape(admin_client, db) -> None:
|
||||
@@ -77,3 +130,521 @@ def test_docs_response_matches_schema_shape(admin_client, db) -> None:
|
||||
for d in body["documents"]:
|
||||
assert set(d) == {"id", "source", "path", "title", "chunks", "indexed_at"}
|
||||
assert isinstance(d["chunks"], int) and d["chunks"] >= 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# GET /api/docs/tree (phase 97, task 02).
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_docs_tree_populated_shape_order_counts_summaries(admin_client, db) -> None:
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
# Registry order (added_at) is Homelab → Deployments — deliberately
|
||||
# NOT alphabetical (the registry order leads, the phase-97 rule).
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
db.add(
|
||||
GitSource(
|
||||
url="https://github.com/reese/Deployments.git",
|
||||
kind="git",
|
||||
added_at=base + timedelta(hours=1),
|
||||
)
|
||||
)
|
||||
_seed_doc(db, "Homelab", "k8s/talos.md", "Talos", 3, base)
|
||||
_seed_doc(db, "Homelab", "k8s/cluster.md", "Cluster", 2, base)
|
||||
_seed_doc(db, "Homelab", "k8s/helm/charts.md", "Charts", 1, base)
|
||||
_seed_doc(db, "Homelab", "root-note.md", "Root note", 4, base)
|
||||
_seed_doc(db, "Deployments", "deploy-a.md", "Deploy A", 1, base)
|
||||
_seed_doc(db, "Deployments", "deploy-b.md", "Deploy B", 0, base)
|
||||
db.add_all(
|
||||
[
|
||||
FolderSummary(source="Homelab", folder_path="", summary="Homelab docs."),
|
||||
FolderSummary(source="Homelab", folder_path="k8s", summary="K8s stuff."),
|
||||
# A stored row for a folder with NO indexed descendants —
|
||||
# e.g. a manual row surviving the prune below the 2-doc
|
||||
# minimum (phase 97, task 01): the tree's folders come from
|
||||
# INDEXED paths (the existence rule), not from summary rows.
|
||||
FolderSummary(source="Deployments", folder_path="orphan", summary="Ghost."),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert set(body) == {"sources"}
|
||||
sources = body["sources"]
|
||||
assert [s["name"] for s in sources] == ["Homelab", "Deployments"]
|
||||
|
||||
homelab, deployments = sources
|
||||
assert set(homelab) == {"name", "documents", "summary", "children"}
|
||||
assert homelab["documents"] == 4 # the whole recursive count
|
||||
assert homelab["summary"] == "Homelab docs." # the (source, "") row
|
||||
assert deployments["summary"] is None # no stored root row
|
||||
assert deployments["documents"] == 2
|
||||
|
||||
# Homelab: subfolder first (path order among folders), then the
|
||||
# root files; the nested folder k8s/helm recurses one level deeper.
|
||||
k8s, root_note = homelab["children"]
|
||||
assert k8s["kind"] == "folder"
|
||||
assert k8s["path"] == "k8s"
|
||||
assert k8s["documents"] == 3 # talos + cluster + charts (subtree)
|
||||
assert k8s["summary"] == "K8s stuff."
|
||||
helm, cluster, talos = k8s["children"]
|
||||
assert (helm["kind"], helm["path"], helm["documents"], helm["summary"]) == (
|
||||
"folder",
|
||||
"k8s/helm",
|
||||
1,
|
||||
None,
|
||||
)
|
||||
assert [
|
||||
(c["kind"], c["path"], c["title"], c["chunks"])
|
||||
for c in (cluster, talos)
|
||||
] == [("file", "k8s/cluster.md", "Cluster", 2), ("file", "k8s/talos.md", "Talos", 3)]
|
||||
chart = helm["children"][0]
|
||||
assert (chart["kind"], chart["path"], chart["title"], chart["chunks"]) == (
|
||||
"file",
|
||||
"k8s/helm/charts.md",
|
||||
"Charts",
|
||||
1,
|
||||
)
|
||||
datetime.fromisoformat(chart["indexed_at"]) # valid ISO-8601
|
||||
assert (root_note["kind"], root_note["path"], root_note["title"], root_note["chunks"]) == (
|
||||
"file",
|
||||
"root-note.md",
|
||||
"Root note",
|
||||
4,
|
||||
)
|
||||
|
||||
# Deployments: flat — direct files in catalog order, and the
|
||||
# ``orphan`` summary row does NOT create a folder node.
|
||||
assert [(c["kind"], c["path"]) for c in deployments["children"]] == [
|
||||
("file", "deploy-a.md"),
|
||||
("file", "deploy-b.md"),
|
||||
]
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_docs_tree_403_anonymous(client, db) -> None:
|
||||
"""Admin-only, like ``GET /api/docs``: anonymous → 403
|
||||
``admin only`` (the RAG view's anonymous gate never fetches the
|
||||
tree)."""
|
||||
_truncate_tree_tables(db)
|
||||
r = client.get("/api/docs/tree")
|
||||
assert r.status_code == 403
|
||||
assert r.json() == {"detail": "admin only"}
|
||||
|
||||
|
||||
def test_docs_tree_empty_registry_and_catalog(
|
||||
admin_client, db, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Nothing registered, nothing indexed → ``{"sources": []}``.
|
||||
|
||||
"Empty registry" means the ``git_sources`` table AND the
|
||||
``BOR_GIT_SOURCES`` env fallback are both empty — the dev ``.env``
|
||||
names a source, so the resolver's ``get_settings`` is patched with
|
||||
a fresh ``Settings(_env_file=None, git_sources="")`` (the
|
||||
``test_sync_api`` / ``test_git_sources_api`` pattern: the dev
|
||||
``.env`` never leaks in)."""
|
||||
monkeypatch.setattr(
|
||||
rag_git_sources,
|
||||
"get_settings",
|
||||
lambda: Settings(_env_file=None, git_sources=""), # pyright: ignore[reportCallIssue]
|
||||
)
|
||||
_truncate_tree_tables(db)
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"sources": []}
|
||||
|
||||
|
||||
def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> None:
|
||||
"""The superset rule: a registered 0-document source still lists
|
||||
(first — the registry leads), and indexed-only sources (documents
|
||||
whose source is not in ``git_sources`` — ad-hoc imports, removed
|
||||
but not-yet-pruned sources) trail in alphabetical order."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Alpha.git", kind="git", added_at=base))
|
||||
_seed_doc(db, "Zeta", "z1.md", "Z1", 1, base)
|
||||
_seed_doc(db, "Midx", "m1.md", "M1", 2, base)
|
||||
db.commit()
|
||||
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
sources = r.json()["sources"]
|
||||
assert [s["name"] for s in sources] == ["Alpha", "Midx", "Zeta"]
|
||||
alpha, midx, zeta = sources
|
||||
assert (alpha["documents"], alpha["children"], alpha["summary"]) == (0, [], None)
|
||||
assert midx["documents"] == 1
|
||||
assert zeta["documents"] == 1
|
||||
assert [c["path"] for c in midx["children"]] == ["m1.md"]
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# PATCH /api/folders/summary (phase 97, task 03) — the admin
|
||||
# folder-description editor: update / create / source-root / clear /
|
||||
# double-clear, the 404s, the 403s, and the round-trip through
|
||||
# ``GET /api/docs/tree`` (task 02).
|
||||
# --------------------------------------------------------------------
|
||||
|
||||
#: A deliberately old stamp: a save must ADVANCE ``updated_at`` past it.
|
||||
OLDER_STAMP = datetime(2020, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def _seed_folder_pair(db, source: str, folder: str, base: datetime) -> None:
|
||||
"""Two documents under ``source/folder`` — the ≥ 2-document
|
||||
minimum a folder must hold to be summarizable."""
|
||||
_seed_doc(db, source, f"{folder}/one.md", f"{folder} one", 1, base)
|
||||
_seed_doc(db, source, f"{folder}/two.md", f"{folder} two", 2, base)
|
||||
|
||||
|
||||
def _get_folder_row(db, source: str, folder: str) -> FolderSummary | None:
|
||||
return db.scalar(
|
||||
select(FolderSummary).where(
|
||||
FolderSummary.source == source, FolderSummary.folder_path == folder
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_folder_summary_update_ai_row_flips_manual_and_advances_stamp(
|
||||
admin_client: TestClient, db
|
||||
) -> None:
|
||||
"""Update an AI-written row: the text is replaced, ``manually_edited``
|
||||
flips to true, ``updated_at`` advances past the seeded stamp — and a
|
||||
second save with different text updates IN PLACE (same PK, one
|
||||
row). The round-trip: ``GET /api/docs/tree`` shows the new text on
|
||||
the folder node."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_folder_pair(db, "Homelab", "k8s", base)
|
||||
db.add(
|
||||
FolderSummary(
|
||||
source="Homelab",
|
||||
folder_path="k8s",
|
||||
summary="AI text.",
|
||||
manually_edited=False,
|
||||
updated_at=OLDER_STAMP,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Homelab", "folder_path": "k8s", "summary": " Owner words. "},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert set(r.json()) == {"source", "folder_path", "summary"}
|
||||
assert r.json() == {"source": "Homelab", "folder_path": "k8s", "summary": "Owner words."}
|
||||
|
||||
db.expire_all()
|
||||
row = _get_folder_row(db, "Homelab", "k8s")
|
||||
assert row is not None
|
||||
assert row.summary == "Owner words." # stripped before storing
|
||||
assert row.manually_edited is True # a manual row from this save on
|
||||
assert row.updated_at > OLDER_STAMP # fresh stamp, not the seeded one
|
||||
|
||||
# The task-02 tree shows the owner's text on the folder node.
|
||||
tree = admin_client.get("/api/docs/tree").json()
|
||||
k8s = next(c for c in tree["sources"][0]["children"] if c["kind"] == "folder")
|
||||
assert (k8s["path"], k8s["summary"]) == ("k8s", "Owner words.")
|
||||
|
||||
# A second save updates in place — same row, no second PK.
|
||||
r2 = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Homelab", "folder_path": "k8s", "summary": "Owner words v2."},
|
||||
)
|
||||
assert r2.status_code == 200, r2.text
|
||||
assert r2.json()["summary"] == "Owner words v2."
|
||||
db.expire_all()
|
||||
assert db.scalar(select(func.count()).select_from(FolderSummary)) == 1 # one row
|
||||
row = _get_folder_row(db, "Homelab", "k8s")
|
||||
assert row is not None
|
||||
assert row.summary == "Owner words v2."
|
||||
assert row.manually_edited is True
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_folder_summary_create_where_no_row_exists(
|
||||
admin_client: TestClient, db
|
||||
) -> None:
|
||||
"""A manual description can be CREATED where no row exists — a
|
||||
< 2-document folder the generator never wrote (or its fail-soft
|
||||
miss): insert, not update, with ``manually_edited = true``."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_doc(db, "Homelab", "solo/only.md", "Only", 1, base) # 1-doc folder
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
assert _get_folder_row(db, "Homelab", "solo") is None # no AI row for < 2 docs
|
||||
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Homelab", "folder_path": "solo", "summary": "One-off scripts."},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json() == {
|
||||
"source": "Homelab",
|
||||
"folder_path": "solo",
|
||||
"summary": "One-off scripts.",
|
||||
}
|
||||
db.expire_all()
|
||||
row = _get_folder_row(db, "Homelab", "solo")
|
||||
assert row is not None
|
||||
assert row.summary == "One-off scripts."
|
||||
assert row.manually_edited is True
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_folder_summary_root_save_on_registered_zero_doc_source(
|
||||
admin_client: TestClient, db
|
||||
) -> None:
|
||||
"""``folder_path = ""`` is valid for ANY allowed source — a registered
|
||||
0-document source (nothing indexed yet) still accepts a root
|
||||
description: the source check is registered-OR-indexed."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Empty.git", kind="git", added_at=base))
|
||||
db.commit()
|
||||
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Empty", "folder_path": "", "summary": "Docs incoming."},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json() == {"source": "Empty", "folder_path": "", "summary": "Docs incoming."}
|
||||
db.expire_all()
|
||||
row = _get_folder_row(db, "Empty", "")
|
||||
assert row is not None
|
||||
assert row.manually_edited is True
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_folder_summary_source_root_round_trips_through_tree(
|
||||
admin_client: TestClient, db
|
||||
) -> None:
|
||||
"""The source root (``folder_path: ""``) round-trips through the
|
||||
task-02 tree endpoint: save → the source node carries the text;
|
||||
clear → it is null again."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_doc(db, "Homelab", "a/b.md", "B", 1, base)
|
||||
_seed_doc(db, "Homelab", "c.md", "C", 1, base)
|
||||
db.commit()
|
||||
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Homelab", "folder_path": "", "summary": "All the homelab docs."},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json() == {
|
||||
"source": "Homelab",
|
||||
"folder_path": "",
|
||||
"summary": "All the homelab docs.",
|
||||
}
|
||||
tree = admin_client.get("/api/docs/tree").json()
|
||||
assert tree["sources"][0]["summary"] == "All the homelab docs."
|
||||
db.expire_all()
|
||||
assert _get_folder_row(db, "Homelab", "") is not None
|
||||
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Homelab", "folder_path": "", "summary": " "},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json() == {"source": "Homelab", "folder_path": "", "summary": None}
|
||||
tree = admin_client.get("/api/docs/tree").json()
|
||||
assert tree["sources"][0]["summary"] is None
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_folder_summary_clear_deletes_row_and_double_clear_is_noop(
|
||||
admin_client: TestClient, db
|
||||
) -> None:
|
||||
"""Empty/whitespace clears: the row is deleted — AI-written OR
|
||||
manual, either way it is gone (the next KB-changing sync regenerates
|
||||
an AI row: the reset path) — and the response ``summary`` is null.
|
||||
A second clear with no row is a 200 no-op. The task-02 tree shows
|
||||
null on both folder nodes after the clears."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_folder_pair(db, "Homelab", "k8s", base)
|
||||
_seed_folder_pair(db, "Homelab", "manual", base)
|
||||
db.add(
|
||||
FolderSummary(
|
||||
source="Homelab",
|
||||
folder_path="k8s",
|
||||
summary="AI text.",
|
||||
manually_edited=False,
|
||||
updated_at=OLDER_STAMP,
|
||||
)
|
||||
)
|
||||
db.add(
|
||||
FolderSummary(
|
||||
source="Homelab",
|
||||
folder_path="manual",
|
||||
summary="Owner text.",
|
||||
manually_edited=True,
|
||||
updated_at=OLDER_STAMP,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
|
||||
for folder, blank in (("k8s", " "), ("manual", "")): # whitespace, then empty
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Homelab", "folder_path": folder, "summary": blank},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json() == {"source": "Homelab", "folder_path": folder, "summary": None}
|
||||
db.expire_all()
|
||||
assert db.scalar(select(func.count()).select_from(FolderSummary)) == 0 # both rows gone
|
||||
|
||||
# A second clear (no row) is a 200 no-op.
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": "Homelab", "folder_path": "k8s", "summary": ""},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json() == {"source": "Homelab", "folder_path": "k8s", "summary": None}
|
||||
db.expire_all()
|
||||
assert db.scalar(select(func.count()).select_from(FolderSummary)) == 0 # still nothing
|
||||
|
||||
# The task-02 tree shows null on both folder nodes after the clears.
|
||||
tree = admin_client.get("/api/docs/tree").json()
|
||||
nodes = {
|
||||
c["path"]: c for c in tree["sources"][0]["children"] if c["kind"] == "folder"
|
||||
}
|
||||
assert nodes["k8s"]["summary"] is None
|
||||
assert nodes["manual"]["summary"] is None
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_folder_summary_404_unknown_source_folder_and_traversal(
|
||||
admin_client: TestClient, db
|
||||
) -> None:
|
||||
"""404s, in check order: an unknown source (neither registered nor
|
||||
indexed) wins over the folder check — including for a folder that
|
||||
EXISTS under another source; an unknown folder, a traversal folder
|
||||
path, and a file merely sharing a folder's name (no indexed
|
||||
descendant) are all ``folder not found``. Nothing is written."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_folder_pair(db, "Homelab", "real", base)
|
||||
_seed_doc(db, "Homelab", "note", "A file named like a folder", 1, base)
|
||||
db.commit()
|
||||
|
||||
cases = (
|
||||
# (source, folder_path, detail)
|
||||
("Ghost", "", "source not found"), # unknown source, root
|
||||
("Ghost", "real", "source not found"), # folder exists ELSEWHERE — source wins
|
||||
("Homelab", "nope", "folder not found"), # unknown folder
|
||||
("Homelab", "../../etc", "folder not found"), # traversal: no prefix match
|
||||
("Homelab", "note", "folder not found"), # file named like a folder, no descendants
|
||||
("Homelab", "real/", "folder not found"), # trailing slash: strict prefix rule
|
||||
)
|
||||
for source, folder, detail in cases:
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
json={"source": source, "folder_path": folder, "summary": "whatever"},
|
||||
)
|
||||
assert r.status_code == 404, (source, folder, r.status_code)
|
||||
assert r.json() == {"detail": detail}, (source, folder)
|
||||
|
||||
db.expire_all()
|
||||
assert db.scalar(select(func.count()).select_from(FolderSummary)) == 0 # nothing written
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_folder_summary_403_anonymous_and_token_user(
|
||||
client: TestClient, db
|
||||
) -> None:
|
||||
"""The ``require_admin`` gate: an anonymous caller AND a live
|
||||
access-token user who is not the admin (phase 79 token users exist)
|
||||
both get 403 ``admin only`` — the endpoint gate is the API-level
|
||||
defense in depth, not the RAG view's render gate."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_folder_pair(db, "Homelab", "k8s", base)
|
||||
db.execute(text("TRUNCATE api_tokens"))
|
||||
db.commit()
|
||||
body = {"source": "Homelab", "folder_path": "k8s", "summary": "x"}
|
||||
try:
|
||||
# Anonymous (the shared ``client`` is unsigned in this module).
|
||||
r = client.patch("/api/folders/summary", json=body)
|
||||
assert r.status_code == 403
|
||||
assert r.json() == {"detail": "admin only"}
|
||||
|
||||
# A live access-token user who is not the admin (phase 79):
|
||||
# signed in via the public token login, the row is active.
|
||||
_row, plaintext = token_service.create_token(db, "pin-holder")
|
||||
db.commit()
|
||||
holder = TestClient(fastapi_app)
|
||||
s = holder.post("/api/token-auth", json={"token": plaintext})
|
||||
assert s.status_code == 204, s.text
|
||||
r = holder.patch("/api/folders/summary", json=body)
|
||||
assert r.status_code == 403
|
||||
assert r.json() == {"detail": "admin only"}
|
||||
finally:
|
||||
db.execute(text("TRUNCATE api_tokens"))
|
||||
db.commit()
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_folder_summary_patch_never_constructs_an_llm_client() -> None:
|
||||
"""Source pin (the house pattern): a folder description is NEVER
|
||||
embedded — no chunk, no retrieval role beyond the ``ls`` line — so
|
||||
the PATCH handler must never touch the LLM client (the deliberate
|
||||
contrast with the phase-57 document-summary re-embed)."""
|
||||
src = inspect.getsource(docs_api.update_folder_summary)
|
||||
assert "LLMClient" not in src
|
||||
|
||||
|
||||
def test_docs_tree_stat_walk_equivalence_with_flat_list(admin_client, db) -> None:
|
||||
"""The RAG view computes its KB-wide stat cards by walking the
|
||||
in-memory tree — the walk must yield EXACTLY what
|
||||
``GET /api/docs`` reports for the same data (the stat-card values
|
||||
are unchanged by the redesign)."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_doc(db, "Homelab", "a/b/c.md", "C", 3, base)
|
||||
_seed_doc(db, "Homelab", "a/d.md", "D", 2, base)
|
||||
_seed_doc(db, "Homelab", "top.md", "Top", 5, base)
|
||||
_seed_doc(db, "Homelab", "solo.md", "Solo", 0, base)
|
||||
db.commit()
|
||||
|
||||
tree = admin_client.get("/api/docs/tree").json()
|
||||
flat = admin_client.get("/api/docs").json()
|
||||
|
||||
files = _tree_file_nodes(tree["sources"])
|
||||
assert len(files) == len(flat["documents"]) # document count
|
||||
assert sum(f["chunks"] for f in files) == sum(d["chunks"] for d in flat["documents"])
|
||||
# The tree's per-source counts match the flat list's per-source counts.
|
||||
for source in tree["sources"]:
|
||||
flat_count = sum(
|
||||
1 for d in flat["documents"] if d["source"] == source["name"]
|
||||
)
|
||||
assert source["documents"] == flat_count
|
||||
source_files = _tree_file_nodes([source])
|
||||
assert sum(f["chunks"] for f in source_files) == sum(
|
||||
d["chunks"] for d in flat["documents"] if d["source"] == source["name"]
|
||||
)
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Integration: migration 0018 (folder_summaries.manually_edited)
|
||||
schema contract (phase 97, task 01).
|
||||
|
||||
Drives the **real Alembic engine** against the live dev database
|
||||
(``podman compose up -d db``), mirroring the house pattern of
|
||||
``test_migration_0017.py`` (information_schema assertions on the state
|
||||
the migration must leave). The tests target the 0017 → 0018 step
|
||||
explicitly so later migrations cannot break them:
|
||||
|
||||
* upgrade 0017 → 0018 → the ``manually_edited`` column exists with the
|
||||
full contract — BOOLEAN NOT NULL, server default ``false`` — while
|
||||
the 0017 ``folder_summaries`` schema (PK, summary, updated_at)
|
||||
survives;
|
||||
* pre-0018 rows backfill ``false`` (an AI-written row stays an AI row)
|
||||
and a row written without the column takes the default;
|
||||
* a row written with ``manually_edited = true`` round-trips the flag;
|
||||
* downgrade to 0017 → the column is GONE (A13 — reversible) while the
|
||||
rows + their summaries survive;
|
||||
* upgrade back to 0018 → the column is back (round-trip).
|
||||
|
||||
The ``alembic`` fixture guarantees the DB ends at head even if a test
|
||||
fails or the process is interrupted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from alembic import command
|
||||
from app.db import db_available
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def alembic(db: Session) -> Iterator[Config]:
|
||||
"""Real Alembic config bound to the dev DB (URL from app settings).
|
||||
|
||||
Starts at head (repairs an interrupted earlier run); teardown upgrades
|
||||
to head no matter what happened, so the dev DB is never left below
|
||||
head.
|
||||
"""
|
||||
if not db_available():
|
||||
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
|
||||
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
|
||||
cfg.set_main_option("script_location", "alembic")
|
||||
command.upgrade(cfg, "head")
|
||||
try:
|
||||
yield cfg
|
||||
finally:
|
||||
# Release the test session's open transaction BEFORE the repair
|
||||
# DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
|
||||
# on ``folder_summaries``, which would deadlock the repair's
|
||||
# ``ALTER TABLE`` (0018) forever.
|
||||
db.rollback()
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
|
||||
def _version(db: Session) -> str | None:
|
||||
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
|
||||
|
||||
|
||||
def _column(db: Session, table: str, column: str) -> tuple[Any, ...] | None:
|
||||
"""(data_type, is_nullable, column_default, character_maximum_length)
|
||||
for one table column."""
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT data_type, is_nullable, column_default, character_maximum_length"
|
||||
" FROM information_schema.columns"
|
||||
" WHERE table_name = :t AND column_name = :c"
|
||||
),
|
||||
{"t": table, "c": column},
|
||||
).fetchone()
|
||||
return tuple(row) if row is not None else None
|
||||
|
||||
|
||||
def _flag(db: Session, source: str, folder_path: str) -> Any:
|
||||
return db.execute(
|
||||
text(
|
||||
"SELECT manually_edited FROM folder_summaries"
|
||||
" WHERE source = :s AND folder_path = :f"
|
||||
),
|
||||
{"s": source, "f": folder_path},
|
||||
).scalar_one()
|
||||
|
||||
|
||||
def _clear_rows(db: Session) -> None:
|
||||
db.execute(text("DELETE FROM folder_summaries"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_upgrade_to_0018_adds_manually_edited(db: Session, alembic: Config) -> None:
|
||||
"""Upgrade 0017 → 0018: the column exists with the full contract
|
||||
(BOOLEAN NOT NULL, server default ``false``), is ABSENT at 0017,
|
||||
pre-0018 rows backfill ``false`` (an AI row stays an AI row), a new
|
||||
row without the column takes the default, and an explicit ``true``
|
||||
round-trips — while the 0017 table contract survives."""
|
||||
command.downgrade(alembic, "0017") # start from the pre-0018 state
|
||||
assert _version(db) == "0017"
|
||||
assert _column(db, "folder_summaries", "manually_edited") is None, (
|
||||
"the flag must be absent at 0017"
|
||||
)
|
||||
|
||||
try:
|
||||
# A pre-0018 AI-written row — must backfill ``false``.
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO folder_summaries (source, folder_path, summary)"
|
||||
" VALUES ('OldSource', 'old/folder', 'pre-0018 summary')"
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
command.upgrade(alembic, "0018")
|
||||
assert _version(db) == "0018", "alembic_version must be at 0018"
|
||||
|
||||
flag = _column(db, "folder_summaries", "manually_edited")
|
||||
assert flag is not None, "folder_summaries.manually_edited is missing"
|
||||
assert flag[0] == "boolean", "manually_edited must be BOOLEAN"
|
||||
assert flag[1] == "NO", "manually_edited must be NOT NULL"
|
||||
assert flag[2] is not None and "false" in str(flag[2]), (
|
||||
"manually_edited must carry the `false` server default"
|
||||
)
|
||||
|
||||
# The pre-0018 row backfilled ``false`` — an AI row stays an AI row.
|
||||
assert _flag(db, "OldSource", "old/folder") is False
|
||||
|
||||
# A row written without the column takes the server default.
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO folder_summaries (source, folder_path, summary)"
|
||||
" VALUES ('NewSource', '', 'root summary')"
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
assert _flag(db, "NewSource", "") is False, (
|
||||
"an omitted flag takes the `false` server default"
|
||||
)
|
||||
|
||||
# The flag round-trips through an explicit ``true``.
|
||||
db.execute(
|
||||
text(
|
||||
"UPDATE folder_summaries SET manually_edited = true"
|
||||
" WHERE source = 'NewSource'"
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
assert _flag(db, "NewSource", "") is True, (
|
||||
"manually_edited = true must round-trip"
|
||||
)
|
||||
|
||||
# The 0017 schema survives the additive upgrade.
|
||||
summary = _column(db, "folder_summaries", "summary")
|
||||
assert summary is not None and summary[0] == "text" and summary[1] == "NO", (
|
||||
"folder_summaries.summary (0017) must survive the upgrade"
|
||||
)
|
||||
folder = _column(db, "folder_summaries", "folder_path")
|
||||
assert folder is not None and folder[3] == 1000, (
|
||||
"folder_summaries.folder_path (0017) must survive the upgrade"
|
||||
)
|
||||
finally:
|
||||
_clear_rows(db)
|
||||
|
||||
|
||||
def test_downgrade_to_0017_drops_the_column(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade 0018 → 0017: the column is gone (A13 — fully
|
||||
reversible) while the rows + their summaries survive, and the rest
|
||||
of the schema (the 0017 table contract, ``documents``) is intact."""
|
||||
command.upgrade(alembic, "head")
|
||||
try:
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO folder_summaries"
|
||||
" (source, folder_path, summary, manually_edited)"
|
||||
" VALUES ('ManualSrc', '', 'owner text', true)"
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
command.downgrade(alembic, "0017")
|
||||
assert _version(db) == "0017"
|
||||
assert _column(db, "folder_summaries", "manually_edited") is None, (
|
||||
"the flag must be dropped"
|
||||
)
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT source, folder_path, summary FROM folder_summaries"
|
||||
" WHERE source = 'ManualSrc'"
|
||||
)
|
||||
).fetchone()
|
||||
assert row is not None and row[2] == "owner text", (
|
||||
"the row and its summary must survive the column drop"
|
||||
)
|
||||
|
||||
summary = _column(db, "folder_summaries", "summary")
|
||||
assert summary is not None and summary[0] == "text", (
|
||||
"the 0017 table contract must survive the downgrade"
|
||||
)
|
||||
doc_path = _column(db, "documents", "path")
|
||||
assert doc_path is not None and doc_path[3] == 1000, (
|
||||
"documents.path must survive the downgrade"
|
||||
)
|
||||
finally:
|
||||
_clear_rows(db)
|
||||
# Repair: the fixture teardown re-upgrades to head.
|
||||
|
||||
|
||||
def test_upgrade_round_trip_restores_the_flag(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade to 0017, then upgrade back to 0018: the column is back
|
||||
with the full contract (BOOLEAN NOT NULL, the `false` default)."""
|
||||
command.downgrade(alembic, "0017")
|
||||
command.upgrade(alembic, "0018")
|
||||
assert _version(db) == "0018", "round-trip upgrade must land at 0018"
|
||||
|
||||
flag = _column(db, "folder_summaries", "manually_edited")
|
||||
assert flag is not None, "folder_summaries.manually_edited must be back"
|
||||
assert flag[0] == "boolean", "manually_edited must be BOOLEAN after the round-trip"
|
||||
assert flag[1] == "NO", "manually_edited must be NOT NULL after the round-trip"
|
||||
assert flag[2] is not None and "false" in str(flag[2]), (
|
||||
"the `false` server default must survive the round-trip"
|
||||
)
|
||||
@@ -275,7 +275,8 @@ def test_changed_import_generates_folder_rows(
|
||||
assert _updated_at(db, "MyDocs", "a") is not None
|
||||
# The stats log line (PLAN §9 ample logging).
|
||||
assert any(
|
||||
"folder_summaries: generated=2 failed=0 pruned=0" in r.getMessage()
|
||||
"folder_summaries: generated=2 failed=0 pruned=0 kept_manual=0"
|
||||
in r.getMessage()
|
||||
for r in records
|
||||
)
|
||||
|
||||
@@ -470,6 +471,96 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green(
|
||||
assert root_stamp_after is not None and root_stamp_after > root_stamp_before
|
||||
|
||||
|
||||
def test_changed_import_never_overwrites_a_manual_row(
|
||||
db: Session,
|
||||
src: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""Phase 97 (task 01): a manually-edited folder description survives
|
||||
a KB-changing sync — the generator SKIPS it (zero
|
||||
``FOLDER_SUMMARY_MODE`` calls for it), the summary-line token
|
||||
STAYS 3 fields (``folder_summaries=<generated>/<failed>/<pruned>``
|
||||
— ``kept_manual`` is a stat, not a token), the manual row's text,
|
||||
stamp, and flag are untouched, and the other folders regenerate
|
||||
(the ``kept_manual`` stat lands on the generator's log line)."""
|
||||
# First sync: full generation — root + a/ (b/ holds 1 doc: none).
|
||||
llm1 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=1 folder_summaries=2/0/0"
|
||||
)
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
|
||||
# The owner edits the source-root description (task 03's PATCH is
|
||||
# the writer; task 01 pins the generator's behavior, so the row is
|
||||
# inserted directly — the ``test_import_docs_overview.py`` pattern).
|
||||
manual_text = "Owner's own words about MyDocs."
|
||||
db.execute(
|
||||
text(
|
||||
"UPDATE folder_summaries SET summary = :s, manually_edited = true"
|
||||
" WHERE source = 'MyDocs' AND folder_path = ''"
|
||||
),
|
||||
{"s": manual_text},
|
||||
)
|
||||
db.commit()
|
||||
root_stamp_before = _updated_at(db, "MyDocs", "")
|
||||
assert root_stamp_before is not None
|
||||
|
||||
records: list[logging.LogRecord] = []
|
||||
|
||||
class _Sink(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
records.append(record)
|
||||
|
||||
fs_logger = logging.getLogger("app.rag.folder_summaries")
|
||||
sink = _Sink()
|
||||
fs_logger.addHandler(sink)
|
||||
fs_logger.setLevel(logging.INFO)
|
||||
try:
|
||||
# A KB-changing re-sync (a new doc under a/) — the gate fires a
|
||||
# full regeneration ...
|
||||
(src / "a" / "three.md").write_text(
|
||||
"# A Three\nAnother folder document.\n", encoding="utf-8"
|
||||
)
|
||||
llm2 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
|
||||
finally:
|
||||
fs_logger.removeHandler(sink)
|
||||
|
||||
assert rc == 0
|
||||
assert "added=1" in out
|
||||
# The token STAYS 3 fields — kept_manual is a stat, not a token.
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=1/0/0"
|
||||
)
|
||||
# Zero folder calls for the owner's folder — only a/ (now 3 docs).
|
||||
calls = _folder_calls(llm2)
|
||||
assert [c[1]["content"].splitlines()[0] for c in calls] == ["Folder: MyDocs/a"]
|
||||
# The owner's text, stamp, and flag are untouched ...
|
||||
rows_after = _rows(db)
|
||||
assert rows_after[("MyDocs", "")] == manual_text
|
||||
assert _updated_at(db, "MyDocs", "") == root_stamp_before, (
|
||||
"the manual row is never re-stamped"
|
||||
)
|
||||
flag = db.execute(
|
||||
text(
|
||||
"SELECT manually_edited FROM folder_summaries"
|
||||
" WHERE source = 'MyDocs' AND folder_path = ''"
|
||||
)
|
||||
).scalar_one()
|
||||
assert flag is True, "the generator never clears the flag"
|
||||
# ... while the other folder regenerates.
|
||||
assert rows_after[("MyDocs", "a")] is not None
|
||||
# The 4-field stats line carries the skip (PLAN §9 ample logging).
|
||||
assert any(
|
||||
"folder_summaries: generated=1 failed=0 pruned=0 kept_manual=1"
|
||||
in r.getMessage()
|
||||
for r in records
|
||||
)
|
||||
|
||||
|
||||
def test_limit_run_skips_folder_generation(
|
||||
db: Session,
|
||||
src: Path,
|
||||
|
||||
Reference in New Issue
Block a user