phase: 98_sync_summary_visibility
All verification complete. Final report: **Phase 98 — Sync summary visibility: final verification pass** (all 5 tasks already complete; implementation verified against the design, no defects found, no code changes needed) - **Implementation checked:** `SyncStatus` phase machine (4 new keys, terminal-keep counts), `on_progress` hook in `generate_folder_summaries`, `summary_pending` on `KbTreeSource`/`KbTreeFolder` + D3 rule in `build_kb_tree`, phase-aware sync labels + pending UI in `sources.js`, `.kb-summary-pending` CSS — all match decisions D1–D5. - **Unit + integration:** `uv run pytest` → 2184 tests, 0 failed/errors (exit 0) - **Coverage:** `uv run pytest --cov=app --cov-report=term-missing` → **99%** on `app/` (criterion >90% ✓; `app/api/sync.py` and `app/rag/folder_summaries.py` at 100%) - **Lint/types:** `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors, 0 warnings - **Phase E2E (isolation):** `uv run pytest tests/e2e/test_sync_summary_visibility.py -v --no-cov` → **3 passed** (phase machine, live label, pending markers + gap-fill self-heal) - **Regression suites (each isolated, `--no-cov`):** test_kb_tree ✓, test_ls_tree_drilldown 3 ✓, test_sync_button 3 ✓, test_sync_upload_progress 4 ✓, test_oneshot_llm_retry 2 ✓, test_local_directory_sources 3 ✓ - **Completion criteria:** all 7 verified green — status phase fields + terminal semantics; `Writing KB overview…`/`Summarizing folders… (n/m)` labels (title + aria-live); pending set == `missing_folder_summaries` (integration cross-check pinned at `test_docs_api.py:428`); CLI/`ls` byte-identity (no changes to those paths, pins green); suite/coverage/lint gates; dedicated + regression E2E. Commit left to the harness per protocol (no `git add`/`commit` run). - **Decisions/deviations:** none — no fixes were required this pass. - **Next pending phase:** `99_kb_tree_table_and_back_nav`.
This commit is contained in:
@@ -14,7 +14,7 @@ from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy import delete, func, select, text
|
||||
|
||||
import app.api.docs as docs_api
|
||||
from app.config import Settings
|
||||
@@ -22,6 +22,7 @@ from app.core import tokens as token_service
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Chunk, Document, FolderSummary, GitSource
|
||||
from app.rag import git_sources as rag_git_sources
|
||||
from app.rag.folder_summaries import missing_folder_summaries
|
||||
|
||||
_TREE_TABLES = "chunks, documents, folder_summaries, git_sources"
|
||||
|
||||
@@ -65,6 +66,25 @@ def _tree_file_nodes(sources) -> list[dict]:
|
||||
return files
|
||||
|
||||
|
||||
def _tree_pending_keys(sources) -> set[tuple[str, str]]:
|
||||
"""Every ``(source, folder_path)`` flagged ``summary_pending`` in a
|
||||
tree response — the SOURCE root rides ``folder_path = ""`` (the
|
||||
``folder_summaries`` convention); walked recursively over the whole
|
||||
tree (the D3 set the cross-check compares against
|
||||
:func:`missing_folder_summaries`)."""
|
||||
keys: set[tuple[str, str]] = set()
|
||||
|
||||
def _walk(source_name: str, node: dict) -> None:
|
||||
if node.get("summary_pending"):
|
||||
keys.add((source_name, node["path"] if node.get("kind") == "folder" else ""))
|
||||
for child in node.get("children", ()):
|
||||
_walk(source_name, child)
|
||||
|
||||
for source in sources:
|
||||
_walk(source["name"], source)
|
||||
return keys
|
||||
|
||||
|
||||
def test_docs_empty_shape(admin_client, db) -> None:
|
||||
db.execute(text("TRUNCATE chunks, documents"))
|
||||
db.commit()
|
||||
@@ -177,9 +197,12 @@ def test_docs_tree_populated_shape_order_counts_summaries(admin_client, db) -> N
|
||||
assert [s["name"] for s in sources] == ["Homelab", "Deployments"]
|
||||
|
||||
homelab, deployments = sources
|
||||
assert set(homelab) == {"name", "documents", "summary", "children"}
|
||||
# Wire-additive (phase 98, task 03): the pre-pending keys are all
|
||||
# still there, joined by ``summary_pending``.
|
||||
assert set(homelab) == {"name", "documents", "summary", "summary_pending", "children"}
|
||||
assert homelab["documents"] == 4 # the whole recursive count
|
||||
assert homelab["summary"] == "Homelab docs." # the (source, "") row
|
||||
assert homelab["summary_pending"] is False # the stored root row covers it
|
||||
assert deployments["summary"] is None # no stored root row
|
||||
assert deployments["documents"] == 2
|
||||
|
||||
@@ -276,13 +299,147 @@ def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> Non
|
||||
assert [s["name"] for s in sources] == ["Alpha", "Midx", "Zeta"]
|
||||
alpha, midx, zeta = sources
|
||||
assert (alpha["documents"], alpha["children"], alpha["summary"]) == (0, [], None)
|
||||
assert alpha["summary_pending"] is False # 0 documents — never pending
|
||||
assert midx["documents"] == 1
|
||||
assert midx["summary_pending"] is False # 1 document — below the minimum
|
||||
assert zeta["documents"] == 1
|
||||
assert zeta["summary_pending"] is False # 1 document — below the minimum
|
||||
assert [c["path"] for c in midx["children"]] == ["m1.md"]
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db) -> None:
|
||||
"""The endpoint returns ``summary_pending`` on SOURCE + FOLDER
|
||||
nodes (phase 98, task 03 — D3): true iff the recursive count is
|
||||
≥ 2 AND no stored row; false WITH a stored row (any — the endpoint
|
||||
cannot tell AI from manual); false for a < 2-document folder
|
||||
(never pending) — including one NESTED. File nodes carry no flag."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
# k8s → 3 documents (talos + cluster + charts), NO stored row → pending
|
||||
_seed_doc(db, "Homelab", "k8s/talos.md", "Talos", 1, base)
|
||||
_seed_doc(db, "Homelab", "k8s/cluster.md", "Cluster", 1, base)
|
||||
# k8s/helm → 1 document — below the 2-doc minimum, never pending
|
||||
_seed_doc(db, "Homelab", "k8s/helm/charts.md", "Charts", 1, base)
|
||||
# wiki → 2 documents, WITH a stored row → not pending
|
||||
_seed_doc(db, "Homelab", "wiki/one.md", "One", 1, base)
|
||||
_seed_doc(db, "Homelab", "wiki/two.md", "Two", 1, base)
|
||||
db.add_all(
|
||||
[
|
||||
# The 5-doc source root IS covered → the source node is not pending
|
||||
FolderSummary(source="Homelab", folder_path="", summary="Homelab docs."),
|
||||
FolderSummary(source="Homelab", folder_path="wiki", summary="Wiki pages."),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
(homelab,) = r.json()["sources"]
|
||||
assert set(homelab) == {"name", "documents", "summary", "summary_pending", "children"}
|
||||
assert homelab["summary"] == "Homelab docs."
|
||||
assert homelab["summary_pending"] is False
|
||||
# Direct subfolders in path order: k8s < wiki.
|
||||
k8s, wiki = [c for c in homelab["children"] if c["kind"] == "folder"]
|
||||
assert k8s["path"] == "k8s"
|
||||
assert k8s["summary"] is None
|
||||
assert k8s["summary_pending"] is True # 3 docs, no stored row
|
||||
helm = k8s["children"][0]
|
||||
assert (helm["kind"], helm["path"]) == ("folder", "k8s/helm")
|
||||
assert helm["summary_pending"] is False # 1 doc — never pending
|
||||
assert wiki["summary"] == "Wiki pages."
|
||||
assert wiki["summary_pending"] is False # 2 docs, but a stored row covers it
|
||||
# File nodes carry no pending flag at all (the file table has no
|
||||
# description column — D3's file exclusion).
|
||||
for node in (k8s, wiki, helm):
|
||||
for child in node["children"]:
|
||||
if child["kind"] == "file":
|
||||
assert "summary_pending" not in child
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db) -> None:
|
||||
"""The D3 cross-check (ONE concept end to end): with a PARTIAL
|
||||
summary table (some rows deleted — the phase-96 gap-fill pattern),
|
||||
the set of ``(source, folder_path)`` flagged pending in the fetched
|
||||
tree (source root = ``""``) equals
|
||||
:func:`missing_folder_summaries` — the marker can never drift from
|
||||
the gap-fill."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Alpha.git", kind="git", added_at=base))
|
||||
db.add(
|
||||
GitSource(
|
||||
url="https://github.com/reese/Beta.git",
|
||||
kind="git",
|
||||
added_at=base + timedelta(hours=1),
|
||||
)
|
||||
)
|
||||
db.add(
|
||||
GitSource(
|
||||
url="https://github.com/reese/Gamma.git",
|
||||
kind="git",
|
||||
added_at=base + timedelta(hours=2),
|
||||
)
|
||||
)
|
||||
# Alpha: a/b holds 2 docs, c holds 1 (NEVER a candidate), the root
|
||||
# holds 4 — candidates (Alpha, ""), (Alpha, "a"), (Alpha, "a/b").
|
||||
_seed_doc(db, "Alpha", "a/b/c1.md", "C1", 1, base)
|
||||
_seed_doc(db, "Alpha", "a/b/c2.md", "C2", 1, base)
|
||||
_seed_doc(db, "Alpha", "c/solo.md", "Solo", 1, base)
|
||||
_seed_doc(db, "Alpha", "top.md", "Top", 1, base)
|
||||
# Beta: x holds 2 docs, the root holds 2 — candidates
|
||||
# (Beta, ""), (Beta, "x").
|
||||
_seed_doc(db, "Beta", "x/one.md", "One", 1, base)
|
||||
_seed_doc(db, "Beta", "x/two.md", "Two", 1, base)
|
||||
# Gamma: registered, 0 documents — no candidates at all.
|
||||
# Seed every candidate row, then DELETE two of them (the phase-96
|
||||
# direct-row-deletion pattern — a fail-soft miss / cleared row).
|
||||
db.add_all(
|
||||
FolderSummary(source=s, folder_path=f, summary=t)
|
||||
for (s, f), t in {
|
||||
("Alpha", ""): "Alpha root.",
|
||||
("Alpha", "a"): "Alpha a.",
|
||||
("Alpha", "a/b"): "Alpha a b.",
|
||||
("Beta", ""): "Beta root.",
|
||||
("Beta", "x"): "Beta x.",
|
||||
}.items()
|
||||
)
|
||||
db.commit()
|
||||
db.execute(
|
||||
delete(FolderSummary).where(
|
||||
FolderSummary.source == "Alpha", FolderSummary.folder_path == "a"
|
||||
)
|
||||
)
|
||||
db.execute(
|
||||
delete(FolderSummary).where(
|
||||
FolderSummary.source == "Beta", FolderSummary.folder_path == ""
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
r = admin_client.get("/api/docs/tree")
|
||||
assert r.status_code == 200
|
||||
pending = _tree_pending_keys(r.json()["sources"])
|
||||
# THE cross-check: the marker set IS the gap-fill's candidate set.
|
||||
assert pending == set(missing_folder_summaries(db))
|
||||
# And the explicit expectation (the test is readable without the
|
||||
# helper): exactly the two deleted keys, root riding "".
|
||||
assert pending == {("Alpha", "a"), ("Beta", "")}
|
||||
# Never flagged: the < 2-doc folder (Alpha/c), the 0-document
|
||||
# registered source (Gamma), and every node that still holds a row.
|
||||
assert ("Alpha", "c") not in pending
|
||||
assert not any(name == "Gamma" for name, _ in pending)
|
||||
assert ("Alpha", "a/b") not in pending
|
||||
assert ("Beta", "x") not in pending
|
||||
assert ("Alpha", "") not in pending
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# PATCH /api/folders/summary (phase 97, task 03) — the admin
|
||||
# folder-description editor: update / create / source-root / clear /
|
||||
|
||||
Reference in New Issue
Block a user