feat(rag): summarize single-document folders (MIN_DOCS_PER_FOLDER 2 → 1)
Relax the phase-94 folder-summary scope rule from ≥ 2 documents to ≥ 1: a folder (or source root) is a candidate while ANY document lives under it, so single-file folders and single-file source roots get their own lite-written description. A row is now pruned only when its folder loses its last document (vanishes from the catalogue). The constant is the single source of truth, so the flip propagates to the generator's candidate set, the prune pass, the missing_folder_summaries gap probe (the next sync self-heals the new gaps), and the KB-tree summary_pending markers (1-doc folders / sources now read "Summary pending" until their row lands). Docstrings/comments across app/, scripts/import_docs.py, and the E2E fixtures updated to the ≥ 1 wording. Unit + integration tests updated to the new semantics (the pruned-below-minimum scenario is now a folder losing its LAST doc; single-doc folders are pinned as candidates/pending). Full suite: 2314 passed, app coverage 99%; ruff + pyright clean; folder-summary E2E stories pass in isolation (ls_tree_drilldown, sync_summary_visibility, kb_tree, kb_tree_nav, document_dates, oneshot_llm_retry).
This commit is contained in:
@@ -26,7 +26,9 @@ real-Sync pattern; no git anywhere):
|
||||
Total: 7 documents; alpha counts 5 (root + 2 + 2), beta counts 2. Every
|
||||
stored description is deterministic: the mock's EXISTING
|
||||
``FOLDER_SUMMARY_MODE`` branch (phase 94 — no mock changes needed)
|
||||
stores, per ≥ 2-doc folder, the canned one-liner naming the folder,
|
||||
stores, per existing folder (the ≥ 1 rule — this fixture holds no
|
||||
single-doc folders, so the pinned rows are unchanged), the canned
|
||||
one-liner naming the folder,
|
||||
``Fixture folder summary for <source>[/<folder>].`` — the ``synced_kb``
|
||||
module fixture pins those exact rows (all with
|
||||
``manually_edited = false``) after the sync, and the tests assert on
|
||||
@@ -154,8 +156,9 @@ TOTAL_DOCS = ALPHA_COUNT + BETA_COUNT
|
||||
#: The sync-time folder descriptions the mock's canned
|
||||
#: ``FOLDER_SUMMARY_MODE`` branch stores (the phase-94 byte-stable
|
||||
#: template — the one-liner names the folder), in ``(source,
|
||||
#: folder_path)`` order: one row per ≥ 2-doc folder (the
|
||||
#: recursive-subtree rule) — the ``""`` rows are the source roots.
|
||||
#: folder_path)`` order: one row per existing folder (the ≥ 1-doc
|
||||
#: recursive-subtree rule — this fixture holds no single-doc folders)
|
||||
#: — the ``""`` rows are the source roots.
|
||||
SUMMARY_FOR = "Fixture folder summary for {}."
|
||||
ALPHA_ROOT_SUM = SUMMARY_FOR.format(ALPHA)
|
||||
ONE_SUM = SUMMARY_FOR.format(f"{ALPHA}/one")
|
||||
@@ -382,7 +385,8 @@ def synced_kb(app_server: str, kb_tree_dirs: tuple[Path, Path]) -> None:
|
||||
walk → chunk → embed → overview → folder summaries → version
|
||||
bump), and pins the stored folder descriptions: the mock's canned
|
||||
``FOLDER_SUMMARY_MODE`` branch (phase 94) makes the sync store one
|
||||
deterministic row per ≥ 2-doc folder — the tests assert on that
|
||||
deterministic row per existing folder (the ≥ 1-doc rule — this
|
||||
fixture holds no single-doc folders) — the tests assert on that
|
||||
exact text (and on the phase-97 ``manually_edited`` flag: every
|
||||
stored row starts out AI-written).
|
||||
"""
|
||||
@@ -409,7 +413,8 @@ def synced_kb(app_server: str, kb_tree_dirs: tuple[Path, Path]) -> None:
|
||||
assert detail["pruned"] == 0, detail
|
||||
assert detail["overview"] is True, detail
|
||||
# The change-gated folder descriptions (phase 94) landed: one row
|
||||
# per ≥ 2-doc folder, the mock's byte-stable text — and every row
|
||||
# per existing folder (the ≥ 1-doc rule — this fixture holds no
|
||||
# single-doc folders), the mock's byte-stable text — and every row
|
||||
# AI-written (the ``manually_edited`` flag starts false).
|
||||
with SessionLocal() as db:
|
||||
rows = db.execute(
|
||||
|
||||
@@ -42,7 +42,8 @@ of the mock's cosine distribution, and the ``<tools>`` section is present
|
||||
(the flow's precondition).
|
||||
|
||||
The mock's canned ``FOLDER_SUMMARY_MODE`` branch (phase 94 task 01)
|
||||
stores, per ≥ 2-doc folder, the deterministic one-liner
|
||||
stores, per existing folder (the ≥ 1-doc rule — this fixture holds
|
||||
no single-doc folders), the deterministic one-liner
|
||||
``Fixture folder summary for <source>[/<folder>].`` — the ``synced_kb``
|
||||
module fixture pins those exact rows in ``folder_summaries`` after the
|
||||
sync, and the drill answers assert on them in the ``ls`` output.
|
||||
@@ -159,8 +160,9 @@ assert "\n" not in TWO_A_CONTENT[:80] # the quote must stay one line
|
||||
|
||||
#: The sync-time folder summaries the mock's canned ``FOLDER_SUMMARY_MODE``
|
||||
#: branch stores (task 01's byte-stable template), in
|
||||
#: ``(source, folder_path)`` order: one row per ≥ 2-doc folder (the
|
||||
#: recursive-subtree rule) — the ``""`` rows are the source roots.
|
||||
#: ``(source, folder_path)`` order: one row per existing folder (the
|
||||
#: ≥ 1-doc recursive-subtree rule — this fixture holds no single-doc
|
||||
#: folders) — the ``""`` rows are the source roots.
|
||||
EXPECTED_SUMMARIES: list[tuple[str, str, str]] = [
|
||||
(ALPHA, "", f"Fixture folder summary for {ALPHA}."),
|
||||
(ALPHA, "one", f"Fixture folder summary for {ALPHA}/one."),
|
||||
@@ -426,7 +428,8 @@ def synced_kb(app_server: str, drill_dirs: tuple[Path, Path]) -> None:
|
||||
walk → chunk → embed → overview → folder summaries → version bump),
|
||||
and pins the stored folder summaries: the mock's canned
|
||||
``FOLDER_SUMMARY_MODE`` branch (task 01) makes the sync store one
|
||||
deterministic row per ≥ 2-doc folder — the drill turns' answers
|
||||
deterministic row per existing folder (the ≥ 1-doc rule — this
|
||||
fixture holds no single-doc folders) — the drill turns' answers
|
||||
assert on that exact text.
|
||||
"""
|
||||
alpha, beta = drill_dirs
|
||||
@@ -452,7 +455,8 @@ def synced_kb(app_server: str, drill_dirs: tuple[Path, Path]) -> None:
|
||||
assert detail["pruned"] == 0, detail
|
||||
assert detail["overview"] is True, detail
|
||||
# The change-gated folder summaries (phase 94 task 02) landed: one
|
||||
# row per ≥ 2-doc folder, the mock's byte-stable text (the drill
|
||||
# row per existing folder (the ≥ 1-doc rule — this fixture holds
|
||||
# no single-doc folders), the mock's byte-stable text (the drill
|
||||
# answers quote exactly these lines).
|
||||
with SessionLocal() as db:
|
||||
rows = db.execute(
|
||||
|
||||
@@ -46,7 +46,9 @@ the same host) registered as a ``kind=local`` source (the
|
||||
pattern; no git anywhere): ``syncsum/`` with TWO ≥ 2-doc folders —
|
||||
``alpha/`` (2 docs) and ``bravo/`` (2 docs), no root-level files. The
|
||||
generator's candidate set is therefore exactly 3: the source root
|
||||
(``folder_path ""``) + the two folders (the recursive-subtree ≥ 2 rule).
|
||||
(``folder_path ""``) + the two folders (the recursive-subtree ≥ 1
|
||||
rule — this fixture holds no single-doc folders, so the rule's
|
||||
relaxation is inert for it).
|
||||
|
||||
Autouse cleanup (the phase-96 pattern): before each test wait for no
|
||||
running sync on EITHER app, then truncate the shared registry + KB
|
||||
@@ -160,7 +162,7 @@ FOLDER_B = "bravo"
|
||||
FOLDERS = (FOLDER_A, FOLDER_B)
|
||||
|
||||
N_FILES = 4 # two folders × 2 docs
|
||||
N_CANDIDATES = 3 # the source root + the two folders (the ≥ 2-doc rule)
|
||||
N_CANDIDATES = 3 # the source root + the two folders (the ≥ 1-doc rule)
|
||||
|
||||
#: The mock's byte-stable ``FOLDER_SUMMARY_MODE`` lines for this fixture
|
||||
#: (the phase-94 template — the label is the ``FOLDER_HEADER_PREFIX``
|
||||
|
||||
@@ -310,9 +310,9 @@ def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> Non
|
||||
assert (alpha["documents"], alpha["children"], alpha["summary"]) == (0, [], None)
|
||||
assert alpha["summary_pending"] is False # 0 documents — never pending
|
||||
assert midx["documents"] == 1
|
||||
assert midx["summary_pending"] is False # 1 document — below the minimum
|
||||
assert midx["summary_pending"] is True # 1 document — the ≥ 1 minimum, no row
|
||||
assert zeta["documents"] == 1
|
||||
assert zeta["summary_pending"] is False # 1 document — below the minimum
|
||||
assert zeta["summary_pending"] is True # 1 document — the ≥ 1 minimum, no row
|
||||
assert [c["path"] for c in midx["children"]] == ["m1.md"]
|
||||
|
||||
_truncate_tree_tables(db)
|
||||
@@ -321,16 +321,16 @@ def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> Non
|
||||
def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db) -> None:
|
||||
"""The endpoint returns ``summary_pending`` on SOURCE + FOLDER
|
||||
nodes (phase 98, task 03 — D3): true iff the recursive count is
|
||||
≥ 2 AND no stored row; false WITH a stored row (any — the endpoint
|
||||
cannot tell AI from manual); false for a < 2-document folder
|
||||
(never pending) — including one NESTED. File nodes carry no flag."""
|
||||
≥ 1 AND no stored row — the NESTED 1-doc folder included (the ≥ 1
|
||||
rule); false WITH a stored row (any — the endpoint cannot tell AI
|
||||
from manual). File nodes carry no flag."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
# k8s → 3 documents (talos + cluster + charts), NO stored row → pending
|
||||
_seed_doc(db, "Homelab", "k8s/talos.md", "Talos", 1, base)
|
||||
_seed_doc(db, "Homelab", "k8s/cluster.md", "Cluster", 1, base)
|
||||
# k8s/helm → 1 document — below the 2-doc minimum, never pending
|
||||
# k8s/helm → 1 document — a candidate at the ≥ 1 rule (no row)
|
||||
_seed_doc(db, "Homelab", "k8s/helm/charts.md", "Charts", 1, base)
|
||||
# wiki → 2 documents, WITH a stored row → not pending
|
||||
_seed_doc(db, "Homelab", "wiki/one.md", "One", 1, base)
|
||||
@@ -360,7 +360,7 @@ def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db)
|
||||
assert k8s["summary_pending"] is True # 3 docs, no stored row
|
||||
helm = k8s["children"][0]
|
||||
assert (helm["kind"], helm["path"]) == ("folder", "k8s/helm")
|
||||
assert helm["summary_pending"] is False # 1 doc — never pending
|
||||
assert helm["summary_pending"] is True # 1 doc — pending at the ≥ 1 rule
|
||||
assert wiki["summary"] == "Wiki pages."
|
||||
assert wiki["summary_pending"] is False # 2 docs, but a stored row covers it
|
||||
# File nodes carry no pending flag at all (the file table has no
|
||||
@@ -397,8 +397,9 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db)
|
||||
added_at=base + timedelta(hours=2),
|
||||
)
|
||||
)
|
||||
# Alpha: a/b holds 2 docs, c holds 1 (NEVER a candidate), the root
|
||||
# holds 4 — candidates (Alpha, ""), (Alpha, "a"), (Alpha, "a/b").
|
||||
# Alpha: a/b holds 2 docs, c holds 1 (a candidate at the ≥ 1
|
||||
# rule), the root holds 4 — candidates (Alpha, ""), (Alpha, "a"),
|
||||
# (Alpha, "a/b"), (Alpha, "c").
|
||||
_seed_doc(db, "Alpha", "a/b/c1.md", "C1", 1, base)
|
||||
_seed_doc(db, "Alpha", "a/b/c2.md", "C2", 1, base)
|
||||
_seed_doc(db, "Alpha", "c/solo.md", "Solo", 1, base)
|
||||
@@ -416,6 +417,7 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db)
|
||||
("Alpha", ""): "Alpha root.",
|
||||
("Alpha", "a"): "Alpha a.",
|
||||
("Alpha", "a/b"): "Alpha a b.",
|
||||
("Alpha", "c"): "Alpha c.",
|
||||
("Beta", ""): "Beta root.",
|
||||
("Beta", "x"): "Beta x.",
|
||||
}.items()
|
||||
@@ -441,8 +443,10 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db)
|
||||
# And the explicit expectation (the test is readable without the
|
||||
# helper): exactly the two deleted keys, root riding "".
|
||||
assert pending == {("Alpha", "a"), ("Beta", "")}
|
||||
# Never flagged: the < 2-doc folder (Alpha/c), the 0-document
|
||||
# registered source (Gamma), and every node that still holds a row.
|
||||
# Never flagged: the 0-document registered source (Gamma) and every
|
||||
# node that still holds a row — incl. the 1-doc Alpha/c, whose
|
||||
# stored row covers it (the ≥ 1 rule makes it a candidate, but a
|
||||
# candidate WITH a row is not missing).
|
||||
assert ("Alpha", "c") not in pending
|
||||
assert not any(name == "Gamma" for name, _ in pending)
|
||||
assert ("Alpha", "a/b") not in pending
|
||||
@@ -464,8 +468,8 @@ OLDER_STAMP = datetime(2020, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def _seed_folder_pair(db, source: str, folder: str, base: datetime) -> None:
|
||||
"""Two documents under ``source/folder`` — the ≥ 2-document
|
||||
minimum a folder must hold to be summarizable."""
|
||||
"""Two documents under ``source/folder`` — a summarizable folder
|
||||
(≥ 1 doc at the current minimum; two keep the pair realistic)."""
|
||||
_seed_doc(db, source, f"{folder}/one.md", f"{folder} one", 1, base)
|
||||
_seed_doc(db, source, f"{folder}/two.md", f"{folder} two", 2, base)
|
||||
|
||||
@@ -543,15 +547,17 @@ def test_folder_summary_create_where_no_row_exists(
|
||||
admin_client: TestClient, db
|
||||
) -> None:
|
||||
"""A manual description can be CREATED where no row exists — a
|
||||
< 2-document folder the generator never wrote (or its fail-soft
|
||||
miss): insert, not update, with ``manually_edited = true``."""
|
||||
folder the generator has not written yet (its fail-soft miss, or
|
||||
a pre-sync state): insert, not update, with
|
||||
``manually_edited = true``."""
|
||||
_truncate_tree_tables(db)
|
||||
base = datetime.now(UTC)
|
||||
db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base))
|
||||
_seed_doc(db, "Homelab", "solo/only.md", "Only", 1, base) # 1-doc folder
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
assert _get_folder_row(db, "Homelab", "solo") is None # no AI row for < 2 docs
|
||||
# No AI row yet (the generator has not run in this test).
|
||||
assert _get_folder_row(db, "Homelab", "solo") is None
|
||||
|
||||
r = admin_client.patch(
|
||||
"/api/folders/summary",
|
||||
|
||||
@@ -11,9 +11,9 @@ convention — the generator only flushes, the sync path commits).
|
||||
Script path (``scripts.import_docs.main`` end to end, fake LLM, real
|
||||
DB, explicit ``--source``):
|
||||
|
||||
- a KB-changing import → one row per ≥ 2-doc subtree (the source root
|
||||
+ the 2-doc folder; the 1-doc folder gets none), committed in the
|
||||
run's transaction, the summary line ending
|
||||
- a KB-changing import → one row per existing subtree (the source
|
||||
root, the 2-doc folder a, AND the 1-doc folder b — the ≥ 1 rule),
|
||||
committed in the run's transaction, the summary line ending
|
||||
``folder_summaries=<generated>/<failed>/<pruned>`` (unchanged by
|
||||
phase 96 — no gap-fill suffix on a full regeneration);
|
||||
- an unchanged re-import with a COMPLETE table → zero ``lite`` calls,
|
||||
@@ -27,10 +27,10 @@ DB, explicit ``--source``):
|
||||
failed folder summary self-heals on the next sync);
|
||||
- a KB change on a second run → still a FULL regeneration (call count
|
||||
== candidate count, every row re-stamped, no gap-fill suffix);
|
||||
- a subtree dropping below 2 docs after a changed re-walk → its row
|
||||
- a subtree losing its LAST doc after a changed re-walk → its row
|
||||
pruned;
|
||||
- one folder's ``lite`` failure → its previous row kept, the other
|
||||
lands, exit code 0, stats ``1/1/0``;
|
||||
land, exit code 0, stats ``2/1/0``;
|
||||
- a ``--limit`` debug run → no generation, no rows,
|
||||
``folder_summaries=skipped``;
|
||||
- a fresh (empty) table after a ``--limit`` first walk → an unchanged
|
||||
@@ -189,7 +189,8 @@ def _fresh_sync_state() -> Iterator[None]:
|
||||
|
||||
@pytest.fixture()
|
||||
def src(tmp_path: Path) -> Path:
|
||||
"""MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1.
|
||||
"""MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1
|
||||
(b/ is a candidate too — the ≥ 1 rule).
|
||||
|
||||
md → no document-summary chat calls, so the ``lite`` traffic is
|
||||
exactly the overview + the folder summaries."""
|
||||
@@ -235,9 +236,9 @@ def test_changed_import_generates_folder_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""A KB-changing import upserts one row per ≥ 2-doc subtree in the
|
||||
run's own transaction — committed and visible afterwards — with the
|
||||
stats on the summary line (PLAN §9)."""
|
||||
"""A KB-changing import upserts one row per existing subtree in
|
||||
the run's own transaction — committed and visible afterwards —
|
||||
with the stats on the summary line (PLAN §9)."""
|
||||
llm = FakeEmbedder()
|
||||
records: list[logging.LogRecord] = []
|
||||
|
||||
@@ -257,25 +258,26 @@ def test_changed_import_generates_folder_rows(
|
||||
assert rc == 0
|
||||
assert "added=3" in out
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=1 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=1 folder_summaries=3/0/0"
|
||||
)
|
||||
# Two FOLDER_SUMMARY_MODE calls — the source root (3 docs) and the
|
||||
# 2-doc folder a; the 1-doc folder b gets no row (no lite burn).
|
||||
# Three FOLDER_SUMMARY_MODE calls — the source root (3 docs), the
|
||||
# 2-doc folder a, and the 1-doc folder b (the ≥ 1 rule: every
|
||||
# existing folder is a candidate).
|
||||
calls = _folder_calls(llm)
|
||||
assert len(calls) == 2
|
||||
assert len(calls) == 3
|
||||
headers = [c[1]["content"].splitlines()[0] for c in calls]
|
||||
assert headers == ["Folder: MyDocs", "Folder: MyDocs/a"]
|
||||
assert headers == ["Folder: MyDocs", "Folder: MyDocs/a", "Folder: MyDocs/b"]
|
||||
for c in calls:
|
||||
assert "FOLDER_SUMMARY_MODE" in c[0]["content"]
|
||||
# The rows land committed (the run's own session committed them) —
|
||||
# one per ≥ 2-doc subtree, never empty, stamped.
|
||||
# one per existing subtree, never empty, stamped.
|
||||
rows = _rows(db)
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
assert all(rows.values())
|
||||
assert _updated_at(db, "MyDocs", "a") is not None
|
||||
# The stats log line (PLAN §9 ample logging).
|
||||
assert any(
|
||||
"folder_summaries: generated=2 failed=0 pruned=0 kept_manual=0"
|
||||
"folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0"
|
||||
in r.getMessage()
|
||||
for r in records
|
||||
)
|
||||
@@ -293,10 +295,10 @@ def test_unchanged_reimport_burns_zero_folder_calls(
|
||||
rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=1 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=1 folder_summaries=3/0/0"
|
||||
)
|
||||
rows = _rows(db)
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
|
||||
llm2 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
|
||||
@@ -326,7 +328,7 @@ def test_unchanged_reimport_with_gap_fills_only_the_missing_row(
|
||||
rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
rows_before = _rows(db)
|
||||
assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
root_stamp_before = _updated_at(db, "MyDocs", "")
|
||||
assert root_stamp_before is not None
|
||||
|
||||
@@ -387,13 +389,15 @@ def test_changed_reimport_is_a_full_regeneration(
|
||||
assert "updated=1" in out
|
||||
# Full-regeneration token — the stats without the gap-fill suffix.
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=2 folder_summaries=3/0/0"
|
||||
)
|
||||
# Call count == candidate count — BOTH folders, not a targeted fill.
|
||||
# Call count == candidate count — ALL existing folders, not a
|
||||
# targeted fill.
|
||||
calls = _folder_calls(llm2)
|
||||
assert [c[1]["content"].splitlines()[0] for c in calls] == [
|
||||
"Folder: MyDocs",
|
||||
"Folder: MyDocs/a",
|
||||
"Folder: MyDocs/b",
|
||||
]
|
||||
# All rows re-stamped (the full regeneration re-writes every
|
||||
# candidate, even the unchanging one).
|
||||
@@ -404,35 +408,37 @@ def test_changed_reimport_is_a_full_regeneration(
|
||||
assert _rows(db) == rows_before # deterministic fake → same texts
|
||||
|
||||
|
||||
def test_subtree_dropping_below_two_docs_is_pruned(
|
||||
def test_subtree_losing_its_last_doc_is_pruned(
|
||||
db: Session,
|
||||
src: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""A subtree that drops below the ≥ 2 rule on a changed re-walk
|
||||
loses its (stale) row; the remaining qualifiers regenerate."""
|
||||
"""A subtree that loses its LAST doc (0 recursive docs — below the
|
||||
≥ 1 minimum) on a changed re-walk loses its (stale) row; the
|
||||
remaining qualifiers (≥ 1 doc — the 1-doc b/ included) regenerate."""
|
||||
llm1 = FakeEmbedder()
|
||||
rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
|
||||
# a/ loses one of its two docs (below the ≥ 2 rule) while b's doc
|
||||
# changes → the re-walk is a KB change, so generation runs and the
|
||||
# stale a/ row is pruned; b (1 doc) still gets no row.
|
||||
# a/ loses BOTH its docs (the folder vanishes — 0 docs) while b's
|
||||
# doc changes → the re-walk is a KB change, so generation runs and
|
||||
# the stale a/ row is pruned; b (1 doc) keeps qualifying.
|
||||
(src / "a" / "one.md").unlink()
|
||||
(src / "a" / "two.md").unlink()
|
||||
(src / "b" / "three.md").write_text("# B Three\nChanged content.\n", encoding="utf-8")
|
||||
llm2 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm2, ["--source", str(src), "--prune"], capsys)
|
||||
|
||||
assert rc == 0
|
||||
assert "pruned=1" in out
|
||||
assert "pruned=2" in out # the two a/ documents left the index
|
||||
assert "updated=1" in out
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=1/0/1"
|
||||
"overview=updated sources_version=2 folder_summaries=2/0/1"
|
||||
)
|
||||
# Only the source root (the 2 remaining docs) still qualifies.
|
||||
assert set(_rows(db)) == {("MyDocs", "")}
|
||||
# The source root (1 remaining doc) and b/ (1 doc) still qualify.
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "b")}
|
||||
assert _updated_at(db, "MyDocs", "a") is None # the row is gone
|
||||
|
||||
|
||||
@@ -443,8 +449,8 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""One folder's ``lite`` failure: its previous row is kept, the
|
||||
other folder lands, and the run's exit code stays 0 — the stats
|
||||
carry the failure (``1/1/0``)."""
|
||||
other folders land, and the run's exit code stays 0 — the stats
|
||||
carry the failure (``2/1/0``)."""
|
||||
llm1 = FakeEmbedder()
|
||||
rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
@@ -462,7 +468,7 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green(
|
||||
assert rc == 0 # a failed folder must not fail the import
|
||||
assert "updated=1" in out
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=1/1/0"
|
||||
"overview=updated sources_version=2 folder_summaries=2/1/0"
|
||||
)
|
||||
rows_after = _rows(db)
|
||||
assert rows_after["MyDocs", "a"] == rows_before["MyDocs", "a"] # kept
|
||||
@@ -484,14 +490,15 @@ def test_changed_import_never_overwrites_a_manual_row(
|
||||
— ``kept_manual`` is a stat, not a token), the manual row's text,
|
||||
stamp, and flag are untouched, and the other folders regenerate
|
||||
(the ``kept_manual`` stat lands on the generator's log line)."""
|
||||
# First sync: full generation — root + a/ (b/ holds 1 doc: none).
|
||||
# First sync: full generation — root + a/ + b/ (the ≥ 1 rule: b/
|
||||
# holds 1 doc — still a candidate).
|
||||
llm1 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=1 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=1 folder_summaries=3/0/0"
|
||||
)
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
|
||||
# The owner edits the source-root description (task 03's PATCH is
|
||||
# the writer; task 01 pins the generator's behavior, so the row is
|
||||
@@ -533,11 +540,15 @@ def test_changed_import_never_overwrites_a_manual_row(
|
||||
assert "added=1" in out
|
||||
# The token STAYS 3 fields — kept_manual is a stat, not a token.
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=1/0/0"
|
||||
"overview=updated sources_version=2 folder_summaries=2/0/0"
|
||||
)
|
||||
# Zero folder calls for the owner's folder — only a/ (now 3 docs).
|
||||
# Zero folder calls for the owner's folder — a/ (now 3 docs) and
|
||||
# b/ regenerate.
|
||||
calls = _folder_calls(llm2)
|
||||
assert [c[1]["content"].splitlines()[0] for c in calls] == ["Folder: MyDocs/a"]
|
||||
assert [c[1]["content"].splitlines()[0] for c in calls] == [
|
||||
"Folder: MyDocs/a",
|
||||
"Folder: MyDocs/b",
|
||||
]
|
||||
# The owner's text, stamp, and flag are untouched ...
|
||||
rows_after = _rows(db)
|
||||
assert rows_after[("MyDocs", "")] == manual_text
|
||||
@@ -555,7 +566,7 @@ def test_changed_import_never_overwrites_a_manual_row(
|
||||
assert rows_after[("MyDocs", "a")] is not None
|
||||
# The 4-field stats line carries the skip (PLAN §9 ample logging).
|
||||
assert any(
|
||||
"folder_summaries: generated=1 failed=0 pruned=0 kept_manual=1"
|
||||
"folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1"
|
||||
in r.getMessage()
|
||||
for r in records
|
||||
)
|
||||
@@ -610,10 +621,10 @@ def test_empty_table_generates_on_unchanged_walk(
|
||||
# `` (gap-fill)`` (the generated set is the full candidate set).
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=skipped "
|
||||
"folder_summaries=2/0/0 (gap-fill)"
|
||||
"folder_summaries=3/0/0 (gap-fill)"
|
||||
)
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert len(_folder_calls(llm2)) == 2
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
assert len(_folder_calls(llm2)) == 3
|
||||
assert current_sources_version(db) == 0 # the unchanged walk never bumps
|
||||
|
||||
|
||||
|
||||
@@ -161,9 +161,11 @@ def test_group_by_folder_nested_multi_source_recursive_subtree() -> None:
|
||||
|
||||
|
||||
def test_group_by_folder_single_doc_folder_is_a_group_too() -> None:
|
||||
"""Grouping is pure subtree membership (≥ 1 docs): the ≥ 2 rule is
|
||||
the GENERATOR's (the recursive count below the minimum yields no
|
||||
row — pinned by the generator tests, not the grouping)."""
|
||||
"""Grouping is pure subtree membership: at the
|
||||
:data:`MIN_DOCS_PER_FOLDER` = 1 rule every group (≥ 1 docs) is
|
||||
already a generator candidate, so grouping and candidacy agree
|
||||
(the generator's filter is an inert safety net — pinned by the
|
||||
generator tests, not the grouping)."""
|
||||
rows = [("S", "a/only.md", "O", None)]
|
||||
groups = group_by_folder(rows)
|
||||
assert len(groups[("S", "a")]) == 1 # present, but below the minimum
|
||||
@@ -417,8 +419,9 @@ def _rows(db: Session) -> dict[tuple[str, str], str]:
|
||||
|
||||
def _seed_catalogue(db: Session) -> None:
|
||||
"""The shared catalogue: FSU has four docs in three candidate
|
||||
folders (root 4, a 3, a/b 2 — all ≥ the minimum); FSU-solo has one
|
||||
doc (its root folder is below the minimum — no row, no call)."""
|
||||
folders (root 4, a 3, a/b 2); FSU-solo has one doc (its root
|
||||
folder is a candidate too — the ≥ 1 rule: a single-file source
|
||||
root gets a row)."""
|
||||
_add_doc(db, "FSU", "a/b/one.md", "One", "One lead.\nSource: FSU/a/b/one.md")
|
||||
_add_doc(db, "FSU", "a/b/two.md", "Two")
|
||||
_add_doc(db, "FSU", "a/three.md", "Three")
|
||||
@@ -436,22 +439,23 @@ def clean_tables(db: Session):
|
||||
def test_generate_happy_path_upserts_every_candidate_folder(
|
||||
db: Session, clean_tables, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""Every folder with ≥ 2 recursive docs gets a row (the source root
|
||||
row included — ``folder_path = ''``); single-doc folders get none;
|
||||
rows are stamped fresh; the stats dict and the log line are right;
|
||||
folders are processed in deterministic (source, folder_path) order."""
|
||||
"""Every folder with ≥ 1 recursive doc gets a row (the source root
|
||||
row included — ``folder_path = ''`` — and the single-doc FSU-solo
|
||||
root: the ≥ 1 rule); rows are stamped fresh; the stats dict and
|
||||
the log line are right; folders are processed in deterministic
|
||||
(source, folder_path) order."""
|
||||
_seed_catalogue(db)
|
||||
llm = _FakeLLM()
|
||||
with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"):
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm))
|
||||
assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 0}
|
||||
assert llm.calls == 3, "one lite call per candidate folder (the solo folder: none)"
|
||||
assert stats == {"generated": 4, "failed": 0, "pruned": 0, "kept_manual": 0}
|
||||
assert llm.calls == 4, "one lite call per candidate folder (the solo root: too)"
|
||||
|
||||
stored = _rows(db)
|
||||
assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b")}
|
||||
assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b"), ("FSU-solo", "")}
|
||||
assert all(summary == REPLY for summary in stored.values())
|
||||
assert ("FSU-solo", "") not in stored, (
|
||||
"a single-doc folder is fully described by its one file line — no row"
|
||||
assert stored[("FSU-solo", "")] == REPLY, (
|
||||
"the single-doc source root is a candidate at the ≥ 1 rule"
|
||||
)
|
||||
|
||||
row = db.get(FolderSummary, ("FSU", "a/b"))
|
||||
@@ -467,6 +471,7 @@ def test_generate_happy_path_upserts_every_candidate_folder(
|
||||
"Folder: FSU",
|
||||
"Folder: FSU/a",
|
||||
"Folder: FSU/a/b",
|
||||
"Folder: FSU-solo",
|
||||
]
|
||||
# The recursive-subtree input: the a/ prompt carries a/b's docs too.
|
||||
a_prompt = llm.requests[1][1]
|
||||
@@ -475,7 +480,7 @@ def test_generate_happy_path_upserts_every_candidate_folder(
|
||||
assert "root.md — Root" not in a_prompt
|
||||
|
||||
assert (
|
||||
"folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0"
|
||||
"folder_summaries: generated=4 failed=0 pruned=0 kept_manual=0"
|
||||
in caplog.text
|
||||
), "the stats line must be greppable (PLAN §9 ample logging)"
|
||||
|
||||
@@ -492,16 +497,18 @@ def test_generate_per_folder_fail_soft_keeps_previous_and_lands_others(
|
||||
llm = _FakeLLM(fail_folders=("FSU/a/b",))
|
||||
with caplog.at_level(logging.ERROR, logger="app.rag.folder_summaries"):
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm))
|
||||
assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0}
|
||||
assert llm.calls == 3 # the failing folder was attempted too
|
||||
assert stats == {"generated": 3, "failed": 1, "pruned": 0, "kept_manual": 0}
|
||||
assert llm.calls == 4 # the failing folder was attempted too
|
||||
|
||||
stored = _rows(db)
|
||||
assert stored[("FSU", "a/b")] == "old summary", (
|
||||
"the previous row survives the per-folder failure"
|
||||
)
|
||||
assert stored[("FSU", "")] == REPLY and stored[("FSU", "a")] == REPLY, (
|
||||
"the other folders still land"
|
||||
)
|
||||
assert (
|
||||
stored[("FSU", "")] == REPLY
|
||||
and stored[("FSU", "a")] == REPLY
|
||||
and stored[("FSU-solo", "")] == REPLY
|
||||
), "the other folders still land"
|
||||
assert "folder summary failed for FSU/a/b" in caplog.text
|
||||
assert "simulated lite-model failure for FSU/a/b" in caplog.text
|
||||
|
||||
@@ -519,24 +526,25 @@ def test_generate_per_folder_fail_soft_without_previous_row_creates_nothing(
|
||||
|
||||
|
||||
def test_generate_prunes_stale_rows_and_keeps_live_ones(db: Session, clean_tables) -> None:
|
||||
"""Rows for folders that dropped below 2 recursive docs are deleted
|
||||
"""Rows for folders that VANISHED (0 recursive docs) are deleted
|
||||
(pruned/renamed — the summary would go stale); rows for folders
|
||||
that still qualify persist (an unchanged folder's summary is still
|
||||
true — regenerated in place)."""
|
||||
that still hold ≥ 1 doc persist (an unchanged folder's summary is
|
||||
still true — regenerated in place), the 1-doc FSU-solo root
|
||||
included (it is a candidate at the ≥ 1 rule)."""
|
||||
_seed_catalogue(db)
|
||||
# A stale row for a folder no longer in the catalogue (3→1 docs /
|
||||
# renamed away) + a live row with old content.
|
||||
# A stale row for a folder no longer in the catalogue (renamed
|
||||
# away — 0 docs) + live rows with old content.
|
||||
db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale"))
|
||||
db.add(FolderSummary(source="FSU", folder_path="a", summary="old a summary"))
|
||||
db.add(FolderSummary(source="FSU-solo", folder_path="", summary="solo stale"))
|
||||
db.commit()
|
||||
stats = asyncio.run(generate_folder_summaries(db, _FakeLLM()))
|
||||
assert stats["pruned"] == 2 # gone/old + the FSU-solo root (1 doc)
|
||||
assert stats["pruned"] == 1 # gone/old only (its folder vanished)
|
||||
|
||||
stored = _rows(db)
|
||||
assert ("FSU", "gone/old") not in stored, "the stale folder row must be pruned"
|
||||
assert ("FSU-solo", "") not in stored, (
|
||||
"a folder that dropped below 2 docs loses its row"
|
||||
assert stored[("FSU-solo", "")] == REPLY, (
|
||||
"the 1-doc root qualifies — regenerated, not pruned"
|
||||
)
|
||||
assert ("FSU", "a") in stored, "the still-qualifying folder keeps its row"
|
||||
assert stored[("FSU", "a")] == REPLY # regenerated, not stale
|
||||
@@ -614,7 +622,7 @@ def test_generate_only_flushes_caller_commits(db: Session, clean_tables) -> None
|
||||
)
|
||||
assert n == 3, "the caller's commit makes the flushed rows durable"
|
||||
|
||||
assert MIN_DOCS_PER_FOLDER == 2 # the ≥ 2 scope rule, pinned by name
|
||||
assert MIN_DOCS_PER_FOLDER == 1 # the ≥ 1 scope rule, pinned by name
|
||||
|
||||
|
||||
# ---------- manually_edited (phase 97, task 01) ----------
|
||||
@@ -644,17 +652,22 @@ def test_manual_row_survives_regeneration(
|
||||
with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"):
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm))
|
||||
|
||||
assert stats == {"generated": 2, "failed": 0, "pruned": 0, "kept_manual": 1}
|
||||
assert llm.calls == 2, "the manual folder burns zero lite calls"
|
||||
assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 1}
|
||||
assert llm.calls == 3, "the manual folder burns zero lite calls"
|
||||
assert [user.splitlines()[0] for _s, user in llm.requests] == [
|
||||
"Folder: FSU",
|
||||
"Folder: FSU/a/b",
|
||||
"Folder: FSU-solo",
|
||||
], "no prompt is ever built for the owner's folder"
|
||||
|
||||
stored = _rows(db)
|
||||
assert stored[("FSU", "a")] == manual_text, "the owner's text survives"
|
||||
assert _updated_at(db, "FSU", "a") == stamp_before, ("never re-stamped")
|
||||
assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY, (
|
||||
assert (
|
||||
stored[("FSU", "")] == REPLY
|
||||
and stored[("FSU", "a/b")] == REPLY
|
||||
and stored[("FSU-solo", "")] == REPLY
|
||||
), (
|
||||
"the non-manual candidates still regenerate (the flag is the difference)"
|
||||
)
|
||||
row = db.get(FolderSummary, ("FSU", "a"))
|
||||
@@ -662,19 +675,21 @@ def test_manual_row_survives_regeneration(
|
||||
"the generator never clears the flag"
|
||||
)
|
||||
assert (
|
||||
"folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1"
|
||||
"folder_summaries: generated=3 failed=0 pruned=0 kept_manual=1"
|
||||
in caplog.text
|
||||
), "the 4-field stats line must be greppable (PLAN §9 ample logging)"
|
||||
|
||||
|
||||
def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None:
|
||||
"""A manual row is NEVER pruned (phase 97, task 01): two folders
|
||||
drop below 2 documents — the MANUAL one keeps its row (owner
|
||||
content persists until cleared — the clear deletes it, so the next
|
||||
KB-changing sync regenerates an AI description) while the
|
||||
NON-manual twin loses its now-stale row; the flag is the only
|
||||
difference. A vanished folder's manual row is kept too, and its
|
||||
non-manual twin is pruned."""
|
||||
"""A manual row is NEVER pruned and NEVER overwritten (phase 97,
|
||||
task 01): a/ and b/ each drop to 1 recursive doc — still
|
||||
candidates at the ≥ 1 rule — so the MANUAL a/ row is SKIPPED
|
||||
(owner text kept, no re-stamp, no lite burn) while the NON-manual
|
||||
b/ twin regenerates in place; the vanished folders' rows show the
|
||||
prune rule — the manual gone/manual row persists (owner content
|
||||
until cleared — the clear deletes it, so the next KB-changing sync
|
||||
regenerates an AI description) and the non-manual gone/ai twin is
|
||||
pruned; the flag is the only difference in each pair."""
|
||||
_add_doc(db, "FSU", "a/one.md", "One")
|
||||
_add_doc(db, "FSU", "a/two.md", "Two")
|
||||
_add_doc(db, "FSU", "b/one.md", "B One")
|
||||
@@ -696,7 +711,8 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None:
|
||||
db.add(FolderSummary(source="FSU", folder_path="gone/ai", summary="stale ai"))
|
||||
db.commit()
|
||||
|
||||
# a/ and b/ each drop below the minimum (2 -> 1 recursive doc).
|
||||
# a/ and b/ each drop to 1 recursive doc — still candidates at
|
||||
# the ≥ 1 rule.
|
||||
db.execute(
|
||||
text(
|
||||
"DELETE FROM documents WHERE source = 'FSU'"
|
||||
@@ -708,14 +724,14 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None:
|
||||
llm = _FakeLLM()
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm))
|
||||
|
||||
assert stats == {"generated": 1, "failed": 0, "pruned": 2, "kept_manual": 0}
|
||||
assert llm.calls == 1, "only the surviving candidate (the root) regenerates"
|
||||
assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 1}
|
||||
assert llm.calls == 2, "root + the non-manual b/ regenerate; manual a/ burns nothing"
|
||||
stored = _rows(db)
|
||||
assert stored[("FSU", "a")] == manual_text, (
|
||||
"the manual row survives its folder dropping below the minimum"
|
||||
"the manual row is skipped — never overwritten, even at 1 doc"
|
||||
)
|
||||
assert ("FSU", "b") not in stored, (
|
||||
"the non-manual twin loses its stale row (the flag is the difference)"
|
||||
assert stored[("FSU", "b")] == REPLY, (
|
||||
"the non-manual twin regenerates — 1 doc still qualifies (the ≥ 1 rule)"
|
||||
)
|
||||
assert stored[("FSU", "gone/manual")] == "owner kept", (
|
||||
"a vanished folder's manual row is kept — owner content until cleared"
|
||||
@@ -731,15 +747,15 @@ def test_missing_fresh_table_is_exactly_the_candidate_set(
|
||||
db: Session, clean_tables
|
||||
) -> None:
|
||||
"""No stored rows → every candidate folder is a gap, sorted by
|
||||
``(source, folder_path)``; the single-doc FSU-solo root is not a
|
||||
candidate and can never be a gap."""
|
||||
``(source, folder_path)`` — the single-doc FSU-solo root
|
||||
included (a candidate at the ≥ 1 rule)."""
|
||||
_seed_catalogue(db)
|
||||
assert missing_folder_summaries(db) == [
|
||||
("FSU", ""),
|
||||
("FSU", "a"),
|
||||
("FSU", "a/b"),
|
||||
("FSU-solo", ""),
|
||||
]
|
||||
assert ("FSU-solo", "") not in missing_folder_summaries(db)
|
||||
|
||||
|
||||
def test_missing_fully_populated_table_is_empty(db: Session, clean_tables) -> None:
|
||||
@@ -770,16 +786,18 @@ def test_missing_empty_kb_empty_table_is_no_gap(db: Session, clean_tables) -> No
|
||||
assert missing_folder_summaries(db) == []
|
||||
|
||||
|
||||
def test_missing_single_doc_folder_is_never_listed(db: Session, clean_tables) -> None:
|
||||
"""A below-minimum folder without a row is NOT a gap — it is not a
|
||||
candidate (its one file line IS its summary)."""
|
||||
def test_missing_single_doc_folder_is_listed(db: Session, clean_tables) -> None:
|
||||
"""The ≥ 1 rule: a single-doc folder WITHOUT a row IS a gap (it is
|
||||
a candidate — the one file line no longer exempts it), alongside
|
||||
its source root (1 doc)."""
|
||||
_add_doc(db, "FSU", "solo/one.md", "One")
|
||||
assert missing_folder_summaries(db) == []
|
||||
assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "solo")]
|
||||
|
||||
|
||||
def test_missing_stale_row_is_not_a_gap(db: Session, clean_tables) -> None:
|
||||
"""A stored row for a folder that dropped below 2 docs is stale,
|
||||
not missing — the prune pass owns it, the gap detector ignores it."""
|
||||
"""A stored row for a folder that dropped below the minimum
|
||||
(vanished — 0 docs) is stale, not missing — the prune pass owns
|
||||
it, the gap detector ignores it."""
|
||||
_seed_catalogue(db)
|
||||
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
|
||||
db.commit()
|
||||
@@ -865,12 +883,16 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None
|
||||
db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale"))
|
||||
db.commit()
|
||||
a_stamp = _updated_at(db, "FSU", "a")
|
||||
assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "a/b")]
|
||||
assert missing_folder_summaries(db) == [
|
||||
("FSU", ""),
|
||||
("FSU", "a/b"),
|
||||
("FSU-solo", ""),
|
||||
]
|
||||
|
||||
llm = _FakeLLM()
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True))
|
||||
assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 0}
|
||||
assert llm.calls == 2
|
||||
assert stats == {"generated": 3, "failed": 0, "pruned": 1, "kept_manual": 0}
|
||||
assert llm.calls == 3
|
||||
|
||||
stored = _rows(db)
|
||||
assert ("FSU", "gone/old") not in stored, (
|
||||
@@ -878,7 +900,11 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None
|
||||
)
|
||||
assert stored[("FSU", "a")] == "keep me"
|
||||
assert _updated_at(db, "FSU", "a") == a_stamp
|
||||
assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY
|
||||
assert (
|
||||
stored[("FSU", "")] == REPLY
|
||||
and stored[("FSU", "a/b")] == REPLY
|
||||
and stored[("FSU-solo", "")] == REPLY
|
||||
)
|
||||
|
||||
|
||||
def test_only_missing_fail_soft_keeps_prior_and_lands_others(
|
||||
@@ -892,11 +918,12 @@ def test_only_missing_fail_soft_keeps_prior_and_lands_others(
|
||||
db.commit()
|
||||
llm = _FakeLLM(fail_folders=("FSU/a/b",))
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True))
|
||||
assert stats == {"generated": 1, "failed": 1, "pruned": 0, "kept_manual": 0}
|
||||
assert llm.calls == 2 # both missing folders were attempted
|
||||
assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0}
|
||||
assert llm.calls == 3 # all three missing folders were attempted
|
||||
|
||||
stored = _rows(db)
|
||||
assert stored[("FSU", "")] == REPLY, "the other missing folder still lands"
|
||||
assert stored[("FSU", "")] == REPLY, "the other missing folders still land"
|
||||
assert stored[("FSU-solo", "")] == REPLY, "the single-doc root still lands"
|
||||
assert ("FSU", "a/b") not in stored, "the failed folder stays absent"
|
||||
assert stored[("FSU", "a")] == "keep me", "the present row is untouched"
|
||||
|
||||
@@ -941,18 +968,19 @@ def test_on_progress_fires_once_per_candidate_in_sorted_key_order(
|
||||
"""Phase 98 (task 01): the hook fires once per candidate, in the
|
||||
same sorted ``(source, folder_path)`` order the folders are
|
||||
attempted — done climbs 1..total, total = the candidate count
|
||||
(the single-doc FSU-solo root is not a candidate — no event)."""
|
||||
(the single-doc FSU-solo root IS a candidate at the ≥ 1 rule)."""
|
||||
_seed_catalogue(db)
|
||||
llm = _FakeLLM()
|
||||
events, record = _record_progress()
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record))
|
||||
assert events == [
|
||||
(1, 3, "FSU", ""),
|
||||
(2, 3, "FSU", "a"),
|
||||
(3, 3, "FSU", "a/b"),
|
||||
(1, 4, "FSU", ""),
|
||||
(2, 4, "FSU", "a"),
|
||||
(3, 4, "FSU", "a/b"),
|
||||
(4, 4, "FSU-solo", ""),
|
||||
]
|
||||
assert llm.calls == 3 # one event per attempt, in the same order
|
||||
assert stats["generated"] == 3
|
||||
assert llm.calls == 4 # one event per attempt, in the same order
|
||||
assert stats["generated"] == 4
|
||||
|
||||
|
||||
def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> None:
|
||||
@@ -971,12 +999,13 @@ def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> No
|
||||
events, record = _record_progress()
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record))
|
||||
assert events == [
|
||||
(1, 3, "FSU", ""),
|
||||
(2, 3, "FSU", "a"), # the instant manual skip still advances
|
||||
(3, 3, "FSU", "a/b"),
|
||||
(1, 4, "FSU", ""),
|
||||
(2, 4, "FSU", "a"), # the instant manual skip still advances
|
||||
(3, 4, "FSU", "a/b"),
|
||||
(4, 4, "FSU-solo", ""),
|
||||
]
|
||||
assert llm.calls == 2, "the skip itself burns no call"
|
||||
assert stats["kept_manual"] == 1 and stats["generated"] == 2
|
||||
assert llm.calls == 3, "the skip itself burns no call"
|
||||
assert stats["kept_manual"] == 1 and stats["generated"] == 3
|
||||
|
||||
|
||||
def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> None:
|
||||
@@ -989,11 +1018,12 @@ def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> Non
|
||||
events, record = _record_progress()
|
||||
stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record))
|
||||
assert events == [
|
||||
(1, 3, "FSU", ""),
|
||||
(2, 3, "FSU", "a"), # the failed attempt still advances
|
||||
(3, 3, "FSU", "a/b"),
|
||||
(1, 4, "FSU", ""),
|
||||
(2, 4, "FSU", "a"), # the failed attempt still advances
|
||||
(3, 4, "FSU", "a/b"),
|
||||
(4, 4, "FSU-solo", ""),
|
||||
]
|
||||
assert stats["failed"] == 1 and stats["generated"] == 2
|
||||
assert stats["failed"] == 1 and stats["generated"] == 3
|
||||
|
||||
|
||||
def test_on_progress_only_missing_reports_the_missing_count(
|
||||
@@ -1049,7 +1079,7 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None
|
||||
generate_folder_summaries(db, hooked, on_progress=record)
|
||||
)
|
||||
requests_hooked = hooked.requests.copy()
|
||||
assert len(events) == 3, "the hooked run fired (the contrast is real)"
|
||||
assert len(events) == 4, "the hooked run fired (the contrast is real)"
|
||||
db.rollback() # the generator only flushes — drop the uncommitted rows
|
||||
|
||||
plain = _FakeLLM()
|
||||
@@ -1058,5 +1088,8 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None
|
||||
assert plain.requests == requests_hooked, "the fake LLM's call log is unchanged"
|
||||
assert stats_plain == stats_hooked
|
||||
assert _rows(db) == {
|
||||
("FSU", ""): REPLY, ("FSU", "a"): REPLY, ("FSU", "a/b"): REPLY
|
||||
("FSU", ""): REPLY,
|
||||
("FSU", "a"): REPLY,
|
||||
("FSU", "a/b"): REPLY,
|
||||
("FSU-solo", ""): REPLY,
|
||||
}
|
||||
|
||||
@@ -314,7 +314,7 @@ def test_cross_check_nested_level_matches_group_folder_listing() -> None:
|
||||
# --------------------------------------------------------------------
|
||||
# summary_pending (phase 98, task 03) — the D3 rule (ONE concept):
|
||||
# a SOURCE or FOLDER node is pending iff its recursive document count
|
||||
# ≥ MIN_DOCS_PER_FOLDER (2) AND it has NO stored folder_summaries row
|
||||
# ≥ MIN_DOCS_PER_FOLDER (1) AND it has NO stored folder_summaries row
|
||||
# (AI or manual — any row). That is exactly
|
||||
# ``app.rag.folder_summaries.missing_folder_summaries``'s candidate
|
||||
# set — the marker never drifts from the gap-fill (the integration
|
||||
@@ -359,10 +359,11 @@ def test_folder_with_stored_row_is_not_pending() -> None:
|
||||
assert source.summary_pending is True
|
||||
|
||||
|
||||
def test_single_document_folder_never_pending() -> None:
|
||||
"""A < 2-document folder is NEVER pending (it never gets a summary
|
||||
— its one file line IS its description), even with no stored row
|
||||
— while its ≥ 2-doc source root (no root row) still is."""
|
||||
def test_single_document_folder_is_pending() -> None:
|
||||
"""The ≥ 1 rule: a 1-document folder with NO stored row IS pending
|
||||
(its one file line no longer exempts it — the next sync
|
||||
summarizes it), as is its source root (2 docs, no root row).
|
||||
A registered 0-document source is the only never-pending case."""
|
||||
rows = [
|
||||
("S", "solo/only.md", "Only", 1, T0, C0), # 1-doc folder
|
||||
("S", "top.md", "Top", 1, T0, C1), # source total = 2
|
||||
@@ -370,7 +371,7 @@ def test_single_document_folder_never_pending() -> None:
|
||||
(source,) = build_kb_tree(["S"], rows, {})
|
||||
(solo,) = _folder_nodes(source)
|
||||
assert solo.documents == 1
|
||||
assert solo.summary_pending is False
|
||||
assert solo.summary_pending is True
|
||||
assert source.summary_pending is True
|
||||
|
||||
|
||||
@@ -407,7 +408,7 @@ def test_name_collision_pending_follows_recursive_count() -> None:
|
||||
|
||||
|
||||
def test_source_root_pending_and_zero_document_source_never() -> None:
|
||||
"""The source root: a source with ≥ 2 docs and NO ``(source, "")``
|
||||
"""The source root: a source with ≥ 1 docs and NO ``(source, "")``
|
||||
row → the SOURCE node is pending; the stored root row clears it.
|
||||
A registered 0-document source is NEVER pending (0 < the minimum —
|
||||
there is nothing to summarize), with or without a manual row."""
|
||||
|
||||
Reference in New Issue
Block a user