From 35d65d2f25457f0aa804c70104f27bdef4644563 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Mon, 14 Sep 2026 08:57:57 -0400 Subject: [PATCH] =?UTF-8?q?feat(rag):=20summarize=20single-document=20fold?= =?UTF-8?q?ers=20(MIN=5FDOCS=5FPER=5FFOLDER=202=20=E2=86=92=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relax the phase-94 folder-summary scope rule from ≥ 2 documents to ≥ 1: a folder (or source root) is a candidate while ANY document lives under it, so single-file folders and single-file source roots get their own lite-written description. A row is now pruned only when its folder loses its last document (vanishes from the catalogue). The constant is the single source of truth, so the flip propagates to the generator's candidate set, the prune pass, the missing_folder_summaries gap probe (the next sync self-heals the new gaps), and the KB-tree summary_pending markers (1-doc folders / sources now read "Summary pending" until their row lands). Docstrings/comments across app/, scripts/import_docs.py, and the E2E fixtures updated to the ≥ 1 wording. Unit + integration tests updated to the new semantics (the pruned-below-minimum scenario is now a folder losing its LAST doc; single-doc folders are pinned as candidates/pending). Full suite: 2314 passed, app coverage 99%; ruff + pyright clean; folder-summary E2E stories pass in isolation (ls_tree_drilldown, sync_summary_visibility, kb_tree, kb_tree_nav, document_dates, oneshot_llm_retry). --- app/api/docs.py | 18 +- app/api/sync.py | 3 +- app/models.py | 24 ++- app/rag/folder_summaries.py | 57 ++++-- app/schemas.py | 14 +- scripts/import_docs.py | 11 +- tests/e2e/test_kb_tree.py | 15 +- tests/e2e/test_ls_tree_drilldown.py | 14 +- tests/e2e/test_sync_summary_visibility.py | 6 +- tests/integration/test_docs_api.py | 38 ++-- .../integration/test_sync_folder_summaries.py | 103 +++++----- tests/unit/test_folder_summaries.py | 191 ++++++++++-------- tests/unit/test_kb_tree_builder.py | 15 +- 13 files changed, 297 insertions(+), 212 deletions(-) diff --git a/app/api/docs.py b/app/api/docs.py index e48e598..3ee3c98 100644 --- a/app/api/docs.py +++ b/app/api/docs.py @@ -576,7 +576,7 @@ def build_kb_tree( * **Pending** — ``summary_pending`` on the SOURCE and every FOLDER node (phase 98, decision D3 — ONE concept): true iff the node's recursive ``documents`` count ≥ - :data:`app.rag.folder_summaries.MIN_DOCS_PER_FOLDER` (2) AND it + :data:`app.rag.folder_summaries.MIN_DOCS_PER_FOLDER` (1) AND it has NO stored ``folder_summaries`` row (AI or manual — any row; the builder sees stored rows only). That is EXACTLY :func:`app.rag.folder_summaries.missing_folder_summaries`'s @@ -584,9 +584,10 @@ def build_kb_tree( keys on the next sync — the marker is honest: "waiting to generate", and the integration cross-check pins the tree's pending set to that function so the marker can never drift from - the gap-fill). A < 2-document folder is NEVER pending (it never - gets a summary — its one file line IS its description), and a - registered 0-document source never is. + the gap-fill). A 0-document folder cannot exist (a folder is a + catalogue prefix only) — every existing folder with no row is + pending, single-file folders included — and a registered + 0-document source never is. ONE concept end to end: the builder reuses :func:`app.rag.folder_summaries.folder_of` and the phase-94 @@ -674,10 +675,11 @@ def list_kb_tree( exactly as ``app/api/chat.py`` does) + the SAME outerjoin/grouped catalogue query ``GET /api/docs`` runs (the document ``id`` excluded — the tree has no document ids) + ALL stored - ``folder_summaries`` rows (a bounded select — one row per ≥ 2-doc - folder; rows for sources the tree does not list are never - referenced by the builder) — through the pure - :func:`build_kb_tree`. ``GET /api/docs`` itself is untouched. + ``folder_summaries`` rows (a bounded select — one row per + existing folder at the ≥ 1-doc rule; rows for sources the tree + does not list are never referenced by the builder) — through the + pure :func:`build_kb_tree`. ``GET /api/docs`` itself is + untouched. """ names = list_source_names(db) rows = db.execute( diff --git a/app/api/sync.py b/app/api/sync.py index 47e4063..e5959a8 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -353,7 +353,8 @@ async def _run_sync() -> None: # changed KB (added + updated > 0) is a full regeneration # (today's behavior, byte-identical); an unchanged re-sync # takes the GAP probe instead of the old table-empty one — a - # candidate folder (at least 2 docs) with no stored row: a gap + # candidate folder (at least 1 doc — every existing folder) + # with no stored row: a gap # fills ONLY the missing rows (only_missing=True — the # subsumed table-empty first-sync trigger included, where # every candidate is missing), a complete table burns zero diff --git a/app/models.py b/app/models.py index 3bb21eb..14c1ce9 100644 --- a/app/models.py +++ b/app/models.py @@ -65,8 +65,9 @@ Data model — see ``.agents/PLAN.md`` §Data Model: (``id = 1``); every column NULL = "use the default" (env value for the strings, the built-in palette for the colors — task 01). -* ``folder_summaries`` — one row per folder with ≥ 2 documents: - the sync-time ``lite`` summary the drill-down +* ``folder_summaries`` — one row per folder (recursive count + ≥ 1 — every existing folder): the sync-time + ``lite`` summary the drill-down ``ls`` shows next to each folder (phase 94; ``folder_path = ""`` = the source root). """ @@ -501,14 +502,15 @@ class FolderSummary(Base): ``documents.path`` (String(1000)) and is the source-relative folder prefix. ``folder_path = ""`` is the SOURCE ROOT — the top-level source summary (the whole source's recursive subtree). - * Rows exist only for folders with ≥ 2 documents (recursive count - — the same set the ``ls`` count rule counts): a - single-document folder is fully described by its one file line, - so no ``lite`` burn. After a changed sync, rows whose folder - dropped below 2 documents are pruned (a pruned/renamed folder's - summary would otherwise go stale); rows for folders that still - have ≥ 2 documents persist (an unchanged folder's summary is - still true). Both rules are generator policy (``app.rag. + * Rows exist for folders with ≥ 1 document (recursive count — + the same set the ``ls`` count rule counts): a folder is a + candidate while any document lives under it, single-file folders + and single-file source roots included. After a changed sync, + rows whose folder dropped below 1 document (vanished from the + catalogue) are pruned (a pruned/renamed folder's summary would + otherwise go stale); rows for folders that still hold at least + one document persist (an unchanged folder's summary is still + true). Both rules are generator policy (``app.rag. folder_summaries``), not schema constraints. * ``summary`` — the 1–3 sentence plain-text description the aipi ``lite`` model wrote at sync time (``FOLDER_SUMMARY_MODE``, @@ -521,7 +523,7 @@ class FolderSummary(Base): folder_summaries``) SKIPS a manual row on regeneration (no ``lite`` burn on owner text — counted ``kept_manual``) and never prunes it (owner content persists until cleared — even for a - folder below the 2-document minimum); an owner correction is + folder below the 1-document minimum); an owner correction is never silently rewritten (the phase-97 ``00_phase.md`` decision). Clearing the description deletes the row — the next KB-changing sync regenerates an AI description (the reset path). diff --git a/app/rag/folder_summaries.py b/app/rag/folder_summaries.py index 5cb6c5b..e476168 100644 --- a/app/rag/folder_summaries.py +++ b/app/rag/folder_summaries.py @@ -20,13 +20,16 @@ summary describes. Storage: ``folder_summaries`` (migration 0017) — PK ``(source, folder_path)``; ``folder_path = ""`` is the SOURCE ROOT -(the top-level source summary). Rows exist only for folders with -≥ 2 documents (the :data:`MIN_DOCS_PER_FOLDER` rule — a -single-document folder is fully described by its one file line, so no -``lite`` burn); after a changed sync, rows whose folder dropped below -2 documents are pruned (a pruned/renamed folder's summary would -otherwise go stale), while rows for folders that still have -≥ 2 documents persist (an unchanged folder's summary is still true). +(the top-level source summary). Rows exist for every folder whose +recursive subtree holds ≥ 1 document (the +:data:`MIN_DOCS_PER_FOLDER` rule — a folder is a catalogue prefix +only while some document lives under it, so EVERY existing folder is +a candidate, single-file folders and single-file source roots +included); after a changed sync, rows whose folder dropped below 1 +document (vanished from the catalogue) are pruned (a pruned/renamed +folder's summary would otherwise go stale), while rows for folders +that still hold at least one document persist (an unchanged folder's +summary is still true). Manual rows (phase 97, task 01): ``manually_edited`` (migration 0018) marks the descriptions the OWNER edited — ``PATCH /api/folders/ @@ -104,10 +107,13 @@ SYSTEM_PROMPT = f"{FOLDER_SUMMARY_MODE}: {FOLDER_SUMMARY_INSTRUCTION}" FOLDER_HEADER_PREFIX = "Folder: " #: A folder is summarized only while its recursive subtree holds at -#: least this many documents — a single-document folder is fully -#: described by its one file line, so no ``lite`` burn (phase 94 -#: ``00_phase.md`` scope rule; the prune rule applies the same count). -MIN_DOCS_PER_FOLDER = 2 +#: least this many documents — 1: EVERY folder the catalogue knows +#: (a folder exists only while some document lives under it, so the +#: filter admits every group :func:`group_by_folder` returns, +#: single-file folders included) gets a row (the phase-94 ≥ 2 scope +#: rule relaxed by owner decision; the prune rule applies the same +#: count — a row is pruned when its folder loses its last document). +MIN_DOCS_PER_FOLDER = 1 #: One document row for the grouping/prompting: #: ``(source, path, title, summary)`` — the ``app.rag.overview`` @@ -164,8 +170,10 @@ def group_by_folder(rows: Sequence[DocRow]) -> dict[tuple[str, str], list[DocRow catalogue query). Returns ``{(source, folder_path): [rows]}`` — group lists keep the input (catalogue) order, so downstream prompt building is deterministic. Groups of ANY size (≥ 1) are returned; - the ≥ 2 :data:`MIN_DOCS_PER_FOLDER` rule is applied by - :func:`generate_folder_summaries`, not here. + at the :data:`MIN_DOCS_PER_FOLDER` = 1 rule every group is + already a candidate, so :func:`generate_folder_summaries`'s + filter is an inert safety net (it would bite only if the minimum + were ever raised above 1), not a behavioral gate. """ # Pass 1: the distinct TRUE folder prefixes of the catalogue — a @@ -336,7 +344,9 @@ detector and the fill can never disagree about what the catalogue def _candidates(rows: Sequence[DocRow]) -> dict[tuple[str, str], list[DocRow]]: - """The generator's candidate map (recursive subtree ≥ 2 docs). + """The generator's candidate map (recursive subtree ≥ + :data:`MIN_DOCS_PER_FOLDER` docs — at the current minimum of 1, + every :func:`group_by_folder` group). The :func:`group_by_folder` groups filtered by :data:`MIN_DOCS_PER_FOLDER` — the EXACT set a full regeneration @@ -379,10 +389,11 @@ gap persisted until the next KB change. This function names the gap: Returns the missing keys sorted by ``(source, folder_path)``; ``[]`` when there is no gap — including an empty catalogue over an - empty table (no candidates, no gap). A single-document folder is - never listed (it is not a candidate), and a stored row for a folder - that dropped below the minimum is NOT missing (it is stale — the - prune pass owns it). + empty table (no candidates, no gap). A stored row for a folder + that dropped below the minimum (vanished — 0 documents) is NOT + missing (it is stale — the prune pass owns it); a single-document + folder without a row IS listed (it is a candidate at the ≥ 1 + rule). """ return sorted( key for key in _candidates(_catalog_rows(db)) if key not in _stored_keys(db) @@ -408,8 +419,9 @@ async def generate_folder_summaries( 2. Group the document catalogue by :func:`group_by_folder` (the recursive-subtree concept) and keep the candidate folders — the ones whose recursive subtree holds - :data:`MIN_DOCS_PER_FOLDER` (≥ 2) documents. Single-document - folders get no row (their one file line IS their summary). + :data:`MIN_DOCS_PER_FOLDER` (1) documents: at the current + minimum every existing folder (single-file ones included, and + the single-file source root) is a candidate. 3. For each candidate (deterministic ``(source, folder_path)`` order) call :func:`summarize_folder` and UPSERT — fail-soft PER FOLDER: one folder's :class:`LLMError` is logged and counted, @@ -430,8 +442,9 @@ async def generate_folder_summaries( staleness is the changed-KB regeneration's job) and no ``lite`` call is burned for a folder that already has a summary — the unchanged-sync self-heal fill (phase 96, task 02). - 4. DELETE rows whose folder no longer has ≥ 2 documents — a - pruned/renamed folder's summary goes stale and is dropped — + 4. DELETE rows whose folder no longer has ≥ 1 document (vanished + from the catalogue) — a pruned/renamed folder's summary goes + stale and is dropped — EXCEPT a manual row: owner content persists until the owner clears it, even for a folder that dropped below the minimum (phase 97, task 01; the clear deletes the row, so the next diff --git a/app/schemas.py b/app/schemas.py index ffa8ea1..b048545 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -288,12 +288,13 @@ class KbTreeFolder(BaseModel): resolves it with ``from __future__ import annotations``). ``summary_pending`` (phase 98, D3 — ONE concept): true iff this - folder's recursive count ≥ ``MIN_DOCS_PER_FOLDER`` (2) AND it has + folder's recursive count ≥ ``MIN_DOCS_PER_FOLDER`` (1) AND it has NO stored ``folder_summaries`` row (AI or manual — any row) — exactly the candidate the sync-time gap-fill regenerates (the :func:`app.rag.folder_summaries.missing_folder_summaries` set). - A < 2-document folder is never pending (it never gets a summary — - its one file line IS its description). + A 0-document folder cannot exist (a folder is a catalogue prefix + only), so every existing folder with no stored row is pending — + single-file folders included. """ kind: Literal["folder"] = "folder" @@ -325,12 +326,13 @@ class KbTreeSource(BaseModel): ``summary_pending`` (phase 98, D3 — ONE concept): true iff the source's recursive ``documents`` count ≥ ``MIN_DOCS_PER_FOLDER`` - (2) AND no stored ``(source, "")`` row (AI or manual — any row) — + (1) AND no stored ``(source, "")`` row (AI or manual — any row) — exactly the source-root candidate the sync-time gap-fill regenerates (the :func:`app.rag.folder_summaries.missing_folder_summaries` set). A registered 0-document source is never pending (there is nothing - to summarize). + to summarize); a single-document source IS pending while its root + row is absent. """ name: str @@ -353,7 +355,7 @@ class KbTree(BaseModel): Every SOURCE and FOLDER node carries ``summary_pending`` (phase 98, D3): true iff its recursive document count ≥ - ``MIN_DOCS_PER_FOLDER`` (2) AND it has no stored ``folder_summaries`` + ``MIN_DOCS_PER_FOLDER`` (1) AND it has no stored ``folder_summaries`` row — exactly ``missing_folder_summaries``'s candidate set (the marker never drifts from the gap-fill); FILE nodes carry no flag (the file table has no description column). diff --git a/scripts/import_docs.py b/scripts/import_docs.py index 8dd05c1..d5be4ac 100644 --- a/scripts/import_docs.py +++ b/scripts/import_docs.py @@ -68,7 +68,8 @@ The stored folder summaries (phase 94 — the drill-down ``ls``'s per-level descriptions, the ``folder_summaries`` table) regenerate in the same run under the same gate: a changed KB (added + updated > 0 — full regeneration), or, after an unchanged walk, a GAP — a candidate -folder (≥ 2 docs) with no stored row (phase 96: this subsumes the old +folder (≥ 1 doc — every existing folder) with no stored row +(phase 96: this subsumes the old table-empty trigger exactly — an empty table leaves EVERY candidate missing, as after the first full run after migration 0017 or a ``--limit`` first walk that skipped them — and catches the single row @@ -282,7 +283,8 @@ def _overview_row_exists() -> bool: def _folder_summaries_gap() -> list[tuple[str, str]]: """The folder-summary gaps (phase 96, task 03): the candidate - folders (≥ 2 docs) with no stored row, sorted. + folders (≥ 1 doc — every existing folder) with no stored row, + sorted. The unchanged-walk self-heal trigger, replacing the phase-94 table-emptiness probe (which the gap subsumes exactly: an empty @@ -358,8 +360,9 @@ def main(argv: list[str] | None = None) -> int: The folder summaries (phase 94, task 02; phase 96, task 03) follow the same gate — a changed KB (full regeneration), or, - after an unchanged walk, a GAP: a candidate folder (≥ 2 docs) - with no stored row (the subsumed table-empty trigger — an + after an unchanged walk, a GAP: a candidate folder (≥ 1 doc — + every existing folder) with no stored row (the subsumed + table-empty trigger — an empty table leaves every candidate missing — plus a row an exhausted one-shot retry lost) — with per-folder fail-soft inside the generator (a ``lite`` failure keeps the failed diff --git a/tests/e2e/test_kb_tree.py b/tests/e2e/test_kb_tree.py index 39a279d..c65b6fb 100644 --- a/tests/e2e/test_kb_tree.py +++ b/tests/e2e/test_kb_tree.py @@ -26,7 +26,9 @@ real-Sync pattern; no git anywhere): Total: 7 documents; alpha counts 5 (root + 2 + 2), beta counts 2. Every stored description is deterministic: the mock's EXISTING ``FOLDER_SUMMARY_MODE`` branch (phase 94 — no mock changes needed) -stores, per ≥ 2-doc folder, the canned one-liner naming the folder, +stores, per existing folder (the ≥ 1 rule — this fixture holds no +single-doc folders, so the pinned rows are unchanged), the canned +one-liner naming the folder, ``Fixture folder summary for [/].`` — the ``synced_kb`` module fixture pins those exact rows (all with ``manually_edited = false``) after the sync, and the tests assert on @@ -154,8 +156,9 @@ TOTAL_DOCS = ALPHA_COUNT + BETA_COUNT #: The sync-time folder descriptions the mock's canned #: ``FOLDER_SUMMARY_MODE`` branch stores (the phase-94 byte-stable #: template — the one-liner names the folder), in ``(source, -#: folder_path)`` order: one row per ≥ 2-doc folder (the -#: recursive-subtree rule) — the ``""`` rows are the source roots. +#: folder_path)`` order: one row per existing folder (the ≥ 1-doc +#: recursive-subtree rule — this fixture holds no single-doc folders) +#: — the ``""`` rows are the source roots. SUMMARY_FOR = "Fixture folder summary for {}." ALPHA_ROOT_SUM = SUMMARY_FOR.format(ALPHA) ONE_SUM = SUMMARY_FOR.format(f"{ALPHA}/one") @@ -382,7 +385,8 @@ def synced_kb(app_server: str, kb_tree_dirs: tuple[Path, Path]) -> None: walk → chunk → embed → overview → folder summaries → version bump), and pins the stored folder descriptions: the mock's canned ``FOLDER_SUMMARY_MODE`` branch (phase 94) makes the sync store one - deterministic row per ≥ 2-doc folder — the tests assert on that + deterministic row per existing folder (the ≥ 1-doc rule — this + fixture holds no single-doc folders) — the tests assert on that exact text (and on the phase-97 ``manually_edited`` flag: every stored row starts out AI-written). """ @@ -409,7 +413,8 @@ def synced_kb(app_server: str, kb_tree_dirs: tuple[Path, Path]) -> None: assert detail["pruned"] == 0, detail assert detail["overview"] is True, detail # The change-gated folder descriptions (phase 94) landed: one row - # per ≥ 2-doc folder, the mock's byte-stable text — and every row + # per existing folder (the ≥ 1-doc rule — this fixture holds no + # single-doc folders), the mock's byte-stable text — and every row # AI-written (the ``manually_edited`` flag starts false). with SessionLocal() as db: rows = db.execute( diff --git a/tests/e2e/test_ls_tree_drilldown.py b/tests/e2e/test_ls_tree_drilldown.py index 3d4a4b1..89a0260 100644 --- a/tests/e2e/test_ls_tree_drilldown.py +++ b/tests/e2e/test_ls_tree_drilldown.py @@ -42,7 +42,8 @@ of the mock's cosine distribution, and the ```` section is present (the flow's precondition). The mock's canned ``FOLDER_SUMMARY_MODE`` branch (phase 94 task 01) -stores, per ≥ 2-doc folder, the deterministic one-liner +stores, per existing folder (the ≥ 1-doc rule — this fixture holds +no single-doc folders), the deterministic one-liner ``Fixture folder summary for [/].`` — the ``synced_kb`` module fixture pins those exact rows in ``folder_summaries`` after the sync, and the drill answers assert on them in the ``ls`` output. @@ -159,8 +160,9 @@ assert "\n" not in TWO_A_CONTENT[:80] # the quote must stay one line #: The sync-time folder summaries the mock's canned ``FOLDER_SUMMARY_MODE`` #: branch stores (task 01's byte-stable template), in -#: ``(source, folder_path)`` order: one row per ≥ 2-doc folder (the -#: recursive-subtree rule) — the ``""`` rows are the source roots. +#: ``(source, folder_path)`` order: one row per existing folder (the +#: ≥ 1-doc recursive-subtree rule — this fixture holds no single-doc +#: folders) — the ``""`` rows are the source roots. EXPECTED_SUMMARIES: list[tuple[str, str, str]] = [ (ALPHA, "", f"Fixture folder summary for {ALPHA}."), (ALPHA, "one", f"Fixture folder summary for {ALPHA}/one."), @@ -426,7 +428,8 @@ def synced_kb(app_server: str, drill_dirs: tuple[Path, Path]) -> None: walk → chunk → embed → overview → folder summaries → version bump), and pins the stored folder summaries: the mock's canned ``FOLDER_SUMMARY_MODE`` branch (task 01) makes the sync store one - deterministic row per ≥ 2-doc folder — the drill turns' answers + deterministic row per existing folder (the ≥ 1-doc rule — this + fixture holds no single-doc folders) — the drill turns' answers assert on that exact text. """ alpha, beta = drill_dirs @@ -452,7 +455,8 @@ def synced_kb(app_server: str, drill_dirs: tuple[Path, Path]) -> None: assert detail["pruned"] == 0, detail assert detail["overview"] is True, detail # The change-gated folder summaries (phase 94 task 02) landed: one - # row per ≥ 2-doc folder, the mock's byte-stable text (the drill + # row per existing folder (the ≥ 1-doc rule — this fixture holds + # no single-doc folders), the mock's byte-stable text (the drill # answers quote exactly these lines). with SessionLocal() as db: rows = db.execute( diff --git a/tests/e2e/test_sync_summary_visibility.py b/tests/e2e/test_sync_summary_visibility.py index add32d5..6f81ffd 100644 --- a/tests/e2e/test_sync_summary_visibility.py +++ b/tests/e2e/test_sync_summary_visibility.py @@ -46,7 +46,9 @@ the same host) registered as a ``kind=local`` source (the pattern; no git anywhere): ``syncsum/`` with TWO ≥ 2-doc folders — ``alpha/`` (2 docs) and ``bravo/`` (2 docs), no root-level files. The generator's candidate set is therefore exactly 3: the source root -(``folder_path ""``) + the two folders (the recursive-subtree ≥ 2 rule). +(``folder_path ""``) + the two folders (the recursive-subtree ≥ 1 +rule — this fixture holds no single-doc folders, so the rule's +relaxation is inert for it). Autouse cleanup (the phase-96 pattern): before each test wait for no running sync on EITHER app, then truncate the shared registry + KB @@ -160,7 +162,7 @@ FOLDER_B = "bravo" FOLDERS = (FOLDER_A, FOLDER_B) N_FILES = 4 # two folders × 2 docs -N_CANDIDATES = 3 # the source root + the two folders (the ≥ 2-doc rule) +N_CANDIDATES = 3 # the source root + the two folders (the ≥ 1-doc rule) #: The mock's byte-stable ``FOLDER_SUMMARY_MODE`` lines for this fixture #: (the phase-94 template — the label is the ``FOLDER_HEADER_PREFIX`` diff --git a/tests/integration/test_docs_api.py b/tests/integration/test_docs_api.py index 73886c8..efbaf16 100644 --- a/tests/integration/test_docs_api.py +++ b/tests/integration/test_docs_api.py @@ -310,9 +310,9 @@ def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> Non assert (alpha["documents"], alpha["children"], alpha["summary"]) == (0, [], None) assert alpha["summary_pending"] is False # 0 documents — never pending assert midx["documents"] == 1 - assert midx["summary_pending"] is False # 1 document — below the minimum + assert midx["summary_pending"] is True # 1 document — the ≥ 1 minimum, no row assert zeta["documents"] == 1 - assert zeta["summary_pending"] is False # 1 document — below the minimum + assert zeta["summary_pending"] is True # 1 document — the ≥ 1 minimum, no row assert [c["path"] for c in midx["children"]] == ["m1.md"] _truncate_tree_tables(db) @@ -321,16 +321,16 @@ def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> Non def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db) -> None: """The endpoint returns ``summary_pending`` on SOURCE + FOLDER nodes (phase 98, task 03 — D3): true iff the recursive count is - ≥ 2 AND no stored row; false WITH a stored row (any — the endpoint - cannot tell AI from manual); false for a < 2-document folder - (never pending) — including one NESTED. File nodes carry no flag.""" + ≥ 1 AND no stored row — the NESTED 1-doc folder included (the ≥ 1 + rule); false WITH a stored row (any — the endpoint cannot tell AI + from manual). File nodes carry no flag.""" _truncate_tree_tables(db) base = datetime.now(UTC) db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) # k8s → 3 documents (talos + cluster + charts), NO stored row → pending _seed_doc(db, "Homelab", "k8s/talos.md", "Talos", 1, base) _seed_doc(db, "Homelab", "k8s/cluster.md", "Cluster", 1, base) - # k8s/helm → 1 document — below the 2-doc minimum, never pending + # k8s/helm → 1 document — a candidate at the ≥ 1 rule (no row) _seed_doc(db, "Homelab", "k8s/helm/charts.md", "Charts", 1, base) # wiki → 2 documents, WITH a stored row → not pending _seed_doc(db, "Homelab", "wiki/one.md", "One", 1, base) @@ -360,7 +360,7 @@ def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db) assert k8s["summary_pending"] is True # 3 docs, no stored row helm = k8s["children"][0] assert (helm["kind"], helm["path"]) == ("folder", "k8s/helm") - assert helm["summary_pending"] is False # 1 doc — never pending + assert helm["summary_pending"] is True # 1 doc — pending at the ≥ 1 rule assert wiki["summary"] == "Wiki pages." assert wiki["summary_pending"] is False # 2 docs, but a stored row covers it # File nodes carry no pending flag at all (the file table has no @@ -397,8 +397,9 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db) added_at=base + timedelta(hours=2), ) ) - # Alpha: a/b holds 2 docs, c holds 1 (NEVER a candidate), the root - # holds 4 — candidates (Alpha, ""), (Alpha, "a"), (Alpha, "a/b"). + # Alpha: a/b holds 2 docs, c holds 1 (a candidate at the ≥ 1 + # rule), the root holds 4 — candidates (Alpha, ""), (Alpha, "a"), + # (Alpha, "a/b"), (Alpha, "c"). _seed_doc(db, "Alpha", "a/b/c1.md", "C1", 1, base) _seed_doc(db, "Alpha", "a/b/c2.md", "C2", 1, base) _seed_doc(db, "Alpha", "c/solo.md", "Solo", 1, base) @@ -416,6 +417,7 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db) ("Alpha", ""): "Alpha root.", ("Alpha", "a"): "Alpha a.", ("Alpha", "a/b"): "Alpha a b.", + ("Alpha", "c"): "Alpha c.", ("Beta", ""): "Beta root.", ("Beta", "x"): "Beta x.", }.items() @@ -441,8 +443,10 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db) # And the explicit expectation (the test is readable without the # helper): exactly the two deleted keys, root riding "". assert pending == {("Alpha", "a"), ("Beta", "")} - # Never flagged: the < 2-doc folder (Alpha/c), the 0-document - # registered source (Gamma), and every node that still holds a row. + # Never flagged: the 0-document registered source (Gamma) and every + # node that still holds a row — incl. the 1-doc Alpha/c, whose + # stored row covers it (the ≥ 1 rule makes it a candidate, but a + # candidate WITH a row is not missing). assert ("Alpha", "c") not in pending assert not any(name == "Gamma" for name, _ in pending) assert ("Alpha", "a/b") not in pending @@ -464,8 +468,8 @@ OLDER_STAMP = datetime(2020, 1, 1, tzinfo=UTC) def _seed_folder_pair(db, source: str, folder: str, base: datetime) -> None: - """Two documents under ``source/folder`` — the ≥ 2-document - minimum a folder must hold to be summarizable.""" + """Two documents under ``source/folder`` — a summarizable folder + (≥ 1 doc at the current minimum; two keep the pair realistic).""" _seed_doc(db, source, f"{folder}/one.md", f"{folder} one", 1, base) _seed_doc(db, source, f"{folder}/two.md", f"{folder} two", 2, base) @@ -543,15 +547,17 @@ def test_folder_summary_create_where_no_row_exists( admin_client: TestClient, db ) -> None: """A manual description can be CREATED where no row exists — a - < 2-document folder the generator never wrote (or its fail-soft - miss): insert, not update, with ``manually_edited = true``.""" + folder the generator has not written yet (its fail-soft miss, or + a pre-sync state): insert, not update, with + ``manually_edited = true``.""" _truncate_tree_tables(db) base = datetime.now(UTC) db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) _seed_doc(db, "Homelab", "solo/only.md", "Only", 1, base) # 1-doc folder db.commit() db.expire_all() - assert _get_folder_row(db, "Homelab", "solo") is None # no AI row for < 2 docs + # No AI row yet (the generator has not run in this test). + assert _get_folder_row(db, "Homelab", "solo") is None r = admin_client.patch( "/api/folders/summary", diff --git a/tests/integration/test_sync_folder_summaries.py b/tests/integration/test_sync_folder_summaries.py index 75814c5..b3be71f 100644 --- a/tests/integration/test_sync_folder_summaries.py +++ b/tests/integration/test_sync_folder_summaries.py @@ -11,9 +11,9 @@ convention — the generator only flushes, the sync path commits). Script path (``scripts.import_docs.main`` end to end, fake LLM, real DB, explicit ``--source``): -- a KB-changing import → one row per ≥ 2-doc subtree (the source root - + the 2-doc folder; the 1-doc folder gets none), committed in the - run's transaction, the summary line ending +- a KB-changing import → one row per existing subtree (the source + root, the 2-doc folder a, AND the 1-doc folder b — the ≥ 1 rule), + committed in the run's transaction, the summary line ending ``folder_summaries=//`` (unchanged by phase 96 — no gap-fill suffix on a full regeneration); - an unchanged re-import with a COMPLETE table → zero ``lite`` calls, @@ -27,10 +27,10 @@ DB, explicit ``--source``): failed folder summary self-heals on the next sync); - a KB change on a second run → still a FULL regeneration (call count == candidate count, every row re-stamped, no gap-fill suffix); -- a subtree dropping below 2 docs after a changed re-walk → its row +- a subtree losing its LAST doc after a changed re-walk → its row pruned; - one folder's ``lite`` failure → its previous row kept, the other - lands, exit code 0, stats ``1/1/0``; + land, exit code 0, stats ``2/1/0``; - a ``--limit`` debug run → no generation, no rows, ``folder_summaries=skipped``; - a fresh (empty) table after a ``--limit`` first walk → an unchanged @@ -189,7 +189,8 @@ def _fresh_sync_state() -> Iterator[None]: @pytest.fixture() def src(tmp_path: Path) -> Path: - """MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1. + """MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1 + (b/ is a candidate too — the ≥ 1 rule). md → no document-summary chat calls, so the ``lite`` traffic is exactly the overview + the folder summaries.""" @@ -235,9 +236,9 @@ def test_changed_import_generates_folder_rows( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - """A KB-changing import upserts one row per ≥ 2-doc subtree in the - run's own transaction — committed and visible afterwards — with the - stats on the summary line (PLAN §9).""" + """A KB-changing import upserts one row per existing subtree in + the run's own transaction — committed and visible afterwards — + with the stats on the summary line (PLAN §9).""" llm = FakeEmbedder() records: list[logging.LogRecord] = [] @@ -257,25 +258,26 @@ def test_changed_import_generates_folder_rows( assert rc == 0 assert "added=3" in out assert out.rstrip().endswith( - "overview=updated sources_version=1 folder_summaries=2/0/0" + "overview=updated sources_version=1 folder_summaries=3/0/0" ) - # Two FOLDER_SUMMARY_MODE calls — the source root (3 docs) and the - # 2-doc folder a; the 1-doc folder b gets no row (no lite burn). + # Three FOLDER_SUMMARY_MODE calls — the source root (3 docs), the + # 2-doc folder a, and the 1-doc folder b (the ≥ 1 rule: every + # existing folder is a candidate). calls = _folder_calls(llm) - assert len(calls) == 2 + assert len(calls) == 3 headers = [c[1]["content"].splitlines()[0] for c in calls] - assert headers == ["Folder: MyDocs", "Folder: MyDocs/a"] + assert headers == ["Folder: MyDocs", "Folder: MyDocs/a", "Folder: MyDocs/b"] for c in calls: assert "FOLDER_SUMMARY_MODE" in c[0]["content"] # The rows land committed (the run's own session committed them) — - # one per ≥ 2-doc subtree, never empty, stamped. + # one per existing subtree, never empty, stamped. rows = _rows(db) - assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} assert all(rows.values()) assert _updated_at(db, "MyDocs", "a") is not None # The stats log line (PLAN §9 ample logging). assert any( - "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=0" + "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0" in r.getMessage() for r in records ) @@ -293,10 +295,10 @@ def test_unchanged_reimport_burns_zero_folder_calls( rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 assert out.rstrip().endswith( - "overview=updated sources_version=1 folder_summaries=2/0/0" + "overview=updated sources_version=1 folder_summaries=3/0/0" ) rows = _rows(db) - assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} llm2 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys) @@ -326,7 +328,7 @@ def test_unchanged_reimport_with_gap_fills_only_the_missing_row( rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 rows_before = _rows(db) - assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} root_stamp_before = _updated_at(db, "MyDocs", "") assert root_stamp_before is not None @@ -387,13 +389,15 @@ def test_changed_reimport_is_a_full_regeneration( assert "updated=1" in out # Full-regeneration token — the stats without the gap-fill suffix. assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=2/0/0" + "overview=updated sources_version=2 folder_summaries=3/0/0" ) - # Call count == candidate count — BOTH folders, not a targeted fill. + # Call count == candidate count — ALL existing folders, not a + # targeted fill. calls = _folder_calls(llm2) assert [c[1]["content"].splitlines()[0] for c in calls] == [ "Folder: MyDocs", "Folder: MyDocs/a", + "Folder: MyDocs/b", ] # All rows re-stamped (the full regeneration re-writes every # candidate, even the unchanging one). @@ -404,35 +408,37 @@ def test_changed_reimport_is_a_full_regeneration( assert _rows(db) == rows_before # deterministic fake → same texts -def test_subtree_dropping_below_two_docs_is_pruned( +def test_subtree_losing_its_last_doc_is_pruned( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - """A subtree that drops below the ≥ 2 rule on a changed re-walk - loses its (stale) row; the remaining qualifiers regenerate.""" + """A subtree that loses its LAST doc (0 recursive docs — below the + ≥ 1 minimum) on a changed re-walk loses its (stale) row; the + remaining qualifiers (≥ 1 doc — the 1-doc b/ included) regenerate.""" llm1 = FakeEmbedder() rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 - assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} - # a/ loses one of its two docs (below the ≥ 2 rule) while b's doc - # changes → the re-walk is a KB change, so generation runs and the - # stale a/ row is pruned; b (1 doc) still gets no row. + # a/ loses BOTH its docs (the folder vanishes — 0 docs) while b's + # doc changes → the re-walk is a KB change, so generation runs and + # the stale a/ row is pruned; b (1 doc) keeps qualifying. + (src / "a" / "one.md").unlink() (src / "a" / "two.md").unlink() (src / "b" / "three.md").write_text("# B Three\nChanged content.\n", encoding="utf-8") llm2 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm2, ["--source", str(src), "--prune"], capsys) assert rc == 0 - assert "pruned=1" in out + assert "pruned=2" in out # the two a/ documents left the index assert "updated=1" in out assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=1/0/1" + "overview=updated sources_version=2 folder_summaries=2/0/1" ) - # Only the source root (the 2 remaining docs) still qualifies. - assert set(_rows(db)) == {("MyDocs", "")} + # The source root (1 remaining doc) and b/ (1 doc) still qualify. + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "b")} assert _updated_at(db, "MyDocs", "a") is None # the row is gone @@ -443,8 +449,8 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green( capsys: pytest.CaptureFixture[str], ) -> None: """One folder's ``lite`` failure: its previous row is kept, the - other folder lands, and the run's exit code stays 0 — the stats - carry the failure (``1/1/0``).""" + other folders land, and the run's exit code stays 0 — the stats + carry the failure (``2/1/0``).""" llm1 = FakeEmbedder() rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 @@ -462,7 +468,7 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green( assert rc == 0 # a failed folder must not fail the import assert "updated=1" in out assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=1/1/0" + "overview=updated sources_version=2 folder_summaries=2/1/0" ) rows_after = _rows(db) assert rows_after["MyDocs", "a"] == rows_before["MyDocs", "a"] # kept @@ -484,14 +490,15 @@ def test_changed_import_never_overwrites_a_manual_row( — ``kept_manual`` is a stat, not a token), the manual row's text, stamp, and flag are untouched, and the other folders regenerate (the ``kept_manual`` stat lands on the generator's log line).""" - # First sync: full generation — root + a/ (b/ holds 1 doc: none). + # First sync: full generation — root + a/ + b/ (the ≥ 1 rule: b/ + # holds 1 doc — still a candidate). llm1 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 assert out.rstrip().endswith( - "overview=updated sources_version=1 folder_summaries=2/0/0" + "overview=updated sources_version=1 folder_summaries=3/0/0" ) - assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} # The owner edits the source-root description (task 03's PATCH is # the writer; task 01 pins the generator's behavior, so the row is @@ -533,11 +540,15 @@ def test_changed_import_never_overwrites_a_manual_row( assert "added=1" in out # The token STAYS 3 fields — kept_manual is a stat, not a token. assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=1/0/0" + "overview=updated sources_version=2 folder_summaries=2/0/0" ) - # Zero folder calls for the owner's folder — only a/ (now 3 docs). + # Zero folder calls for the owner's folder — a/ (now 3 docs) and + # b/ regenerate. calls = _folder_calls(llm2) - assert [c[1]["content"].splitlines()[0] for c in calls] == ["Folder: MyDocs/a"] + assert [c[1]["content"].splitlines()[0] for c in calls] == [ + "Folder: MyDocs/a", + "Folder: MyDocs/b", + ] # The owner's text, stamp, and flag are untouched ... rows_after = _rows(db) assert rows_after[("MyDocs", "")] == manual_text @@ -555,7 +566,7 @@ def test_changed_import_never_overwrites_a_manual_row( assert rows_after[("MyDocs", "a")] is not None # The 4-field stats line carries the skip (PLAN §9 ample logging). assert any( - "folder_summaries: generated=1 failed=0 pruned=0 kept_manual=1" + "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1" in r.getMessage() for r in records ) @@ -610,10 +621,10 @@ def test_empty_table_generates_on_unchanged_walk( # `` (gap-fill)`` (the generated set is the full candidate set). assert out.rstrip().endswith( "overview=updated sources_version=skipped " - "folder_summaries=2/0/0 (gap-fill)" + "folder_summaries=3/0/0 (gap-fill)" ) - assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} - assert len(_folder_calls(llm2)) == 2 + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} + assert len(_folder_calls(llm2)) == 3 assert current_sources_version(db) == 0 # the unchanged walk never bumps diff --git a/tests/unit/test_folder_summaries.py b/tests/unit/test_folder_summaries.py index 3bba5c9..d82bd33 100644 --- a/tests/unit/test_folder_summaries.py +++ b/tests/unit/test_folder_summaries.py @@ -161,9 +161,11 @@ def test_group_by_folder_nested_multi_source_recursive_subtree() -> None: def test_group_by_folder_single_doc_folder_is_a_group_too() -> None: - """Grouping is pure subtree membership (≥ 1 docs): the ≥ 2 rule is - the GENERATOR's (the recursive count below the minimum yields no - row — pinned by the generator tests, not the grouping).""" + """Grouping is pure subtree membership: at the + :data:`MIN_DOCS_PER_FOLDER` = 1 rule every group (≥ 1 docs) is + already a generator candidate, so grouping and candidacy agree + (the generator's filter is an inert safety net — pinned by the + generator tests, not the grouping).""" rows = [("S", "a/only.md", "O", None)] groups = group_by_folder(rows) assert len(groups[("S", "a")]) == 1 # present, but below the minimum @@ -417,8 +419,9 @@ def _rows(db: Session) -> dict[tuple[str, str], str]: def _seed_catalogue(db: Session) -> None: """The shared catalogue: FSU has four docs in three candidate - folders (root 4, a 3, a/b 2 — all ≥ the minimum); FSU-solo has one - doc (its root folder is below the minimum — no row, no call).""" + folders (root 4, a 3, a/b 2); FSU-solo has one doc (its root + folder is a candidate too — the ≥ 1 rule: a single-file source + root gets a row).""" _add_doc(db, "FSU", "a/b/one.md", "One", "One lead.\nSource: FSU/a/b/one.md") _add_doc(db, "FSU", "a/b/two.md", "Two") _add_doc(db, "FSU", "a/three.md", "Three") @@ -436,22 +439,23 @@ def clean_tables(db: Session): def test_generate_happy_path_upserts_every_candidate_folder( db: Session, clean_tables, caplog: pytest.LogCaptureFixture ) -> None: - """Every folder with ≥ 2 recursive docs gets a row (the source root - row included — ``folder_path = ''``); single-doc folders get none; - rows are stamped fresh; the stats dict and the log line are right; - folders are processed in deterministic (source, folder_path) order.""" + """Every folder with ≥ 1 recursive doc gets a row (the source root + row included — ``folder_path = ''`` — and the single-doc FSU-solo + root: the ≥ 1 rule); rows are stamped fresh; the stats dict and + the log line are right; folders are processed in deterministic + (source, folder_path) order.""" _seed_catalogue(db) llm = _FakeLLM() with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 0} - assert llm.calls == 3, "one lite call per candidate folder (the solo folder: none)" + assert stats == {"generated": 4, "failed": 0, "pruned": 0, "kept_manual": 0} + assert llm.calls == 4, "one lite call per candidate folder (the solo root: too)" stored = _rows(db) - assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b")} + assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b"), ("FSU-solo", "")} assert all(summary == REPLY for summary in stored.values()) - assert ("FSU-solo", "") not in stored, ( - "a single-doc folder is fully described by its one file line — no row" + assert stored[("FSU-solo", "")] == REPLY, ( + "the single-doc source root is a candidate at the ≥ 1 rule" ) row = db.get(FolderSummary, ("FSU", "a/b")) @@ -467,6 +471,7 @@ def test_generate_happy_path_upserts_every_candidate_folder( "Folder: FSU", "Folder: FSU/a", "Folder: FSU/a/b", + "Folder: FSU-solo", ] # The recursive-subtree input: the a/ prompt carries a/b's docs too. a_prompt = llm.requests[1][1] @@ -475,7 +480,7 @@ def test_generate_happy_path_upserts_every_candidate_folder( assert "root.md — Root" not in a_prompt assert ( - "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0" + "folder_summaries: generated=4 failed=0 pruned=0 kept_manual=0" in caplog.text ), "the stats line must be greppable (PLAN §9 ample logging)" @@ -492,16 +497,18 @@ def test_generate_per_folder_fail_soft_keeps_previous_and_lands_others( llm = _FakeLLM(fail_folders=("FSU/a/b",)) with caplog.at_level(logging.ERROR, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0} - assert llm.calls == 3 # the failing folder was attempted too + assert stats == {"generated": 3, "failed": 1, "pruned": 0, "kept_manual": 0} + assert llm.calls == 4 # the failing folder was attempted too stored = _rows(db) assert stored[("FSU", "a/b")] == "old summary", ( "the previous row survives the per-folder failure" ) - assert stored[("FSU", "")] == REPLY and stored[("FSU", "a")] == REPLY, ( - "the other folders still land" - ) + assert ( + stored[("FSU", "")] == REPLY + and stored[("FSU", "a")] == REPLY + and stored[("FSU-solo", "")] == REPLY + ), "the other folders still land" assert "folder summary failed for FSU/a/b" in caplog.text assert "simulated lite-model failure for FSU/a/b" in caplog.text @@ -519,24 +526,25 @@ def test_generate_per_folder_fail_soft_without_previous_row_creates_nothing( def test_generate_prunes_stale_rows_and_keeps_live_ones(db: Session, clean_tables) -> None: - """Rows for folders that dropped below 2 recursive docs are deleted + """Rows for folders that VANISHED (0 recursive docs) are deleted (pruned/renamed — the summary would go stale); rows for folders - that still qualify persist (an unchanged folder's summary is still - true — regenerated in place).""" + that still hold ≥ 1 doc persist (an unchanged folder's summary is + still true — regenerated in place), the 1-doc FSU-solo root + included (it is a candidate at the ≥ 1 rule).""" _seed_catalogue(db) - # A stale row for a folder no longer in the catalogue (3→1 docs / - # renamed away) + a live row with old content. + # A stale row for a folder no longer in the catalogue (renamed + # away — 0 docs) + live rows with old content. db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale")) db.add(FolderSummary(source="FSU", folder_path="a", summary="old a summary")) db.add(FolderSummary(source="FSU-solo", folder_path="", summary="solo stale")) db.commit() stats = asyncio.run(generate_folder_summaries(db, _FakeLLM())) - assert stats["pruned"] == 2 # gone/old + the FSU-solo root (1 doc) + assert stats["pruned"] == 1 # gone/old only (its folder vanished) stored = _rows(db) assert ("FSU", "gone/old") not in stored, "the stale folder row must be pruned" - assert ("FSU-solo", "") not in stored, ( - "a folder that dropped below 2 docs loses its row" + assert stored[("FSU-solo", "")] == REPLY, ( + "the 1-doc root qualifies — regenerated, not pruned" ) assert ("FSU", "a") in stored, "the still-qualifying folder keeps its row" assert stored[("FSU", "a")] == REPLY # regenerated, not stale @@ -614,7 +622,7 @@ def test_generate_only_flushes_caller_commits(db: Session, clean_tables) -> None ) assert n == 3, "the caller's commit makes the flushed rows durable" - assert MIN_DOCS_PER_FOLDER == 2 # the ≥ 2 scope rule, pinned by name + assert MIN_DOCS_PER_FOLDER == 1 # the ≥ 1 scope rule, pinned by name # ---------- manually_edited (phase 97, task 01) ---------- @@ -644,17 +652,22 @@ def test_manual_row_survives_regeneration( with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 2, "failed": 0, "pruned": 0, "kept_manual": 1} - assert llm.calls == 2, "the manual folder burns zero lite calls" + assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 1} + assert llm.calls == 3, "the manual folder burns zero lite calls" assert [user.splitlines()[0] for _s, user in llm.requests] == [ "Folder: FSU", "Folder: FSU/a/b", + "Folder: FSU-solo", ], "no prompt is ever built for the owner's folder" stored = _rows(db) assert stored[("FSU", "a")] == manual_text, "the owner's text survives" assert _updated_at(db, "FSU", "a") == stamp_before, ("never re-stamped") - assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY, ( + assert ( + stored[("FSU", "")] == REPLY + and stored[("FSU", "a/b")] == REPLY + and stored[("FSU-solo", "")] == REPLY + ), ( "the non-manual candidates still regenerate (the flag is the difference)" ) row = db.get(FolderSummary, ("FSU", "a")) @@ -662,19 +675,21 @@ def test_manual_row_survives_regeneration( "the generator never clears the flag" ) assert ( - "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1" + "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=1" in caplog.text ), "the 4-field stats line must be greppable (PLAN §9 ample logging)" def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None: - """A manual row is NEVER pruned (phase 97, task 01): two folders - drop below 2 documents — the MANUAL one keeps its row (owner - content persists until cleared — the clear deletes it, so the next - KB-changing sync regenerates an AI description) while the - NON-manual twin loses its now-stale row; the flag is the only - difference. A vanished folder's manual row is kept too, and its - non-manual twin is pruned.""" + """A manual row is NEVER pruned and NEVER overwritten (phase 97, + task 01): a/ and b/ each drop to 1 recursive doc — still + candidates at the ≥ 1 rule — so the MANUAL a/ row is SKIPPED + (owner text kept, no re-stamp, no lite burn) while the NON-manual + b/ twin regenerates in place; the vanished folders' rows show the + prune rule — the manual gone/manual row persists (owner content + until cleared — the clear deletes it, so the next KB-changing sync + regenerates an AI description) and the non-manual gone/ai twin is + pruned; the flag is the only difference in each pair.""" _add_doc(db, "FSU", "a/one.md", "One") _add_doc(db, "FSU", "a/two.md", "Two") _add_doc(db, "FSU", "b/one.md", "B One") @@ -696,7 +711,8 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None: db.add(FolderSummary(source="FSU", folder_path="gone/ai", summary="stale ai")) db.commit() - # a/ and b/ each drop below the minimum (2 -> 1 recursive doc). + # a/ and b/ each drop to 1 recursive doc — still candidates at + # the ≥ 1 rule. db.execute( text( "DELETE FROM documents WHERE source = 'FSU'" @@ -708,14 +724,14 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None: llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 1, "failed": 0, "pruned": 2, "kept_manual": 0} - assert llm.calls == 1, "only the surviving candidate (the root) regenerates" + assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 1} + assert llm.calls == 2, "root + the non-manual b/ regenerate; manual a/ burns nothing" stored = _rows(db) assert stored[("FSU", "a")] == manual_text, ( - "the manual row survives its folder dropping below the minimum" + "the manual row is skipped — never overwritten, even at 1 doc" ) - assert ("FSU", "b") not in stored, ( - "the non-manual twin loses its stale row (the flag is the difference)" + assert stored[("FSU", "b")] == REPLY, ( + "the non-manual twin regenerates — 1 doc still qualifies (the ≥ 1 rule)" ) assert stored[("FSU", "gone/manual")] == "owner kept", ( "a vanished folder's manual row is kept — owner content until cleared" @@ -731,15 +747,15 @@ def test_missing_fresh_table_is_exactly_the_candidate_set( db: Session, clean_tables ) -> None: """No stored rows → every candidate folder is a gap, sorted by - ``(source, folder_path)``; the single-doc FSU-solo root is not a - candidate and can never be a gap.""" + ``(source, folder_path)`` — the single-doc FSU-solo root + included (a candidate at the ≥ 1 rule).""" _seed_catalogue(db) assert missing_folder_summaries(db) == [ ("FSU", ""), ("FSU", "a"), ("FSU", "a/b"), + ("FSU-solo", ""), ] - assert ("FSU-solo", "") not in missing_folder_summaries(db) def test_missing_fully_populated_table_is_empty(db: Session, clean_tables) -> None: @@ -770,16 +786,18 @@ def test_missing_empty_kb_empty_table_is_no_gap(db: Session, clean_tables) -> No assert missing_folder_summaries(db) == [] -def test_missing_single_doc_folder_is_never_listed(db: Session, clean_tables) -> None: - """A below-minimum folder without a row is NOT a gap — it is not a - candidate (its one file line IS its summary).""" +def test_missing_single_doc_folder_is_listed(db: Session, clean_tables) -> None: + """The ≥ 1 rule: a single-doc folder WITHOUT a row IS a gap (it is + a candidate — the one file line no longer exempts it), alongside + its source root (1 doc).""" _add_doc(db, "FSU", "solo/one.md", "One") - assert missing_folder_summaries(db) == [] + assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "solo")] def test_missing_stale_row_is_not_a_gap(db: Session, clean_tables) -> None: - """A stored row for a folder that dropped below 2 docs is stale, - not missing — the prune pass owns it, the gap detector ignores it.""" + """A stored row for a folder that dropped below the minimum + (vanished — 0 docs) is stale, not missing — the prune pass owns + it, the gap detector ignores it.""" _seed_catalogue(db) asyncio.run(generate_folder_summaries(db, _FakeLLM())) db.commit() @@ -865,12 +883,16 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale")) db.commit() a_stamp = _updated_at(db, "FSU", "a") - assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "a/b")] + assert missing_folder_summaries(db) == [ + ("FSU", ""), + ("FSU", "a/b"), + ("FSU-solo", ""), + ] llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 0} - assert llm.calls == 2 + assert stats == {"generated": 3, "failed": 0, "pruned": 1, "kept_manual": 0} + assert llm.calls == 3 stored = _rows(db) assert ("FSU", "gone/old") not in stored, ( @@ -878,7 +900,11 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None ) assert stored[("FSU", "a")] == "keep me" assert _updated_at(db, "FSU", "a") == a_stamp - assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY + assert ( + stored[("FSU", "")] == REPLY + and stored[("FSU", "a/b")] == REPLY + and stored[("FSU-solo", "")] == REPLY + ) def test_only_missing_fail_soft_keeps_prior_and_lands_others( @@ -892,11 +918,12 @@ def test_only_missing_fail_soft_keeps_prior_and_lands_others( db.commit() llm = _FakeLLM(fail_folders=("FSU/a/b",)) stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 1, "failed": 1, "pruned": 0, "kept_manual": 0} - assert llm.calls == 2 # both missing folders were attempted + assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0} + assert llm.calls == 3 # all three missing folders were attempted stored = _rows(db) - assert stored[("FSU", "")] == REPLY, "the other missing folder still lands" + assert stored[("FSU", "")] == REPLY, "the other missing folders still land" + assert stored[("FSU-solo", "")] == REPLY, "the single-doc root still lands" assert ("FSU", "a/b") not in stored, "the failed folder stays absent" assert stored[("FSU", "a")] == "keep me", "the present row is untouched" @@ -941,18 +968,19 @@ def test_on_progress_fires_once_per_candidate_in_sorted_key_order( """Phase 98 (task 01): the hook fires once per candidate, in the same sorted ``(source, folder_path)`` order the folders are attempted — done climbs 1..total, total = the candidate count - (the single-doc FSU-solo root is not a candidate — no event).""" + (the single-doc FSU-solo root IS a candidate at the ≥ 1 rule).""" _seed_catalogue(db) llm = _FakeLLM() events, record = _record_progress() stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record)) assert events == [ - (1, 3, "FSU", ""), - (2, 3, "FSU", "a"), - (3, 3, "FSU", "a/b"), + (1, 4, "FSU", ""), + (2, 4, "FSU", "a"), + (3, 4, "FSU", "a/b"), + (4, 4, "FSU-solo", ""), ] - assert llm.calls == 3 # one event per attempt, in the same order - assert stats["generated"] == 3 + assert llm.calls == 4 # one event per attempt, in the same order + assert stats["generated"] == 4 def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> None: @@ -971,12 +999,13 @@ def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> No events, record = _record_progress() stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record)) assert events == [ - (1, 3, "FSU", ""), - (2, 3, "FSU", "a"), # the instant manual skip still advances - (3, 3, "FSU", "a/b"), + (1, 4, "FSU", ""), + (2, 4, "FSU", "a"), # the instant manual skip still advances + (3, 4, "FSU", "a/b"), + (4, 4, "FSU-solo", ""), ] - assert llm.calls == 2, "the skip itself burns no call" - assert stats["kept_manual"] == 1 and stats["generated"] == 2 + assert llm.calls == 3, "the skip itself burns no call" + assert stats["kept_manual"] == 1 and stats["generated"] == 3 def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> None: @@ -989,11 +1018,12 @@ def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> Non events, record = _record_progress() stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record)) assert events == [ - (1, 3, "FSU", ""), - (2, 3, "FSU", "a"), # the failed attempt still advances - (3, 3, "FSU", "a/b"), + (1, 4, "FSU", ""), + (2, 4, "FSU", "a"), # the failed attempt still advances + (3, 4, "FSU", "a/b"), + (4, 4, "FSU-solo", ""), ] - assert stats["failed"] == 1 and stats["generated"] == 2 + assert stats["failed"] == 1 and stats["generated"] == 3 def test_on_progress_only_missing_reports_the_missing_count( @@ -1049,7 +1079,7 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None generate_folder_summaries(db, hooked, on_progress=record) ) requests_hooked = hooked.requests.copy() - assert len(events) == 3, "the hooked run fired (the contrast is real)" + assert len(events) == 4, "the hooked run fired (the contrast is real)" db.rollback() # the generator only flushes — drop the uncommitted rows plain = _FakeLLM() @@ -1058,5 +1088,8 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None assert plain.requests == requests_hooked, "the fake LLM's call log is unchanged" assert stats_plain == stats_hooked assert _rows(db) == { - ("FSU", ""): REPLY, ("FSU", "a"): REPLY, ("FSU", "a/b"): REPLY + ("FSU", ""): REPLY, + ("FSU", "a"): REPLY, + ("FSU", "a/b"): REPLY, + ("FSU-solo", ""): REPLY, } diff --git a/tests/unit/test_kb_tree_builder.py b/tests/unit/test_kb_tree_builder.py index 4d2d440..b4a1a68 100644 --- a/tests/unit/test_kb_tree_builder.py +++ b/tests/unit/test_kb_tree_builder.py @@ -314,7 +314,7 @@ def test_cross_check_nested_level_matches_group_folder_listing() -> None: # -------------------------------------------------------------------- # summary_pending (phase 98, task 03) — the D3 rule (ONE concept): # a SOURCE or FOLDER node is pending iff its recursive document count -# ≥ MIN_DOCS_PER_FOLDER (2) AND it has NO stored folder_summaries row +# ≥ MIN_DOCS_PER_FOLDER (1) AND it has NO stored folder_summaries row # (AI or manual — any row). That is exactly # ``app.rag.folder_summaries.missing_folder_summaries``'s candidate # set — the marker never drifts from the gap-fill (the integration @@ -359,10 +359,11 @@ def test_folder_with_stored_row_is_not_pending() -> None: assert source.summary_pending is True -def test_single_document_folder_never_pending() -> None: - """A < 2-document folder is NEVER pending (it never gets a summary - — its one file line IS its description), even with no stored row - — while its ≥ 2-doc source root (no root row) still is.""" +def test_single_document_folder_is_pending() -> None: + """The ≥ 1 rule: a 1-document folder with NO stored row IS pending + (its one file line no longer exempts it — the next sync + summarizes it), as is its source root (2 docs, no root row). + A registered 0-document source is the only never-pending case.""" rows = [ ("S", "solo/only.md", "Only", 1, T0, C0), # 1-doc folder ("S", "top.md", "Top", 1, T0, C1), # source total = 2 @@ -370,7 +371,7 @@ def test_single_document_folder_never_pending() -> None: (source,) = build_kb_tree(["S"], rows, {}) (solo,) = _folder_nodes(source) assert solo.documents == 1 - assert solo.summary_pending is False + assert solo.summary_pending is True assert source.summary_pending is True @@ -407,7 +408,7 @@ def test_name_collision_pending_follows_recursive_count() -> None: def test_source_root_pending_and_zero_document_source_never() -> None: - """The source root: a source with ≥ 2 docs and NO ``(source, "")`` + """The source root: a source with ≥ 1 docs and NO ``(source, "")`` row → the SOURCE node is pending; the stored root row clears it. A registered 0-document source is NEVER pending (0 < the minimum — there is nothing to summarize), with or without a manual row."""