diff --git a/app/api/docs.py b/app/api/docs.py index e48e598..3ee3c98 100644 --- a/app/api/docs.py +++ b/app/api/docs.py @@ -576,7 +576,7 @@ def build_kb_tree( * **Pending** — ``summary_pending`` on the SOURCE and every FOLDER node (phase 98, decision D3 — ONE concept): true iff the node's recursive ``documents`` count ≥ - :data:`app.rag.folder_summaries.MIN_DOCS_PER_FOLDER` (2) AND it + :data:`app.rag.folder_summaries.MIN_DOCS_PER_FOLDER` (1) AND it has NO stored ``folder_summaries`` row (AI or manual — any row; the builder sees stored rows only). That is EXACTLY :func:`app.rag.folder_summaries.missing_folder_summaries`'s @@ -584,9 +584,10 @@ def build_kb_tree( keys on the next sync — the marker is honest: "waiting to generate", and the integration cross-check pins the tree's pending set to that function so the marker can never drift from - the gap-fill). A < 2-document folder is NEVER pending (it never - gets a summary — its one file line IS its description), and a - registered 0-document source never is. + the gap-fill). A 0-document folder cannot exist (a folder is a + catalogue prefix only) — every existing folder with no row is + pending, single-file folders included — and a registered + 0-document source never is. ONE concept end to end: the builder reuses :func:`app.rag.folder_summaries.folder_of` and the phase-94 @@ -674,10 +675,11 @@ def list_kb_tree( exactly as ``app/api/chat.py`` does) + the SAME outerjoin/grouped catalogue query ``GET /api/docs`` runs (the document ``id`` excluded — the tree has no document ids) + ALL stored - ``folder_summaries`` rows (a bounded select — one row per ≥ 2-doc - folder; rows for sources the tree does not list are never - referenced by the builder) — through the pure - :func:`build_kb_tree`. ``GET /api/docs`` itself is untouched. + ``folder_summaries`` rows (a bounded select — one row per + existing folder at the ≥ 1-doc rule; rows for sources the tree + does not list are never referenced by the builder) — through the + pure :func:`build_kb_tree`. ``GET /api/docs`` itself is + untouched. """ names = list_source_names(db) rows = db.execute( diff --git a/app/api/sync.py b/app/api/sync.py index 47e4063..e5959a8 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -353,7 +353,8 @@ async def _run_sync() -> None: # changed KB (added + updated > 0) is a full regeneration # (today's behavior, byte-identical); an unchanged re-sync # takes the GAP probe instead of the old table-empty one — a - # candidate folder (at least 2 docs) with no stored row: a gap + # candidate folder (at least 1 doc — every existing folder) + # with no stored row: a gap # fills ONLY the missing rows (only_missing=True — the # subsumed table-empty first-sync trigger included, where # every candidate is missing), a complete table burns zero diff --git a/app/models.py b/app/models.py index 3bb21eb..14c1ce9 100644 --- a/app/models.py +++ b/app/models.py @@ -65,8 +65,9 @@ Data model — see ``.agents/PLAN.md`` §Data Model: (``id = 1``); every column NULL = "use the default" (env value for the strings, the built-in palette for the colors — task 01). -* ``folder_summaries`` — one row per folder with ≥ 2 documents: - the sync-time ``lite`` summary the drill-down +* ``folder_summaries`` — one row per folder (recursive count + ≥ 1 — every existing folder): the sync-time + ``lite`` summary the drill-down ``ls`` shows next to each folder (phase 94; ``folder_path = ""`` = the source root). """ @@ -501,14 +502,15 @@ class FolderSummary(Base): ``documents.path`` (String(1000)) and is the source-relative folder prefix. ``folder_path = ""`` is the SOURCE ROOT — the top-level source summary (the whole source's recursive subtree). - * Rows exist only for folders with ≥ 2 documents (recursive count - — the same set the ``ls`` count rule counts): a - single-document folder is fully described by its one file line, - so no ``lite`` burn. After a changed sync, rows whose folder - dropped below 2 documents are pruned (a pruned/renamed folder's - summary would otherwise go stale); rows for folders that still - have ≥ 2 documents persist (an unchanged folder's summary is - still true). Both rules are generator policy (``app.rag. + * Rows exist for folders with ≥ 1 document (recursive count — + the same set the ``ls`` count rule counts): a folder is a + candidate while any document lives under it, single-file folders + and single-file source roots included. After a changed sync, + rows whose folder dropped below 1 document (vanished from the + catalogue) are pruned (a pruned/renamed folder's summary would + otherwise go stale); rows for folders that still hold at least + one document persist (an unchanged folder's summary is still + true). Both rules are generator policy (``app.rag. folder_summaries``), not schema constraints. * ``summary`` — the 1–3 sentence plain-text description the aipi ``lite`` model wrote at sync time (``FOLDER_SUMMARY_MODE``, @@ -521,7 +523,7 @@ class FolderSummary(Base): folder_summaries``) SKIPS a manual row on regeneration (no ``lite`` burn on owner text — counted ``kept_manual``) and never prunes it (owner content persists until cleared — even for a - folder below the 2-document minimum); an owner correction is + folder below the 1-document minimum); an owner correction is never silently rewritten (the phase-97 ``00_phase.md`` decision). Clearing the description deletes the row — the next KB-changing sync regenerates an AI description (the reset path). diff --git a/app/rag/folder_summaries.py b/app/rag/folder_summaries.py index 5cb6c5b..e476168 100644 --- a/app/rag/folder_summaries.py +++ b/app/rag/folder_summaries.py @@ -20,13 +20,16 @@ summary describes. Storage: ``folder_summaries`` (migration 0017) — PK ``(source, folder_path)``; ``folder_path = ""`` is the SOURCE ROOT -(the top-level source summary). Rows exist only for folders with -≥ 2 documents (the :data:`MIN_DOCS_PER_FOLDER` rule — a -single-document folder is fully described by its one file line, so no -``lite`` burn); after a changed sync, rows whose folder dropped below -2 documents are pruned (a pruned/renamed folder's summary would -otherwise go stale), while rows for folders that still have -≥ 2 documents persist (an unchanged folder's summary is still true). +(the top-level source summary). Rows exist for every folder whose +recursive subtree holds ≥ 1 document (the +:data:`MIN_DOCS_PER_FOLDER` rule — a folder is a catalogue prefix +only while some document lives under it, so EVERY existing folder is +a candidate, single-file folders and single-file source roots +included); after a changed sync, rows whose folder dropped below 1 +document (vanished from the catalogue) are pruned (a pruned/renamed +folder's summary would otherwise go stale), while rows for folders +that still hold at least one document persist (an unchanged folder's +summary is still true). Manual rows (phase 97, task 01): ``manually_edited`` (migration 0018) marks the descriptions the OWNER edited — ``PATCH /api/folders/ @@ -104,10 +107,13 @@ SYSTEM_PROMPT = f"{FOLDER_SUMMARY_MODE}: {FOLDER_SUMMARY_INSTRUCTION}" FOLDER_HEADER_PREFIX = "Folder: " #: A folder is summarized only while its recursive subtree holds at -#: least this many documents — a single-document folder is fully -#: described by its one file line, so no ``lite`` burn (phase 94 -#: ``00_phase.md`` scope rule; the prune rule applies the same count). -MIN_DOCS_PER_FOLDER = 2 +#: least this many documents — 1: EVERY folder the catalogue knows +#: (a folder exists only while some document lives under it, so the +#: filter admits every group :func:`group_by_folder` returns, +#: single-file folders included) gets a row (the phase-94 ≥ 2 scope +#: rule relaxed by owner decision; the prune rule applies the same +#: count — a row is pruned when its folder loses its last document). +MIN_DOCS_PER_FOLDER = 1 #: One document row for the grouping/prompting: #: ``(source, path, title, summary)`` — the ``app.rag.overview`` @@ -164,8 +170,10 @@ def group_by_folder(rows: Sequence[DocRow]) -> dict[tuple[str, str], list[DocRow catalogue query). Returns ``{(source, folder_path): [rows]}`` — group lists keep the input (catalogue) order, so downstream prompt building is deterministic. Groups of ANY size (≥ 1) are returned; - the ≥ 2 :data:`MIN_DOCS_PER_FOLDER` rule is applied by - :func:`generate_folder_summaries`, not here. + at the :data:`MIN_DOCS_PER_FOLDER` = 1 rule every group is + already a candidate, so :func:`generate_folder_summaries`'s + filter is an inert safety net (it would bite only if the minimum + were ever raised above 1), not a behavioral gate. """ # Pass 1: the distinct TRUE folder prefixes of the catalogue — a @@ -336,7 +344,9 @@ detector and the fill can never disagree about what the catalogue def _candidates(rows: Sequence[DocRow]) -> dict[tuple[str, str], list[DocRow]]: - """The generator's candidate map (recursive subtree ≥ 2 docs). + """The generator's candidate map (recursive subtree ≥ + :data:`MIN_DOCS_PER_FOLDER` docs — at the current minimum of 1, + every :func:`group_by_folder` group). The :func:`group_by_folder` groups filtered by :data:`MIN_DOCS_PER_FOLDER` — the EXACT set a full regeneration @@ -379,10 +389,11 @@ gap persisted until the next KB change. This function names the gap: Returns the missing keys sorted by ``(source, folder_path)``; ``[]`` when there is no gap — including an empty catalogue over an - empty table (no candidates, no gap). A single-document folder is - never listed (it is not a candidate), and a stored row for a folder - that dropped below the minimum is NOT missing (it is stale — the - prune pass owns it). + empty table (no candidates, no gap). A stored row for a folder + that dropped below the minimum (vanished — 0 documents) is NOT + missing (it is stale — the prune pass owns it); a single-document + folder without a row IS listed (it is a candidate at the ≥ 1 + rule). """ return sorted( key for key in _candidates(_catalog_rows(db)) if key not in _stored_keys(db) @@ -408,8 +419,9 @@ async def generate_folder_summaries( 2. Group the document catalogue by :func:`group_by_folder` (the recursive-subtree concept) and keep the candidate folders — the ones whose recursive subtree holds - :data:`MIN_DOCS_PER_FOLDER` (≥ 2) documents. Single-document - folders get no row (their one file line IS their summary). + :data:`MIN_DOCS_PER_FOLDER` (1) documents: at the current + minimum every existing folder (single-file ones included, and + the single-file source root) is a candidate. 3. For each candidate (deterministic ``(source, folder_path)`` order) call :func:`summarize_folder` and UPSERT — fail-soft PER FOLDER: one folder's :class:`LLMError` is logged and counted, @@ -430,8 +442,9 @@ async def generate_folder_summaries( staleness is the changed-KB regeneration's job) and no ``lite`` call is burned for a folder that already has a summary — the unchanged-sync self-heal fill (phase 96, task 02). - 4. DELETE rows whose folder no longer has ≥ 2 documents — a - pruned/renamed folder's summary goes stale and is dropped — + 4. DELETE rows whose folder no longer has ≥ 1 document (vanished + from the catalogue) — a pruned/renamed folder's summary goes + stale and is dropped — EXCEPT a manual row: owner content persists until the owner clears it, even for a folder that dropped below the minimum (phase 97, task 01; the clear deletes the row, so the next diff --git a/app/schemas.py b/app/schemas.py index ffa8ea1..b048545 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -288,12 +288,13 @@ class KbTreeFolder(BaseModel): resolves it with ``from __future__ import annotations``). ``summary_pending`` (phase 98, D3 — ONE concept): true iff this - folder's recursive count ≥ ``MIN_DOCS_PER_FOLDER`` (2) AND it has + folder's recursive count ≥ ``MIN_DOCS_PER_FOLDER`` (1) AND it has NO stored ``folder_summaries`` row (AI or manual — any row) — exactly the candidate the sync-time gap-fill regenerates (the :func:`app.rag.folder_summaries.missing_folder_summaries` set). - A < 2-document folder is never pending (it never gets a summary — - its one file line IS its description). + A 0-document folder cannot exist (a folder is a catalogue prefix + only), so every existing folder with no stored row is pending — + single-file folders included. """ kind: Literal["folder"] = "folder" @@ -325,12 +326,13 @@ class KbTreeSource(BaseModel): ``summary_pending`` (phase 98, D3 — ONE concept): true iff the source's recursive ``documents`` count ≥ ``MIN_DOCS_PER_FOLDER`` - (2) AND no stored ``(source, "")`` row (AI or manual — any row) — + (1) AND no stored ``(source, "")`` row (AI or manual — any row) — exactly the source-root candidate the sync-time gap-fill regenerates (the :func:`app.rag.folder_summaries.missing_folder_summaries` set). A registered 0-document source is never pending (there is nothing - to summarize). + to summarize); a single-document source IS pending while its root + row is absent. """ name: str @@ -353,7 +355,7 @@ class KbTree(BaseModel): Every SOURCE and FOLDER node carries ``summary_pending`` (phase 98, D3): true iff its recursive document count ≥ - ``MIN_DOCS_PER_FOLDER`` (2) AND it has no stored ``folder_summaries`` + ``MIN_DOCS_PER_FOLDER`` (1) AND it has no stored ``folder_summaries`` row — exactly ``missing_folder_summaries``'s candidate set (the marker never drifts from the gap-fill); FILE nodes carry no flag (the file table has no description column). diff --git a/scripts/import_docs.py b/scripts/import_docs.py index 8dd05c1..d5be4ac 100644 --- a/scripts/import_docs.py +++ b/scripts/import_docs.py @@ -68,7 +68,8 @@ The stored folder summaries (phase 94 — the drill-down ``ls``'s per-level descriptions, the ``folder_summaries`` table) regenerate in the same run under the same gate: a changed KB (added + updated > 0 — full regeneration), or, after an unchanged walk, a GAP — a candidate -folder (≥ 2 docs) with no stored row (phase 96: this subsumes the old +folder (≥ 1 doc — every existing folder) with no stored row +(phase 96: this subsumes the old table-empty trigger exactly — an empty table leaves EVERY candidate missing, as after the first full run after migration 0017 or a ``--limit`` first walk that skipped them — and catches the single row @@ -282,7 +283,8 @@ def _overview_row_exists() -> bool: def _folder_summaries_gap() -> list[tuple[str, str]]: """The folder-summary gaps (phase 96, task 03): the candidate - folders (≥ 2 docs) with no stored row, sorted. + folders (≥ 1 doc — every existing folder) with no stored row, + sorted. The unchanged-walk self-heal trigger, replacing the phase-94 table-emptiness probe (which the gap subsumes exactly: an empty @@ -358,8 +360,9 @@ def main(argv: list[str] | None = None) -> int: The folder summaries (phase 94, task 02; phase 96, task 03) follow the same gate — a changed KB (full regeneration), or, - after an unchanged walk, a GAP: a candidate folder (≥ 2 docs) - with no stored row (the subsumed table-empty trigger — an + after an unchanged walk, a GAP: a candidate folder (≥ 1 doc — + every existing folder) with no stored row (the subsumed + table-empty trigger — an empty table leaves every candidate missing — plus a row an exhausted one-shot retry lost) — with per-folder fail-soft inside the generator (a ``lite`` failure keeps the failed diff --git a/tests/e2e/test_kb_tree.py b/tests/e2e/test_kb_tree.py index 39a279d..c65b6fb 100644 --- a/tests/e2e/test_kb_tree.py +++ b/tests/e2e/test_kb_tree.py @@ -26,7 +26,9 @@ real-Sync pattern; no git anywhere): Total: 7 documents; alpha counts 5 (root + 2 + 2), beta counts 2. Every stored description is deterministic: the mock's EXISTING ``FOLDER_SUMMARY_MODE`` branch (phase 94 — no mock changes needed) -stores, per ≥ 2-doc folder, the canned one-liner naming the folder, +stores, per existing folder (the ≥ 1 rule — this fixture holds no +single-doc folders, so the pinned rows are unchanged), the canned +one-liner naming the folder, ``Fixture folder summary for [/].`` — the ``synced_kb`` module fixture pins those exact rows (all with ``manually_edited = false``) after the sync, and the tests assert on @@ -154,8 +156,9 @@ TOTAL_DOCS = ALPHA_COUNT + BETA_COUNT #: The sync-time folder descriptions the mock's canned #: ``FOLDER_SUMMARY_MODE`` branch stores (the phase-94 byte-stable #: template — the one-liner names the folder), in ``(source, -#: folder_path)`` order: one row per ≥ 2-doc folder (the -#: recursive-subtree rule) — the ``""`` rows are the source roots. +#: folder_path)`` order: one row per existing folder (the ≥ 1-doc +#: recursive-subtree rule — this fixture holds no single-doc folders) +#: — the ``""`` rows are the source roots. SUMMARY_FOR = "Fixture folder summary for {}." ALPHA_ROOT_SUM = SUMMARY_FOR.format(ALPHA) ONE_SUM = SUMMARY_FOR.format(f"{ALPHA}/one") @@ -382,7 +385,8 @@ def synced_kb(app_server: str, kb_tree_dirs: tuple[Path, Path]) -> None: walk → chunk → embed → overview → folder summaries → version bump), and pins the stored folder descriptions: the mock's canned ``FOLDER_SUMMARY_MODE`` branch (phase 94) makes the sync store one - deterministic row per ≥ 2-doc folder — the tests assert on that + deterministic row per existing folder (the ≥ 1-doc rule — this + fixture holds no single-doc folders) — the tests assert on that exact text (and on the phase-97 ``manually_edited`` flag: every stored row starts out AI-written). """ @@ -409,7 +413,8 @@ def synced_kb(app_server: str, kb_tree_dirs: tuple[Path, Path]) -> None: assert detail["pruned"] == 0, detail assert detail["overview"] is True, detail # The change-gated folder descriptions (phase 94) landed: one row - # per ≥ 2-doc folder, the mock's byte-stable text — and every row + # per existing folder (the ≥ 1-doc rule — this fixture holds no + # single-doc folders), the mock's byte-stable text — and every row # AI-written (the ``manually_edited`` flag starts false). with SessionLocal() as db: rows = db.execute( diff --git a/tests/e2e/test_ls_tree_drilldown.py b/tests/e2e/test_ls_tree_drilldown.py index 3d4a4b1..89a0260 100644 --- a/tests/e2e/test_ls_tree_drilldown.py +++ b/tests/e2e/test_ls_tree_drilldown.py @@ -42,7 +42,8 @@ of the mock's cosine distribution, and the ```` section is present (the flow's precondition). The mock's canned ``FOLDER_SUMMARY_MODE`` branch (phase 94 task 01) -stores, per ≥ 2-doc folder, the deterministic one-liner +stores, per existing folder (the ≥ 1-doc rule — this fixture holds +no single-doc folders), the deterministic one-liner ``Fixture folder summary for [/].`` — the ``synced_kb`` module fixture pins those exact rows in ``folder_summaries`` after the sync, and the drill answers assert on them in the ``ls`` output. @@ -159,8 +160,9 @@ assert "\n" not in TWO_A_CONTENT[:80] # the quote must stay one line #: The sync-time folder summaries the mock's canned ``FOLDER_SUMMARY_MODE`` #: branch stores (task 01's byte-stable template), in -#: ``(source, folder_path)`` order: one row per ≥ 2-doc folder (the -#: recursive-subtree rule) — the ``""`` rows are the source roots. +#: ``(source, folder_path)`` order: one row per existing folder (the +#: ≥ 1-doc recursive-subtree rule — this fixture holds no single-doc +#: folders) — the ``""`` rows are the source roots. EXPECTED_SUMMARIES: list[tuple[str, str, str]] = [ (ALPHA, "", f"Fixture folder summary for {ALPHA}."), (ALPHA, "one", f"Fixture folder summary for {ALPHA}/one."), @@ -426,7 +428,8 @@ def synced_kb(app_server: str, drill_dirs: tuple[Path, Path]) -> None: walk → chunk → embed → overview → folder summaries → version bump), and pins the stored folder summaries: the mock's canned ``FOLDER_SUMMARY_MODE`` branch (task 01) makes the sync store one - deterministic row per ≥ 2-doc folder — the drill turns' answers + deterministic row per existing folder (the ≥ 1-doc rule — this + fixture holds no single-doc folders) — the drill turns' answers assert on that exact text. """ alpha, beta = drill_dirs @@ -452,7 +455,8 @@ def synced_kb(app_server: str, drill_dirs: tuple[Path, Path]) -> None: assert detail["pruned"] == 0, detail assert detail["overview"] is True, detail # The change-gated folder summaries (phase 94 task 02) landed: one - # row per ≥ 2-doc folder, the mock's byte-stable text (the drill + # row per existing folder (the ≥ 1-doc rule — this fixture holds + # no single-doc folders), the mock's byte-stable text (the drill # answers quote exactly these lines). with SessionLocal() as db: rows = db.execute( diff --git a/tests/e2e/test_sync_summary_visibility.py b/tests/e2e/test_sync_summary_visibility.py index add32d5..6f81ffd 100644 --- a/tests/e2e/test_sync_summary_visibility.py +++ b/tests/e2e/test_sync_summary_visibility.py @@ -46,7 +46,9 @@ the same host) registered as a ``kind=local`` source (the pattern; no git anywhere): ``syncsum/`` with TWO ≥ 2-doc folders — ``alpha/`` (2 docs) and ``bravo/`` (2 docs), no root-level files. The generator's candidate set is therefore exactly 3: the source root -(``folder_path ""``) + the two folders (the recursive-subtree ≥ 2 rule). +(``folder_path ""``) + the two folders (the recursive-subtree ≥ 1 +rule — this fixture holds no single-doc folders, so the rule's +relaxation is inert for it). Autouse cleanup (the phase-96 pattern): before each test wait for no running sync on EITHER app, then truncate the shared registry + KB @@ -160,7 +162,7 @@ FOLDER_B = "bravo" FOLDERS = (FOLDER_A, FOLDER_B) N_FILES = 4 # two folders × 2 docs -N_CANDIDATES = 3 # the source root + the two folders (the ≥ 2-doc rule) +N_CANDIDATES = 3 # the source root + the two folders (the ≥ 1-doc rule) #: The mock's byte-stable ``FOLDER_SUMMARY_MODE`` lines for this fixture #: (the phase-94 template — the label is the ``FOLDER_HEADER_PREFIX`` diff --git a/tests/integration/test_docs_api.py b/tests/integration/test_docs_api.py index 73886c8..efbaf16 100644 --- a/tests/integration/test_docs_api.py +++ b/tests/integration/test_docs_api.py @@ -310,9 +310,9 @@ def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> Non assert (alpha["documents"], alpha["children"], alpha["summary"]) == (0, [], None) assert alpha["summary_pending"] is False # 0 documents — never pending assert midx["documents"] == 1 - assert midx["summary_pending"] is False # 1 document — below the minimum + assert midx["summary_pending"] is True # 1 document — the ≥ 1 minimum, no row assert zeta["documents"] == 1 - assert zeta["summary_pending"] is False # 1 document — below the minimum + assert zeta["summary_pending"] is True # 1 document — the ≥ 1 minimum, no row assert [c["path"] for c in midx["children"]] == ["m1.md"] _truncate_tree_tables(db) @@ -321,16 +321,16 @@ def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> Non def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db) -> None: """The endpoint returns ``summary_pending`` on SOURCE + FOLDER nodes (phase 98, task 03 — D3): true iff the recursive count is - ≥ 2 AND no stored row; false WITH a stored row (any — the endpoint - cannot tell AI from manual); false for a < 2-document folder - (never pending) — including one NESTED. File nodes carry no flag.""" + ≥ 1 AND no stored row — the NESTED 1-doc folder included (the ≥ 1 + rule); false WITH a stored row (any — the endpoint cannot tell AI + from manual). File nodes carry no flag.""" _truncate_tree_tables(db) base = datetime.now(UTC) db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) # k8s → 3 documents (talos + cluster + charts), NO stored row → pending _seed_doc(db, "Homelab", "k8s/talos.md", "Talos", 1, base) _seed_doc(db, "Homelab", "k8s/cluster.md", "Cluster", 1, base) - # k8s/helm → 1 document — below the 2-doc minimum, never pending + # k8s/helm → 1 document — a candidate at the ≥ 1 rule (no row) _seed_doc(db, "Homelab", "k8s/helm/charts.md", "Charts", 1, base) # wiki → 2 documents, WITH a stored row → not pending _seed_doc(db, "Homelab", "wiki/one.md", "One", 1, base) @@ -360,7 +360,7 @@ def test_docs_tree_summary_pending_on_source_and_folder_nodes(admin_client, db) assert k8s["summary_pending"] is True # 3 docs, no stored row helm = k8s["children"][0] assert (helm["kind"], helm["path"]) == ("folder", "k8s/helm") - assert helm["summary_pending"] is False # 1 doc — never pending + assert helm["summary_pending"] is True # 1 doc — pending at the ≥ 1 rule assert wiki["summary"] == "Wiki pages." assert wiki["summary_pending"] is False # 2 docs, but a stored row covers it # File nodes carry no pending flag at all (the file table has no @@ -397,8 +397,9 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db) added_at=base + timedelta(hours=2), ) ) - # Alpha: a/b holds 2 docs, c holds 1 (NEVER a candidate), the root - # holds 4 — candidates (Alpha, ""), (Alpha, "a"), (Alpha, "a/b"). + # Alpha: a/b holds 2 docs, c holds 1 (a candidate at the ≥ 1 + # rule), the root holds 4 — candidates (Alpha, ""), (Alpha, "a"), + # (Alpha, "a/b"), (Alpha, "c"). _seed_doc(db, "Alpha", "a/b/c1.md", "C1", 1, base) _seed_doc(db, "Alpha", "a/b/c2.md", "C2", 1, base) _seed_doc(db, "Alpha", "c/solo.md", "Solo", 1, base) @@ -416,6 +417,7 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db) ("Alpha", ""): "Alpha root.", ("Alpha", "a"): "Alpha a.", ("Alpha", "a/b"): "Alpha a b.", + ("Alpha", "c"): "Alpha c.", ("Beta", ""): "Beta root.", ("Beta", "x"): "Beta x.", }.items() @@ -441,8 +443,10 @@ def test_docs_tree_pending_set_equals_missing_folder_summaries(admin_client, db) # And the explicit expectation (the test is readable without the # helper): exactly the two deleted keys, root riding "". assert pending == {("Alpha", "a"), ("Beta", "")} - # Never flagged: the < 2-doc folder (Alpha/c), the 0-document - # registered source (Gamma), and every node that still holds a row. + # Never flagged: the 0-document registered source (Gamma) and every + # node that still holds a row — incl. the 1-doc Alpha/c, whose + # stored row covers it (the ≥ 1 rule makes it a candidate, but a + # candidate WITH a row is not missing). assert ("Alpha", "c") not in pending assert not any(name == "Gamma" for name, _ in pending) assert ("Alpha", "a/b") not in pending @@ -464,8 +468,8 @@ OLDER_STAMP = datetime(2020, 1, 1, tzinfo=UTC) def _seed_folder_pair(db, source: str, folder: str, base: datetime) -> None: - """Two documents under ``source/folder`` — the ≥ 2-document - minimum a folder must hold to be summarizable.""" + """Two documents under ``source/folder`` — a summarizable folder + (≥ 1 doc at the current minimum; two keep the pair realistic).""" _seed_doc(db, source, f"{folder}/one.md", f"{folder} one", 1, base) _seed_doc(db, source, f"{folder}/two.md", f"{folder} two", 2, base) @@ -543,15 +547,17 @@ def test_folder_summary_create_where_no_row_exists( admin_client: TestClient, db ) -> None: """A manual description can be CREATED where no row exists — a - < 2-document folder the generator never wrote (or its fail-soft - miss): insert, not update, with ``manually_edited = true``.""" + folder the generator has not written yet (its fail-soft miss, or + a pre-sync state): insert, not update, with + ``manually_edited = true``.""" _truncate_tree_tables(db) base = datetime.now(UTC) db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) _seed_doc(db, "Homelab", "solo/only.md", "Only", 1, base) # 1-doc folder db.commit() db.expire_all() - assert _get_folder_row(db, "Homelab", "solo") is None # no AI row for < 2 docs + # No AI row yet (the generator has not run in this test). + assert _get_folder_row(db, "Homelab", "solo") is None r = admin_client.patch( "/api/folders/summary", diff --git a/tests/integration/test_sync_folder_summaries.py b/tests/integration/test_sync_folder_summaries.py index 75814c5..b3be71f 100644 --- a/tests/integration/test_sync_folder_summaries.py +++ b/tests/integration/test_sync_folder_summaries.py @@ -11,9 +11,9 @@ convention — the generator only flushes, the sync path commits). Script path (``scripts.import_docs.main`` end to end, fake LLM, real DB, explicit ``--source``): -- a KB-changing import → one row per ≥ 2-doc subtree (the source root - + the 2-doc folder; the 1-doc folder gets none), committed in the - run's transaction, the summary line ending +- a KB-changing import → one row per existing subtree (the source + root, the 2-doc folder a, AND the 1-doc folder b — the ≥ 1 rule), + committed in the run's transaction, the summary line ending ``folder_summaries=//`` (unchanged by phase 96 — no gap-fill suffix on a full regeneration); - an unchanged re-import with a COMPLETE table → zero ``lite`` calls, @@ -27,10 +27,10 @@ DB, explicit ``--source``): failed folder summary self-heals on the next sync); - a KB change on a second run → still a FULL regeneration (call count == candidate count, every row re-stamped, no gap-fill suffix); -- a subtree dropping below 2 docs after a changed re-walk → its row +- a subtree losing its LAST doc after a changed re-walk → its row pruned; - one folder's ``lite`` failure → its previous row kept, the other - lands, exit code 0, stats ``1/1/0``; + land, exit code 0, stats ``2/1/0``; - a ``--limit`` debug run → no generation, no rows, ``folder_summaries=skipped``; - a fresh (empty) table after a ``--limit`` first walk → an unchanged @@ -189,7 +189,8 @@ def _fresh_sync_state() -> Iterator[None]: @pytest.fixture() def src(tmp_path: Path) -> Path: - """MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1. + """MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1 + (b/ is a candidate too — the ≥ 1 rule). md → no document-summary chat calls, so the ``lite`` traffic is exactly the overview + the folder summaries.""" @@ -235,9 +236,9 @@ def test_changed_import_generates_folder_rows( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - """A KB-changing import upserts one row per ≥ 2-doc subtree in the - run's own transaction — committed and visible afterwards — with the - stats on the summary line (PLAN §9).""" + """A KB-changing import upserts one row per existing subtree in + the run's own transaction — committed and visible afterwards — + with the stats on the summary line (PLAN §9).""" llm = FakeEmbedder() records: list[logging.LogRecord] = [] @@ -257,25 +258,26 @@ def test_changed_import_generates_folder_rows( assert rc == 0 assert "added=3" in out assert out.rstrip().endswith( - "overview=updated sources_version=1 folder_summaries=2/0/0" + "overview=updated sources_version=1 folder_summaries=3/0/0" ) - # Two FOLDER_SUMMARY_MODE calls — the source root (3 docs) and the - # 2-doc folder a; the 1-doc folder b gets no row (no lite burn). + # Three FOLDER_SUMMARY_MODE calls — the source root (3 docs), the + # 2-doc folder a, and the 1-doc folder b (the ≥ 1 rule: every + # existing folder is a candidate). calls = _folder_calls(llm) - assert len(calls) == 2 + assert len(calls) == 3 headers = [c[1]["content"].splitlines()[0] for c in calls] - assert headers == ["Folder: MyDocs", "Folder: MyDocs/a"] + assert headers == ["Folder: MyDocs", "Folder: MyDocs/a", "Folder: MyDocs/b"] for c in calls: assert "FOLDER_SUMMARY_MODE" in c[0]["content"] # The rows land committed (the run's own session committed them) — - # one per ≥ 2-doc subtree, never empty, stamped. + # one per existing subtree, never empty, stamped. rows = _rows(db) - assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} assert all(rows.values()) assert _updated_at(db, "MyDocs", "a") is not None # The stats log line (PLAN §9 ample logging). assert any( - "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=0" + "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0" in r.getMessage() for r in records ) @@ -293,10 +295,10 @@ def test_unchanged_reimport_burns_zero_folder_calls( rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 assert out.rstrip().endswith( - "overview=updated sources_version=1 folder_summaries=2/0/0" + "overview=updated sources_version=1 folder_summaries=3/0/0" ) rows = _rows(db) - assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} llm2 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys) @@ -326,7 +328,7 @@ def test_unchanged_reimport_with_gap_fills_only_the_missing_row( rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 rows_before = _rows(db) - assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} root_stamp_before = _updated_at(db, "MyDocs", "") assert root_stamp_before is not None @@ -387,13 +389,15 @@ def test_changed_reimport_is_a_full_regeneration( assert "updated=1" in out # Full-regeneration token — the stats without the gap-fill suffix. assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=2/0/0" + "overview=updated sources_version=2 folder_summaries=3/0/0" ) - # Call count == candidate count — BOTH folders, not a targeted fill. + # Call count == candidate count — ALL existing folders, not a + # targeted fill. calls = _folder_calls(llm2) assert [c[1]["content"].splitlines()[0] for c in calls] == [ "Folder: MyDocs", "Folder: MyDocs/a", + "Folder: MyDocs/b", ] # All rows re-stamped (the full regeneration re-writes every # candidate, even the unchanging one). @@ -404,35 +408,37 @@ def test_changed_reimport_is_a_full_regeneration( assert _rows(db) == rows_before # deterministic fake → same texts -def test_subtree_dropping_below_two_docs_is_pruned( +def test_subtree_losing_its_last_doc_is_pruned( db: Session, src: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - """A subtree that drops below the ≥ 2 rule on a changed re-walk - loses its (stale) row; the remaining qualifiers regenerate.""" + """A subtree that loses its LAST doc (0 recursive docs — below the + ≥ 1 minimum) on a changed re-walk loses its (stale) row; the + remaining qualifiers (≥ 1 doc — the 1-doc b/ included) regenerate.""" llm1 = FakeEmbedder() rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 - assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} - # a/ loses one of its two docs (below the ≥ 2 rule) while b's doc - # changes → the re-walk is a KB change, so generation runs and the - # stale a/ row is pruned; b (1 doc) still gets no row. + # a/ loses BOTH its docs (the folder vanishes — 0 docs) while b's + # doc changes → the re-walk is a KB change, so generation runs and + # the stale a/ row is pruned; b (1 doc) keeps qualifying. + (src / "a" / "one.md").unlink() (src / "a" / "two.md").unlink() (src / "b" / "three.md").write_text("# B Three\nChanged content.\n", encoding="utf-8") llm2 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm2, ["--source", str(src), "--prune"], capsys) assert rc == 0 - assert "pruned=1" in out + assert "pruned=2" in out # the two a/ documents left the index assert "updated=1" in out assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=1/0/1" + "overview=updated sources_version=2 folder_summaries=2/0/1" ) - # Only the source root (the 2 remaining docs) still qualifies. - assert set(_rows(db)) == {("MyDocs", "")} + # The source root (1 remaining doc) and b/ (1 doc) still qualify. + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "b")} assert _updated_at(db, "MyDocs", "a") is None # the row is gone @@ -443,8 +449,8 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green( capsys: pytest.CaptureFixture[str], ) -> None: """One folder's ``lite`` failure: its previous row is kept, the - other folder lands, and the run's exit code stays 0 — the stats - carry the failure (``1/1/0``).""" + other folders land, and the run's exit code stays 0 — the stats + carry the failure (``2/1/0``).""" llm1 = FakeEmbedder() rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 @@ -462,7 +468,7 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green( assert rc == 0 # a failed folder must not fail the import assert "updated=1" in out assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=1/1/0" + "overview=updated sources_version=2 folder_summaries=2/1/0" ) rows_after = _rows(db) assert rows_after["MyDocs", "a"] == rows_before["MyDocs", "a"] # kept @@ -484,14 +490,15 @@ def test_changed_import_never_overwrites_a_manual_row( — ``kept_manual`` is a stat, not a token), the manual row's text, stamp, and flag are untouched, and the other folders regenerate (the ``kept_manual`` stat lands on the generator's log line).""" - # First sync: full generation — root + a/ (b/ holds 1 doc: none). + # First sync: full generation — root + a/ + b/ (the ≥ 1 rule: b/ + # holds 1 doc — still a candidate). llm1 = FakeEmbedder() rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) assert rc == 0 assert out.rstrip().endswith( - "overview=updated sources_version=1 folder_summaries=2/0/0" + "overview=updated sources_version=1 folder_summaries=3/0/0" ) - assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} # The owner edits the source-root description (task 03's PATCH is # the writer; task 01 pins the generator's behavior, so the row is @@ -533,11 +540,15 @@ def test_changed_import_never_overwrites_a_manual_row( assert "added=1" in out # The token STAYS 3 fields — kept_manual is a stat, not a token. assert out.rstrip().endswith( - "overview=updated sources_version=2 folder_summaries=1/0/0" + "overview=updated sources_version=2 folder_summaries=2/0/0" ) - # Zero folder calls for the owner's folder — only a/ (now 3 docs). + # Zero folder calls for the owner's folder — a/ (now 3 docs) and + # b/ regenerate. calls = _folder_calls(llm2) - assert [c[1]["content"].splitlines()[0] for c in calls] == ["Folder: MyDocs/a"] + assert [c[1]["content"].splitlines()[0] for c in calls] == [ + "Folder: MyDocs/a", + "Folder: MyDocs/b", + ] # The owner's text, stamp, and flag are untouched ... rows_after = _rows(db) assert rows_after[("MyDocs", "")] == manual_text @@ -555,7 +566,7 @@ def test_changed_import_never_overwrites_a_manual_row( assert rows_after[("MyDocs", "a")] is not None # The 4-field stats line carries the skip (PLAN §9 ample logging). assert any( - "folder_summaries: generated=1 failed=0 pruned=0 kept_manual=1" + "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1" in r.getMessage() for r in records ) @@ -610,10 +621,10 @@ def test_empty_table_generates_on_unchanged_walk( # `` (gap-fill)`` (the generated set is the full candidate set). assert out.rstrip().endswith( "overview=updated sources_version=skipped " - "folder_summaries=2/0/0 (gap-fill)" + "folder_summaries=3/0/0 (gap-fill)" ) - assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} - assert len(_folder_calls(llm2)) == 2 + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")} + assert len(_folder_calls(llm2)) == 3 assert current_sources_version(db) == 0 # the unchanged walk never bumps diff --git a/tests/unit/test_folder_summaries.py b/tests/unit/test_folder_summaries.py index 3bba5c9..d82bd33 100644 --- a/tests/unit/test_folder_summaries.py +++ b/tests/unit/test_folder_summaries.py @@ -161,9 +161,11 @@ def test_group_by_folder_nested_multi_source_recursive_subtree() -> None: def test_group_by_folder_single_doc_folder_is_a_group_too() -> None: - """Grouping is pure subtree membership (≥ 1 docs): the ≥ 2 rule is - the GENERATOR's (the recursive count below the minimum yields no - row — pinned by the generator tests, not the grouping).""" + """Grouping is pure subtree membership: at the + :data:`MIN_DOCS_PER_FOLDER` = 1 rule every group (≥ 1 docs) is + already a generator candidate, so grouping and candidacy agree + (the generator's filter is an inert safety net — pinned by the + generator tests, not the grouping).""" rows = [("S", "a/only.md", "O", None)] groups = group_by_folder(rows) assert len(groups[("S", "a")]) == 1 # present, but below the minimum @@ -417,8 +419,9 @@ def _rows(db: Session) -> dict[tuple[str, str], str]: def _seed_catalogue(db: Session) -> None: """The shared catalogue: FSU has four docs in three candidate - folders (root 4, a 3, a/b 2 — all ≥ the minimum); FSU-solo has one - doc (its root folder is below the minimum — no row, no call).""" + folders (root 4, a 3, a/b 2); FSU-solo has one doc (its root + folder is a candidate too — the ≥ 1 rule: a single-file source + root gets a row).""" _add_doc(db, "FSU", "a/b/one.md", "One", "One lead.\nSource: FSU/a/b/one.md") _add_doc(db, "FSU", "a/b/two.md", "Two") _add_doc(db, "FSU", "a/three.md", "Three") @@ -436,22 +439,23 @@ def clean_tables(db: Session): def test_generate_happy_path_upserts_every_candidate_folder( db: Session, clean_tables, caplog: pytest.LogCaptureFixture ) -> None: - """Every folder with ≥ 2 recursive docs gets a row (the source root - row included — ``folder_path = ''``); single-doc folders get none; - rows are stamped fresh; the stats dict and the log line are right; - folders are processed in deterministic (source, folder_path) order.""" + """Every folder with ≥ 1 recursive doc gets a row (the source root + row included — ``folder_path = ''`` — and the single-doc FSU-solo + root: the ≥ 1 rule); rows are stamped fresh; the stats dict and + the log line are right; folders are processed in deterministic + (source, folder_path) order.""" _seed_catalogue(db) llm = _FakeLLM() with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 0} - assert llm.calls == 3, "one lite call per candidate folder (the solo folder: none)" + assert stats == {"generated": 4, "failed": 0, "pruned": 0, "kept_manual": 0} + assert llm.calls == 4, "one lite call per candidate folder (the solo root: too)" stored = _rows(db) - assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b")} + assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b"), ("FSU-solo", "")} assert all(summary == REPLY for summary in stored.values()) - assert ("FSU-solo", "") not in stored, ( - "a single-doc folder is fully described by its one file line — no row" + assert stored[("FSU-solo", "")] == REPLY, ( + "the single-doc source root is a candidate at the ≥ 1 rule" ) row = db.get(FolderSummary, ("FSU", "a/b")) @@ -467,6 +471,7 @@ def test_generate_happy_path_upserts_every_candidate_folder( "Folder: FSU", "Folder: FSU/a", "Folder: FSU/a/b", + "Folder: FSU-solo", ] # The recursive-subtree input: the a/ prompt carries a/b's docs too. a_prompt = llm.requests[1][1] @@ -475,7 +480,7 @@ def test_generate_happy_path_upserts_every_candidate_folder( assert "root.md — Root" not in a_prompt assert ( - "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0" + "folder_summaries: generated=4 failed=0 pruned=0 kept_manual=0" in caplog.text ), "the stats line must be greppable (PLAN §9 ample logging)" @@ -492,16 +497,18 @@ def test_generate_per_folder_fail_soft_keeps_previous_and_lands_others( llm = _FakeLLM(fail_folders=("FSU/a/b",)) with caplog.at_level(logging.ERROR, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0} - assert llm.calls == 3 # the failing folder was attempted too + assert stats == {"generated": 3, "failed": 1, "pruned": 0, "kept_manual": 0} + assert llm.calls == 4 # the failing folder was attempted too stored = _rows(db) assert stored[("FSU", "a/b")] == "old summary", ( "the previous row survives the per-folder failure" ) - assert stored[("FSU", "")] == REPLY and stored[("FSU", "a")] == REPLY, ( - "the other folders still land" - ) + assert ( + stored[("FSU", "")] == REPLY + and stored[("FSU", "a")] == REPLY + and stored[("FSU-solo", "")] == REPLY + ), "the other folders still land" assert "folder summary failed for FSU/a/b" in caplog.text assert "simulated lite-model failure for FSU/a/b" in caplog.text @@ -519,24 +526,25 @@ def test_generate_per_folder_fail_soft_without_previous_row_creates_nothing( def test_generate_prunes_stale_rows_and_keeps_live_ones(db: Session, clean_tables) -> None: - """Rows for folders that dropped below 2 recursive docs are deleted + """Rows for folders that VANISHED (0 recursive docs) are deleted (pruned/renamed — the summary would go stale); rows for folders - that still qualify persist (an unchanged folder's summary is still - true — regenerated in place).""" + that still hold ≥ 1 doc persist (an unchanged folder's summary is + still true — regenerated in place), the 1-doc FSU-solo root + included (it is a candidate at the ≥ 1 rule).""" _seed_catalogue(db) - # A stale row for a folder no longer in the catalogue (3→1 docs / - # renamed away) + a live row with old content. + # A stale row for a folder no longer in the catalogue (renamed + # away — 0 docs) + live rows with old content. db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale")) db.add(FolderSummary(source="FSU", folder_path="a", summary="old a summary")) db.add(FolderSummary(source="FSU-solo", folder_path="", summary="solo stale")) db.commit() stats = asyncio.run(generate_folder_summaries(db, _FakeLLM())) - assert stats["pruned"] == 2 # gone/old + the FSU-solo root (1 doc) + assert stats["pruned"] == 1 # gone/old only (its folder vanished) stored = _rows(db) assert ("FSU", "gone/old") not in stored, "the stale folder row must be pruned" - assert ("FSU-solo", "") not in stored, ( - "a folder that dropped below 2 docs loses its row" + assert stored[("FSU-solo", "")] == REPLY, ( + "the 1-doc root qualifies — regenerated, not pruned" ) assert ("FSU", "a") in stored, "the still-qualifying folder keeps its row" assert stored[("FSU", "a")] == REPLY # regenerated, not stale @@ -614,7 +622,7 @@ def test_generate_only_flushes_caller_commits(db: Session, clean_tables) -> None ) assert n == 3, "the caller's commit makes the flushed rows durable" - assert MIN_DOCS_PER_FOLDER == 2 # the ≥ 2 scope rule, pinned by name + assert MIN_DOCS_PER_FOLDER == 1 # the ≥ 1 scope rule, pinned by name # ---------- manually_edited (phase 97, task 01) ---------- @@ -644,17 +652,22 @@ def test_manual_row_survives_regeneration( with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 2, "failed": 0, "pruned": 0, "kept_manual": 1} - assert llm.calls == 2, "the manual folder burns zero lite calls" + assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 1} + assert llm.calls == 3, "the manual folder burns zero lite calls" assert [user.splitlines()[0] for _s, user in llm.requests] == [ "Folder: FSU", "Folder: FSU/a/b", + "Folder: FSU-solo", ], "no prompt is ever built for the owner's folder" stored = _rows(db) assert stored[("FSU", "a")] == manual_text, "the owner's text survives" assert _updated_at(db, "FSU", "a") == stamp_before, ("never re-stamped") - assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY, ( + assert ( + stored[("FSU", "")] == REPLY + and stored[("FSU", "a/b")] == REPLY + and stored[("FSU-solo", "")] == REPLY + ), ( "the non-manual candidates still regenerate (the flag is the difference)" ) row = db.get(FolderSummary, ("FSU", "a")) @@ -662,19 +675,21 @@ def test_manual_row_survives_regeneration( "the generator never clears the flag" ) assert ( - "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1" + "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=1" in caplog.text ), "the 4-field stats line must be greppable (PLAN §9 ample logging)" def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None: - """A manual row is NEVER pruned (phase 97, task 01): two folders - drop below 2 documents — the MANUAL one keeps its row (owner - content persists until cleared — the clear deletes it, so the next - KB-changing sync regenerates an AI description) while the - NON-manual twin loses its now-stale row; the flag is the only - difference. A vanished folder's manual row is kept too, and its - non-manual twin is pruned.""" + """A manual row is NEVER pruned and NEVER overwritten (phase 97, + task 01): a/ and b/ each drop to 1 recursive doc — still + candidates at the ≥ 1 rule — so the MANUAL a/ row is SKIPPED + (owner text kept, no re-stamp, no lite burn) while the NON-manual + b/ twin regenerates in place; the vanished folders' rows show the + prune rule — the manual gone/manual row persists (owner content + until cleared — the clear deletes it, so the next KB-changing sync + regenerates an AI description) and the non-manual gone/ai twin is + pruned; the flag is the only difference in each pair.""" _add_doc(db, "FSU", "a/one.md", "One") _add_doc(db, "FSU", "a/two.md", "Two") _add_doc(db, "FSU", "b/one.md", "B One") @@ -696,7 +711,8 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None: db.add(FolderSummary(source="FSU", folder_path="gone/ai", summary="stale ai")) db.commit() - # a/ and b/ each drop below the minimum (2 -> 1 recursive doc). + # a/ and b/ each drop to 1 recursive doc — still candidates at + # the ≥ 1 rule. db.execute( text( "DELETE FROM documents WHERE source = 'FSU'" @@ -708,14 +724,14 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None: llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 1, "failed": 0, "pruned": 2, "kept_manual": 0} - assert llm.calls == 1, "only the surviving candidate (the root) regenerates" + assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 1} + assert llm.calls == 2, "root + the non-manual b/ regenerate; manual a/ burns nothing" stored = _rows(db) assert stored[("FSU", "a")] == manual_text, ( - "the manual row survives its folder dropping below the minimum" + "the manual row is skipped — never overwritten, even at 1 doc" ) - assert ("FSU", "b") not in stored, ( - "the non-manual twin loses its stale row (the flag is the difference)" + assert stored[("FSU", "b")] == REPLY, ( + "the non-manual twin regenerates — 1 doc still qualifies (the ≥ 1 rule)" ) assert stored[("FSU", "gone/manual")] == "owner kept", ( "a vanished folder's manual row is kept — owner content until cleared" @@ -731,15 +747,15 @@ def test_missing_fresh_table_is_exactly_the_candidate_set( db: Session, clean_tables ) -> None: """No stored rows → every candidate folder is a gap, sorted by - ``(source, folder_path)``; the single-doc FSU-solo root is not a - candidate and can never be a gap.""" + ``(source, folder_path)`` — the single-doc FSU-solo root + included (a candidate at the ≥ 1 rule).""" _seed_catalogue(db) assert missing_folder_summaries(db) == [ ("FSU", ""), ("FSU", "a"), ("FSU", "a/b"), + ("FSU-solo", ""), ] - assert ("FSU-solo", "") not in missing_folder_summaries(db) def test_missing_fully_populated_table_is_empty(db: Session, clean_tables) -> None: @@ -770,16 +786,18 @@ def test_missing_empty_kb_empty_table_is_no_gap(db: Session, clean_tables) -> No assert missing_folder_summaries(db) == [] -def test_missing_single_doc_folder_is_never_listed(db: Session, clean_tables) -> None: - """A below-minimum folder without a row is NOT a gap — it is not a - candidate (its one file line IS its summary).""" +def test_missing_single_doc_folder_is_listed(db: Session, clean_tables) -> None: + """The ≥ 1 rule: a single-doc folder WITHOUT a row IS a gap (it is + a candidate — the one file line no longer exempts it), alongside + its source root (1 doc).""" _add_doc(db, "FSU", "solo/one.md", "One") - assert missing_folder_summaries(db) == [] + assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "solo")] def test_missing_stale_row_is_not_a_gap(db: Session, clean_tables) -> None: - """A stored row for a folder that dropped below 2 docs is stale, - not missing — the prune pass owns it, the gap detector ignores it.""" + """A stored row for a folder that dropped below the minimum + (vanished — 0 docs) is stale, not missing — the prune pass owns + it, the gap detector ignores it.""" _seed_catalogue(db) asyncio.run(generate_folder_summaries(db, _FakeLLM())) db.commit() @@ -865,12 +883,16 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale")) db.commit() a_stamp = _updated_at(db, "FSU", "a") - assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "a/b")] + assert missing_folder_summaries(db) == [ + ("FSU", ""), + ("FSU", "a/b"), + ("FSU-solo", ""), + ] llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 0} - assert llm.calls == 2 + assert stats == {"generated": 3, "failed": 0, "pruned": 1, "kept_manual": 0} + assert llm.calls == 3 stored = _rows(db) assert ("FSU", "gone/old") not in stored, ( @@ -878,7 +900,11 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None ) assert stored[("FSU", "a")] == "keep me" assert _updated_at(db, "FSU", "a") == a_stamp - assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY + assert ( + stored[("FSU", "")] == REPLY + and stored[("FSU", "a/b")] == REPLY + and stored[("FSU-solo", "")] == REPLY + ) def test_only_missing_fail_soft_keeps_prior_and_lands_others( @@ -892,11 +918,12 @@ def test_only_missing_fail_soft_keeps_prior_and_lands_others( db.commit() llm = _FakeLLM(fail_folders=("FSU/a/b",)) stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 1, "failed": 1, "pruned": 0, "kept_manual": 0} - assert llm.calls == 2 # both missing folders were attempted + assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0} + assert llm.calls == 3 # all three missing folders were attempted stored = _rows(db) - assert stored[("FSU", "")] == REPLY, "the other missing folder still lands" + assert stored[("FSU", "")] == REPLY, "the other missing folders still land" + assert stored[("FSU-solo", "")] == REPLY, "the single-doc root still lands" assert ("FSU", "a/b") not in stored, "the failed folder stays absent" assert stored[("FSU", "a")] == "keep me", "the present row is untouched" @@ -941,18 +968,19 @@ def test_on_progress_fires_once_per_candidate_in_sorted_key_order( """Phase 98 (task 01): the hook fires once per candidate, in the same sorted ``(source, folder_path)`` order the folders are attempted — done climbs 1..total, total = the candidate count - (the single-doc FSU-solo root is not a candidate — no event).""" + (the single-doc FSU-solo root IS a candidate at the ≥ 1 rule).""" _seed_catalogue(db) llm = _FakeLLM() events, record = _record_progress() stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record)) assert events == [ - (1, 3, "FSU", ""), - (2, 3, "FSU", "a"), - (3, 3, "FSU", "a/b"), + (1, 4, "FSU", ""), + (2, 4, "FSU", "a"), + (3, 4, "FSU", "a/b"), + (4, 4, "FSU-solo", ""), ] - assert llm.calls == 3 # one event per attempt, in the same order - assert stats["generated"] == 3 + assert llm.calls == 4 # one event per attempt, in the same order + assert stats["generated"] == 4 def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> None: @@ -971,12 +999,13 @@ def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> No events, record = _record_progress() stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record)) assert events == [ - (1, 3, "FSU", ""), - (2, 3, "FSU", "a"), # the instant manual skip still advances - (3, 3, "FSU", "a/b"), + (1, 4, "FSU", ""), + (2, 4, "FSU", "a"), # the instant manual skip still advances + (3, 4, "FSU", "a/b"), + (4, 4, "FSU-solo", ""), ] - assert llm.calls == 2, "the skip itself burns no call" - assert stats["kept_manual"] == 1 and stats["generated"] == 2 + assert llm.calls == 3, "the skip itself burns no call" + assert stats["kept_manual"] == 1 and stats["generated"] == 3 def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> None: @@ -989,11 +1018,12 @@ def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> Non events, record = _record_progress() stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record)) assert events == [ - (1, 3, "FSU", ""), - (2, 3, "FSU", "a"), # the failed attempt still advances - (3, 3, "FSU", "a/b"), + (1, 4, "FSU", ""), + (2, 4, "FSU", "a"), # the failed attempt still advances + (3, 4, "FSU", "a/b"), + (4, 4, "FSU-solo", ""), ] - assert stats["failed"] == 1 and stats["generated"] == 2 + assert stats["failed"] == 1 and stats["generated"] == 3 def test_on_progress_only_missing_reports_the_missing_count( @@ -1049,7 +1079,7 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None generate_folder_summaries(db, hooked, on_progress=record) ) requests_hooked = hooked.requests.copy() - assert len(events) == 3, "the hooked run fired (the contrast is real)" + assert len(events) == 4, "the hooked run fired (the contrast is real)" db.rollback() # the generator only flushes — drop the uncommitted rows plain = _FakeLLM() @@ -1058,5 +1088,8 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None assert plain.requests == requests_hooked, "the fake LLM's call log is unchanged" assert stats_plain == stats_hooked assert _rows(db) == { - ("FSU", ""): REPLY, ("FSU", "a"): REPLY, ("FSU", "a/b"): REPLY + ("FSU", ""): REPLY, + ("FSU", "a"): REPLY, + ("FSU", "a/b"): REPLY, + ("FSU-solo", ""): REPLY, } diff --git a/tests/unit/test_kb_tree_builder.py b/tests/unit/test_kb_tree_builder.py index 4d2d440..b4a1a68 100644 --- a/tests/unit/test_kb_tree_builder.py +++ b/tests/unit/test_kb_tree_builder.py @@ -314,7 +314,7 @@ def test_cross_check_nested_level_matches_group_folder_listing() -> None: # -------------------------------------------------------------------- # summary_pending (phase 98, task 03) — the D3 rule (ONE concept): # a SOURCE or FOLDER node is pending iff its recursive document count -# ≥ MIN_DOCS_PER_FOLDER (2) AND it has NO stored folder_summaries row +# ≥ MIN_DOCS_PER_FOLDER (1) AND it has NO stored folder_summaries row # (AI or manual — any row). That is exactly # ``app.rag.folder_summaries.missing_folder_summaries``'s candidate # set — the marker never drifts from the gap-fill (the integration @@ -359,10 +359,11 @@ def test_folder_with_stored_row_is_not_pending() -> None: assert source.summary_pending is True -def test_single_document_folder_never_pending() -> None: - """A < 2-document folder is NEVER pending (it never gets a summary - — its one file line IS its description), even with no stored row - — while its ≥ 2-doc source root (no root row) still is.""" +def test_single_document_folder_is_pending() -> None: + """The ≥ 1 rule: a 1-document folder with NO stored row IS pending + (its one file line no longer exempts it — the next sync + summarizes it), as is its source root (2 docs, no root row). + A registered 0-document source is the only never-pending case.""" rows = [ ("S", "solo/only.md", "Only", 1, T0, C0), # 1-doc folder ("S", "top.md", "Top", 1, T0, C1), # source total = 2 @@ -370,7 +371,7 @@ def test_single_document_folder_never_pending() -> None: (source,) = build_kb_tree(["S"], rows, {}) (solo,) = _folder_nodes(source) assert solo.documents == 1 - assert solo.summary_pending is False + assert solo.summary_pending is True assert source.summary_pending is True @@ -407,7 +408,7 @@ def test_name_collision_pending_follows_recursive_count() -> None: def test_source_root_pending_and_zero_document_source_never() -> None: - """The source root: a source with ≥ 2 docs and NO ``(source, "")`` + """The source root: a source with ≥ 1 docs and NO ``(source, "")`` row → the SOURCE node is pending; the stored root row clears it. A registered 0-document source is NEVER pending (0 < the minimum — there is nothing to summarize), with or without a manual row."""