feat(rag): summarize single-document folders (MIN_DOCS_PER_FOLDER 2 → 1)
Build and Push Containers / build-and-push-app (push) Successful in 2m10s
Build and Push Containers / build-and-push-db (push) Successful in 13s

Relax the phase-94 folder-summary scope rule from ≥ 2 documents to
≥ 1: a folder (or source root) is a candidate while ANY document
lives under it, so single-file folders and single-file source roots
get their own lite-written description. A row is now pruned only
when its folder loses its last document (vanishes from the
catalogue).

The constant is the single source of truth, so the flip propagates
to the generator's candidate set, the prune pass, the
missing_folder_summaries gap probe (the next sync self-heals the
new gaps), and the KB-tree summary_pending markers (1-doc folders /
sources now read "Summary pending" until their row lands).

Docstrings/comments across app/, scripts/import_docs.py, and the
E2E fixtures updated to the ≥ 1 wording. Unit + integration tests
updated to the new semantics (the pruned-below-minimum scenario is
now a folder losing its LAST doc; single-doc folders are pinned as
candidates/pending). Full suite: 2314 passed, app coverage 99%;
ruff + pyright clean; folder-summary E2E stories pass in isolation
(ls_tree_drilldown, sync_summary_visibility, kb_tree,
kb_tree_nav, document_dates, oneshot_llm_retry).
This commit is contained in:
2026-09-14 08:57:57 -04:00
parent 3a81793565
commit 35d65d2f25
13 changed files with 297 additions and 212 deletions
+112 -79
View File
@@ -161,9 +161,11 @@ def test_group_by_folder_nested_multi_source_recursive_subtree() -> None:
def test_group_by_folder_single_doc_folder_is_a_group_too() -> None:
"""Grouping is pure subtree membership (≥ 1 docs): the ≥ 2 rule is
the GENERATOR's (the recursive count below the minimum yields no
row — pinned by the generator tests, not the grouping)."""
"""Grouping is pure subtree membership: at the
:data:`MIN_DOCS_PER_FOLDER` = 1 rule every group (≥ 1 docs) is
already a generator candidate, so grouping and candidacy agree
(the generator's filter is an inert safety net — pinned by the
generator tests, not the grouping)."""
rows = [("S", "a/only.md", "O", None)]
groups = group_by_folder(rows)
assert len(groups[("S", "a")]) == 1 # present, but below the minimum
@@ -417,8 +419,9 @@ def _rows(db: Session) -> dict[tuple[str, str], str]:
def _seed_catalogue(db: Session) -> None:
"""The shared catalogue: FSU has four docs in three candidate
folders (root 4, a 3, a/b 2 — all ≥ the minimum); FSU-solo has one
doc (its root folder is below the minimum — no row, no call)."""
folders (root 4, a 3, a/b 2); FSU-solo has one doc (its root
folder is a candidate too — the ≥ 1 rule: a single-file source
root gets a row)."""
_add_doc(db, "FSU", "a/b/one.md", "One", "One lead.\nSource: FSU/a/b/one.md")
_add_doc(db, "FSU", "a/b/two.md", "Two")
_add_doc(db, "FSU", "a/three.md", "Three")
@@ -436,22 +439,23 @@ def clean_tables(db: Session):
def test_generate_happy_path_upserts_every_candidate_folder(
db: Session, clean_tables, caplog: pytest.LogCaptureFixture
) -> None:
"""Every folder with ≥ 2 recursive docs gets a row (the source root
row included — ``folder_path = ''``); single-doc folders get none;
rows are stamped fresh; the stats dict and the log line are right;
folders are processed in deterministic (source, folder_path) order."""
"""Every folder with ≥ 1 recursive doc gets a row (the source root
row included — ``folder_path = ''`` — and the single-doc FSU-solo
root: the ≥ 1 rule); rows are stamped fresh; the stats dict and
the log line are right; folders are processed in deterministic
(source, folder_path) order."""
_seed_catalogue(db)
llm = _FakeLLM()
with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"):
stats = asyncio.run(generate_folder_summaries(db, llm))
assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 0}
assert llm.calls == 3, "one lite call per candidate folder (the solo folder: none)"
assert stats == {"generated": 4, "failed": 0, "pruned": 0, "kept_manual": 0}
assert llm.calls == 4, "one lite call per candidate folder (the solo root: too)"
stored = _rows(db)
assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b")}
assert set(stored) == {("FSU", ""), ("FSU", "a"), ("FSU", "a/b"), ("FSU-solo", "")}
assert all(summary == REPLY for summary in stored.values())
assert ("FSU-solo", "") not in stored, (
"a single-doc folder is fully described by its one file line — no row"
assert stored[("FSU-solo", "")] == REPLY, (
"the single-doc source root is a candidate at the ≥ 1 rule"
)
row = db.get(FolderSummary, ("FSU", "a/b"))
@@ -467,6 +471,7 @@ def test_generate_happy_path_upserts_every_candidate_folder(
"Folder: FSU",
"Folder: FSU/a",
"Folder: FSU/a/b",
"Folder: FSU-solo",
]
# The recursive-subtree input: the a/ prompt carries a/b's docs too.
a_prompt = llm.requests[1][1]
@@ -475,7 +480,7 @@ def test_generate_happy_path_upserts_every_candidate_folder(
assert "root.md — Root" not in a_prompt
assert (
"folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0"
"folder_summaries: generated=4 failed=0 pruned=0 kept_manual=0"
in caplog.text
), "the stats line must be greppable (PLAN §9 ample logging)"
@@ -492,16 +497,18 @@ def test_generate_per_folder_fail_soft_keeps_previous_and_lands_others(
llm = _FakeLLM(fail_folders=("FSU/a/b",))
with caplog.at_level(logging.ERROR, logger="app.rag.folder_summaries"):
stats = asyncio.run(generate_folder_summaries(db, llm))
assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0}
assert llm.calls == 3 # the failing folder was attempted too
assert stats == {"generated": 3, "failed": 1, "pruned": 0, "kept_manual": 0}
assert llm.calls == 4 # the failing folder was attempted too
stored = _rows(db)
assert stored[("FSU", "a/b")] == "old summary", (
"the previous row survives the per-folder failure"
)
assert stored[("FSU", "")] == REPLY and stored[("FSU", "a")] == REPLY, (
"the other folders still land"
)
assert (
stored[("FSU", "")] == REPLY
and stored[("FSU", "a")] == REPLY
and stored[("FSU-solo", "")] == REPLY
), "the other folders still land"
assert "folder summary failed for FSU/a/b" in caplog.text
assert "simulated lite-model failure for FSU/a/b" in caplog.text
@@ -519,24 +526,25 @@ def test_generate_per_folder_fail_soft_without_previous_row_creates_nothing(
def test_generate_prunes_stale_rows_and_keeps_live_ones(db: Session, clean_tables) -> None:
"""Rows for folders that dropped below 2 recursive docs are deleted
"""Rows for folders that VANISHED (0 recursive docs) are deleted
(pruned/renamed — the summary would go stale); rows for folders
that still qualify persist (an unchanged folder's summary is still
true — regenerated in place)."""
that still hold ≥ 1 doc persist (an unchanged folder's summary is
still true — regenerated in place), the 1-doc FSU-solo root
included (it is a candidate at the ≥ 1 rule)."""
_seed_catalogue(db)
# A stale row for a folder no longer in the catalogue (3→1 docs /
# renamed away) + a live row with old content.
# A stale row for a folder no longer in the catalogue (renamed
# away — 0 docs) + live rows with old content.
db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale"))
db.add(FolderSummary(source="FSU", folder_path="a", summary="old a summary"))
db.add(FolderSummary(source="FSU-solo", folder_path="", summary="solo stale"))
db.commit()
stats = asyncio.run(generate_folder_summaries(db, _FakeLLM()))
assert stats["pruned"] == 2 # gone/old + the FSU-solo root (1 doc)
assert stats["pruned"] == 1 # gone/old only (its folder vanished)
stored = _rows(db)
assert ("FSU", "gone/old") not in stored, "the stale folder row must be pruned"
assert ("FSU-solo", "") not in stored, (
"a folder that dropped below 2 docs loses its row"
assert stored[("FSU-solo", "")] == REPLY, (
"the 1-doc root qualifies — regenerated, not pruned"
)
assert ("FSU", "a") in stored, "the still-qualifying folder keeps its row"
assert stored[("FSU", "a")] == REPLY # regenerated, not stale
@@ -614,7 +622,7 @@ def test_generate_only_flushes_caller_commits(db: Session, clean_tables) -> None
)
assert n == 3, "the caller's commit makes the flushed rows durable"
assert MIN_DOCS_PER_FOLDER == 2 # the ≥ 2 scope rule, pinned by name
assert MIN_DOCS_PER_FOLDER == 1 # the ≥ 1 scope rule, pinned by name
# ---------- manually_edited (phase 97, task 01) ----------
@@ -644,17 +652,22 @@ def test_manual_row_survives_regeneration(
with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"):
stats = asyncio.run(generate_folder_summaries(db, llm))
assert stats == {"generated": 2, "failed": 0, "pruned": 0, "kept_manual": 1}
assert llm.calls == 2, "the manual folder burns zero lite calls"
assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 1}
assert llm.calls == 3, "the manual folder burns zero lite calls"
assert [user.splitlines()[0] for _s, user in llm.requests] == [
"Folder: FSU",
"Folder: FSU/a/b",
"Folder: FSU-solo",
], "no prompt is ever built for the owner's folder"
stored = _rows(db)
assert stored[("FSU", "a")] == manual_text, "the owner's text survives"
assert _updated_at(db, "FSU", "a") == stamp_before, ("never re-stamped")
assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY, (
assert (
stored[("FSU", "")] == REPLY
and stored[("FSU", "a/b")] == REPLY
and stored[("FSU-solo", "")] == REPLY
), (
"the non-manual candidates still regenerate (the flag is the difference)"
)
row = db.get(FolderSummary, ("FSU", "a"))
@@ -662,19 +675,21 @@ def test_manual_row_survives_regeneration(
"the generator never clears the flag"
)
assert (
"folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1"
"folder_summaries: generated=3 failed=0 pruned=0 kept_manual=1"
in caplog.text
), "the 4-field stats line must be greppable (PLAN §9 ample logging)"
def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None:
"""A manual row is NEVER pruned (phase 97, task 01): two folders
drop below 2 documents — the MANUAL one keeps its row (owner
content persists until cleared — the clear deletes it, so the next
KB-changing sync regenerates an AI description) while the
NON-manual twin loses its now-stale row; the flag is the only
difference. A vanished folder's manual row is kept too, and its
non-manual twin is pruned."""
"""A manual row is NEVER pruned and NEVER overwritten (phase 97,
task 01): a/ and b/ each drop to 1 recursive doc — still
candidates at the ≥ 1 rule — so the MANUAL a/ row is SKIPPED
(owner text kept, no re-stamp, no lite burn) while the NON-manual
b/ twin regenerates in place; the vanished folders' rows show the
prune rule — the manual gone/manual row persists (owner content
until cleared — the clear deletes it, so the next KB-changing sync
regenerates an AI description) and the non-manual gone/ai twin is
pruned; the flag is the only difference in each pair."""
_add_doc(db, "FSU", "a/one.md", "One")
_add_doc(db, "FSU", "a/two.md", "Two")
_add_doc(db, "FSU", "b/one.md", "B One")
@@ -696,7 +711,8 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None:
db.add(FolderSummary(source="FSU", folder_path="gone/ai", summary="stale ai"))
db.commit()
# a/ and b/ each drop below the minimum (2 -> 1 recursive doc).
# a/ and b/ each drop to 1 recursive doc — still candidates at
# the ≥ 1 rule.
db.execute(
text(
"DELETE FROM documents WHERE source = 'FSU'"
@@ -708,14 +724,14 @@ def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None:
llm = _FakeLLM()
stats = asyncio.run(generate_folder_summaries(db, llm))
assert stats == {"generated": 1, "failed": 0, "pruned": 2, "kept_manual": 0}
assert llm.calls == 1, "only the surviving candidate (the root) regenerates"
assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 1}
assert llm.calls == 2, "root + the non-manual b/ regenerate; manual a/ burns nothing"
stored = _rows(db)
assert stored[("FSU", "a")] == manual_text, (
"the manual row survives its folder dropping below the minimum"
"the manual row is skipped — never overwritten, even at 1 doc"
)
assert ("FSU", "b") not in stored, (
"the non-manual twin loses its stale row (the flag is the difference)"
assert stored[("FSU", "b")] == REPLY, (
"the non-manual twin regenerates — 1 doc still qualifies (the ≥ 1 rule)"
)
assert stored[("FSU", "gone/manual")] == "owner kept", (
"a vanished folder's manual row is kept — owner content until cleared"
@@ -731,15 +747,15 @@ def test_missing_fresh_table_is_exactly_the_candidate_set(
db: Session, clean_tables
) -> None:
"""No stored rows → every candidate folder is a gap, sorted by
``(source, folder_path)``; the single-doc FSU-solo root is not a
candidate and can never be a gap."""
``(source, folder_path)`` — the single-doc FSU-solo root
included (a candidate at the ≥ 1 rule)."""
_seed_catalogue(db)
assert missing_folder_summaries(db) == [
("FSU", ""),
("FSU", "a"),
("FSU", "a/b"),
("FSU-solo", ""),
]
assert ("FSU-solo", "") not in missing_folder_summaries(db)
def test_missing_fully_populated_table_is_empty(db: Session, clean_tables) -> None:
@@ -770,16 +786,18 @@ def test_missing_empty_kb_empty_table_is_no_gap(db: Session, clean_tables) -> No
assert missing_folder_summaries(db) == []
def test_missing_single_doc_folder_is_never_listed(db: Session, clean_tables) -> None:
"""A below-minimum folder without a row is NOT a gap — it is not a
candidate (its one file line IS its summary)."""
def test_missing_single_doc_folder_is_listed(db: Session, clean_tables) -> None:
"""The ≥ 1 rule: a single-doc folder WITHOUT a row IS a gap (it is
a candidate — the one file line no longer exempts it), alongside
its source root (1 doc)."""
_add_doc(db, "FSU", "solo/one.md", "One")
assert missing_folder_summaries(db) == []
assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "solo")]
def test_missing_stale_row_is_not_a_gap(db: Session, clean_tables) -> None:
"""A stored row for a folder that dropped below 2 docs is stale,
not missing — the prune pass owns it, the gap detector ignores it."""
"""A stored row for a folder that dropped below the minimum
(vanished — 0 docs) is stale, not missing — the prune pass owns
it, the gap detector ignores it."""
_seed_catalogue(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
@@ -865,12 +883,16 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None
db.add(FolderSummary(source="FSU", folder_path="gone/old", summary="stale"))
db.commit()
a_stamp = _updated_at(db, "FSU", "a")
assert missing_folder_summaries(db) == [("FSU", ""), ("FSU", "a/b")]
assert missing_folder_summaries(db) == [
("FSU", ""),
("FSU", "a/b"),
("FSU-solo", ""),
]
llm = _FakeLLM()
stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True))
assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 0}
assert llm.calls == 2
assert stats == {"generated": 3, "failed": 0, "pruned": 1, "kept_manual": 0}
assert llm.calls == 3
stored = _rows(db)
assert ("FSU", "gone/old") not in stored, (
@@ -878,7 +900,11 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None
)
assert stored[("FSU", "a")] == "keep me"
assert _updated_at(db, "FSU", "a") == a_stamp
assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY
assert (
stored[("FSU", "")] == REPLY
and stored[("FSU", "a/b")] == REPLY
and stored[("FSU-solo", "")] == REPLY
)
def test_only_missing_fail_soft_keeps_prior_and_lands_others(
@@ -892,11 +918,12 @@ def test_only_missing_fail_soft_keeps_prior_and_lands_others(
db.commit()
llm = _FakeLLM(fail_folders=("FSU/a/b",))
stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True))
assert stats == {"generated": 1, "failed": 1, "pruned": 0, "kept_manual": 0}
assert llm.calls == 2 # both missing folders were attempted
assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0}
assert llm.calls == 3 # all three missing folders were attempted
stored = _rows(db)
assert stored[("FSU", "")] == REPLY, "the other missing folder still lands"
assert stored[("FSU", "")] == REPLY, "the other missing folders still land"
assert stored[("FSU-solo", "")] == REPLY, "the single-doc root still lands"
assert ("FSU", "a/b") not in stored, "the failed folder stays absent"
assert stored[("FSU", "a")] == "keep me", "the present row is untouched"
@@ -941,18 +968,19 @@ def test_on_progress_fires_once_per_candidate_in_sorted_key_order(
"""Phase 98 (task 01): the hook fires once per candidate, in the
same sorted ``(source, folder_path)`` order the folders are
attempted — done climbs 1..total, total = the candidate count
(the single-doc FSU-solo root is not a candidate — no event)."""
(the single-doc FSU-solo root IS a candidate at the ≥ 1 rule)."""
_seed_catalogue(db)
llm = _FakeLLM()
events, record = _record_progress()
stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record))
assert events == [
(1, 3, "FSU", ""),
(2, 3, "FSU", "a"),
(3, 3, "FSU", "a/b"),
(1, 4, "FSU", ""),
(2, 4, "FSU", "a"),
(3, 4, "FSU", "a/b"),
(4, 4, "FSU-solo", ""),
]
assert llm.calls == 3 # one event per attempt, in the same order
assert stats["generated"] == 3
assert llm.calls == 4 # one event per attempt, in the same order
assert stats["generated"] == 4
def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> None:
@@ -971,12 +999,13 @@ def test_on_progress_manual_skip_still_advances(db: Session, clean_tables) -> No
events, record = _record_progress()
stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record))
assert events == [
(1, 3, "FSU", ""),
(2, 3, "FSU", "a"), # the instant manual skip still advances
(3, 3, "FSU", "a/b"),
(1, 4, "FSU", ""),
(2, 4, "FSU", "a"), # the instant manual skip still advances
(3, 4, "FSU", "a/b"),
(4, 4, "FSU-solo", ""),
]
assert llm.calls == 2, "the skip itself burns no call"
assert stats["kept_manual"] == 1 and stats["generated"] == 2
assert llm.calls == 3, "the skip itself burns no call"
assert stats["kept_manual"] == 1 and stats["generated"] == 3
def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> None:
@@ -989,11 +1018,12 @@ def test_on_progress_failed_key_still_advances(db: Session, clean_tables) -> Non
events, record = _record_progress()
stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record))
assert events == [
(1, 3, "FSU", ""),
(2, 3, "FSU", "a"), # the failed attempt still advances
(3, 3, "FSU", "a/b"),
(1, 4, "FSU", ""),
(2, 4, "FSU", "a"), # the failed attempt still advances
(3, 4, "FSU", "a/b"),
(4, 4, "FSU-solo", ""),
]
assert stats["failed"] == 1 and stats["generated"] == 2
assert stats["failed"] == 1 and stats["generated"] == 3
def test_on_progress_only_missing_reports_the_missing_count(
@@ -1049,7 +1079,7 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None
generate_folder_summaries(db, hooked, on_progress=record)
)
requests_hooked = hooked.requests.copy()
assert len(events) == 3, "the hooked run fired (the contrast is real)"
assert len(events) == 4, "the hooked run fired (the contrast is real)"
db.rollback() # the generator only flushes — drop the uncommitted rows
plain = _FakeLLM()
@@ -1058,5 +1088,8 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None
assert plain.requests == requests_hooked, "the fake LLM's call log is unchanged"
assert stats_plain == stats_hooked
assert _rows(db) == {
("FSU", ""): REPLY, ("FSU", "a"): REPLY, ("FSU", "a/b"): REPLY
("FSU", ""): REPLY,
("FSU", "a"): REPLY,
("FSU", "a/b"): REPLY,
("FSU-solo", ""): REPLY,
}
+8 -7
View File
@@ -314,7 +314,7 @@ def test_cross_check_nested_level_matches_group_folder_listing() -> None:
# --------------------------------------------------------------------
# summary_pending (phase 98, task 03) — the D3 rule (ONE concept):
# a SOURCE or FOLDER node is pending iff its recursive document count
# ≥ MIN_DOCS_PER_FOLDER (2) AND it has NO stored folder_summaries row
# ≥ MIN_DOCS_PER_FOLDER (1) AND it has NO stored folder_summaries row
# (AI or manual — any row). That is exactly
# ``app.rag.folder_summaries.missing_folder_summaries``'s candidate
# set — the marker never drifts from the gap-fill (the integration
@@ -359,10 +359,11 @@ def test_folder_with_stored_row_is_not_pending() -> None:
assert source.summary_pending is True
def test_single_document_folder_never_pending() -> None:
"""A < 2-document folder is NEVER pending (it never gets a summary
— its one file line IS its description), even with no stored row
— while its ≥ 2-doc source root (no root row) still is."""
def test_single_document_folder_is_pending() -> None:
"""The ≥ 1 rule: a 1-document folder with NO stored row IS pending
(its one file line no longer exempts it — the next sync
summarizes it), as is its source root (2 docs, no root row).
A registered 0-document source is the only never-pending case."""
rows = [
("S", "solo/only.md", "Only", 1, T0, C0), # 1-doc folder
("S", "top.md", "Top", 1, T0, C1), # source total = 2
@@ -370,7 +371,7 @@ def test_single_document_folder_never_pending() -> None:
(source,) = build_kb_tree(["S"], rows, {})
(solo,) = _folder_nodes(source)
assert solo.documents == 1
assert solo.summary_pending is False
assert solo.summary_pending is True
assert source.summary_pending is True
@@ -407,7 +408,7 @@ def test_name_collision_pending_follows_recursive_count() -> None:
def test_source_root_pending_and_zero_document_source_never() -> None:
"""The source root: a source with ≥ 2 docs and NO ``(source, "")``
"""The source root: a source with ≥ 1 docs and NO ``(source, "")``
row → the SOURCE node is pending; the stored root row clears it.
A registered 0-document source is NEVER pending (0 < the minimum —
there is nothing to summarize), with or without a manual row."""