feat(rag): summarize single-document folders (MIN_DOCS_PER_FOLDER 2 → 1)
Relax the phase-94 folder-summary scope rule from ≥ 2 documents to ≥ 1: a folder (or source root) is a candidate while ANY document lives under it, so single-file folders and single-file source roots get their own lite-written description. A row is now pruned only when its folder loses its last document (vanishes from the catalogue). The constant is the single source of truth, so the flip propagates to the generator's candidate set, the prune pass, the missing_folder_summaries gap probe (the next sync self-heals the new gaps), and the KB-tree summary_pending markers (1-doc folders / sources now read "Summary pending" until their row lands). Docstrings/comments across app/, scripts/import_docs.py, and the E2E fixtures updated to the ≥ 1 wording. Unit + integration tests updated to the new semantics (the pruned-below-minimum scenario is now a folder losing its LAST doc; single-doc folders are pinned as candidates/pending). Full suite: 2314 passed, app coverage 99%; ruff + pyright clean; folder-summary E2E stories pass in isolation (ls_tree_drilldown, sync_summary_visibility, kb_tree, kb_tree_nav, document_dates, oneshot_llm_retry).
This commit is contained in:
@@ -11,9 +11,9 @@ convention — the generator only flushes, the sync path commits).
|
||||
Script path (``scripts.import_docs.main`` end to end, fake LLM, real
|
||||
DB, explicit ``--source``):
|
||||
|
||||
- a KB-changing import → one row per ≥ 2-doc subtree (the source root
|
||||
+ the 2-doc folder; the 1-doc folder gets none), committed in the
|
||||
run's transaction, the summary line ending
|
||||
- a KB-changing import → one row per existing subtree (the source
|
||||
root, the 2-doc folder a, AND the 1-doc folder b — the ≥ 1 rule),
|
||||
committed in the run's transaction, the summary line ending
|
||||
``folder_summaries=<generated>/<failed>/<pruned>`` (unchanged by
|
||||
phase 96 — no gap-fill suffix on a full regeneration);
|
||||
- an unchanged re-import with a COMPLETE table → zero ``lite`` calls,
|
||||
@@ -27,10 +27,10 @@ DB, explicit ``--source``):
|
||||
failed folder summary self-heals on the next sync);
|
||||
- a KB change on a second run → still a FULL regeneration (call count
|
||||
== candidate count, every row re-stamped, no gap-fill suffix);
|
||||
- a subtree dropping below 2 docs after a changed re-walk → its row
|
||||
- a subtree losing its LAST doc after a changed re-walk → its row
|
||||
pruned;
|
||||
- one folder's ``lite`` failure → its previous row kept, the other
|
||||
lands, exit code 0, stats ``1/1/0``;
|
||||
land, exit code 0, stats ``2/1/0``;
|
||||
- a ``--limit`` debug run → no generation, no rows,
|
||||
``folder_summaries=skipped``;
|
||||
- a fresh (empty) table after a ``--limit`` first walk → an unchanged
|
||||
@@ -189,7 +189,8 @@ def _fresh_sync_state() -> Iterator[None]:
|
||||
|
||||
@pytest.fixture()
|
||||
def src(tmp_path: Path) -> Path:
|
||||
"""MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1.
|
||||
"""MyDocs: a/ (2 docs) + b/ (1 doc) → subtree counts root 3, a 2, b 1
|
||||
(b/ is a candidate too — the ≥ 1 rule).
|
||||
|
||||
md → no document-summary chat calls, so the ``lite`` traffic is
|
||||
exactly the overview + the folder summaries."""
|
||||
@@ -235,9 +236,9 @@ def test_changed_import_generates_folder_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""A KB-changing import upserts one row per ≥ 2-doc subtree in the
|
||||
run's own transaction — committed and visible afterwards — with the
|
||||
stats on the summary line (PLAN §9)."""
|
||||
"""A KB-changing import upserts one row per existing subtree in
|
||||
the run's own transaction — committed and visible afterwards —
|
||||
with the stats on the summary line (PLAN §9)."""
|
||||
llm = FakeEmbedder()
|
||||
records: list[logging.LogRecord] = []
|
||||
|
||||
@@ -257,25 +258,26 @@ def test_changed_import_generates_folder_rows(
|
||||
assert rc == 0
|
||||
assert "added=3" in out
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=1 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=1 folder_summaries=3/0/0"
|
||||
)
|
||||
# Two FOLDER_SUMMARY_MODE calls — the source root (3 docs) and the
|
||||
# 2-doc folder a; the 1-doc folder b gets no row (no lite burn).
|
||||
# Three FOLDER_SUMMARY_MODE calls — the source root (3 docs), the
|
||||
# 2-doc folder a, and the 1-doc folder b (the ≥ 1 rule: every
|
||||
# existing folder is a candidate).
|
||||
calls = _folder_calls(llm)
|
||||
assert len(calls) == 2
|
||||
assert len(calls) == 3
|
||||
headers = [c[1]["content"].splitlines()[0] for c in calls]
|
||||
assert headers == ["Folder: MyDocs", "Folder: MyDocs/a"]
|
||||
assert headers == ["Folder: MyDocs", "Folder: MyDocs/a", "Folder: MyDocs/b"]
|
||||
for c in calls:
|
||||
assert "FOLDER_SUMMARY_MODE" in c[0]["content"]
|
||||
# The rows land committed (the run's own session committed them) —
|
||||
# one per ≥ 2-doc subtree, never empty, stamped.
|
||||
# one per existing subtree, never empty, stamped.
|
||||
rows = _rows(db)
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
assert all(rows.values())
|
||||
assert _updated_at(db, "MyDocs", "a") is not None
|
||||
# The stats log line (PLAN §9 ample logging).
|
||||
assert any(
|
||||
"folder_summaries: generated=2 failed=0 pruned=0 kept_manual=0"
|
||||
"folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0"
|
||||
in r.getMessage()
|
||||
for r in records
|
||||
)
|
||||
@@ -293,10 +295,10 @@ def test_unchanged_reimport_burns_zero_folder_calls(
|
||||
rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=1 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=1 folder_summaries=3/0/0"
|
||||
)
|
||||
rows = _rows(db)
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(rows) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
|
||||
llm2 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys)
|
||||
@@ -326,7 +328,7 @@ def test_unchanged_reimport_with_gap_fills_only_the_missing_row(
|
||||
rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
rows_before = _rows(db)
|
||||
assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(rows_before) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
root_stamp_before = _updated_at(db, "MyDocs", "")
|
||||
assert root_stamp_before is not None
|
||||
|
||||
@@ -387,13 +389,15 @@ def test_changed_reimport_is_a_full_regeneration(
|
||||
assert "updated=1" in out
|
||||
# Full-regeneration token — the stats without the gap-fill suffix.
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=2 folder_summaries=3/0/0"
|
||||
)
|
||||
# Call count == candidate count — BOTH folders, not a targeted fill.
|
||||
# Call count == candidate count — ALL existing folders, not a
|
||||
# targeted fill.
|
||||
calls = _folder_calls(llm2)
|
||||
assert [c[1]["content"].splitlines()[0] for c in calls] == [
|
||||
"Folder: MyDocs",
|
||||
"Folder: MyDocs/a",
|
||||
"Folder: MyDocs/b",
|
||||
]
|
||||
# All rows re-stamped (the full regeneration re-writes every
|
||||
# candidate, even the unchanging one).
|
||||
@@ -404,35 +408,37 @@ def test_changed_reimport_is_a_full_regeneration(
|
||||
assert _rows(db) == rows_before # deterministic fake → same texts
|
||||
|
||||
|
||||
def test_subtree_dropping_below_two_docs_is_pruned(
|
||||
def test_subtree_losing_its_last_doc_is_pruned(
|
||||
db: Session,
|
||||
src: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""A subtree that drops below the ≥ 2 rule on a changed re-walk
|
||||
loses its (stale) row; the remaining qualifiers regenerate."""
|
||||
"""A subtree that loses its LAST doc (0 recursive docs — below the
|
||||
≥ 1 minimum) on a changed re-walk loses its (stale) row; the
|
||||
remaining qualifiers (≥ 1 doc — the 1-doc b/ included) regenerate."""
|
||||
llm1 = FakeEmbedder()
|
||||
rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
|
||||
# a/ loses one of its two docs (below the ≥ 2 rule) while b's doc
|
||||
# changes → the re-walk is a KB change, so generation runs and the
|
||||
# stale a/ row is pruned; b (1 doc) still gets no row.
|
||||
# a/ loses BOTH its docs (the folder vanishes — 0 docs) while b's
|
||||
# doc changes → the re-walk is a KB change, so generation runs and
|
||||
# the stale a/ row is pruned; b (1 doc) keeps qualifying.
|
||||
(src / "a" / "one.md").unlink()
|
||||
(src / "a" / "two.md").unlink()
|
||||
(src / "b" / "three.md").write_text("# B Three\nChanged content.\n", encoding="utf-8")
|
||||
llm2 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm2, ["--source", str(src), "--prune"], capsys)
|
||||
|
||||
assert rc == 0
|
||||
assert "pruned=1" in out
|
||||
assert "pruned=2" in out # the two a/ documents left the index
|
||||
assert "updated=1" in out
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=1/0/1"
|
||||
"overview=updated sources_version=2 folder_summaries=2/0/1"
|
||||
)
|
||||
# Only the source root (the 2 remaining docs) still qualifies.
|
||||
assert set(_rows(db)) == {("MyDocs", "")}
|
||||
# The source root (1 remaining doc) and b/ (1 doc) still qualify.
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "b")}
|
||||
assert _updated_at(db, "MyDocs", "a") is None # the row is gone
|
||||
|
||||
|
||||
@@ -443,8 +449,8 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""One folder's ``lite`` failure: its previous row is kept, the
|
||||
other folder lands, and the run's exit code stays 0 — the stats
|
||||
carry the failure (``1/1/0``)."""
|
||||
other folders land, and the run's exit code stays 0 — the stats
|
||||
carry the failure (``2/1/0``)."""
|
||||
llm1 = FakeEmbedder()
|
||||
rc, _ = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
@@ -462,7 +468,7 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green(
|
||||
assert rc == 0 # a failed folder must not fail the import
|
||||
assert "updated=1" in out
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=1/1/0"
|
||||
"overview=updated sources_version=2 folder_summaries=2/1/0"
|
||||
)
|
||||
rows_after = _rows(db)
|
||||
assert rows_after["MyDocs", "a"] == rows_before["MyDocs", "a"] # kept
|
||||
@@ -484,14 +490,15 @@ def test_changed_import_never_overwrites_a_manual_row(
|
||||
— ``kept_manual`` is a stat, not a token), the manual row's text,
|
||||
stamp, and flag are untouched, and the other folders regenerate
|
||||
(the ``kept_manual`` stat lands on the generator's log line)."""
|
||||
# First sync: full generation — root + a/ (b/ holds 1 doc: none).
|
||||
# First sync: full generation — root + a/ + b/ (the ≥ 1 rule: b/
|
||||
# holds 1 doc — still a candidate).
|
||||
llm1 = FakeEmbedder()
|
||||
rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys)
|
||||
assert rc == 0
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=1 folder_summaries=2/0/0"
|
||||
"overview=updated sources_version=1 folder_summaries=3/0/0"
|
||||
)
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
|
||||
# The owner edits the source-root description (task 03's PATCH is
|
||||
# the writer; task 01 pins the generator's behavior, so the row is
|
||||
@@ -533,11 +540,15 @@ def test_changed_import_never_overwrites_a_manual_row(
|
||||
assert "added=1" in out
|
||||
# The token STAYS 3 fields — kept_manual is a stat, not a token.
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=2 folder_summaries=1/0/0"
|
||||
"overview=updated sources_version=2 folder_summaries=2/0/0"
|
||||
)
|
||||
# Zero folder calls for the owner's folder — only a/ (now 3 docs).
|
||||
# Zero folder calls for the owner's folder — a/ (now 3 docs) and
|
||||
# b/ regenerate.
|
||||
calls = _folder_calls(llm2)
|
||||
assert [c[1]["content"].splitlines()[0] for c in calls] == ["Folder: MyDocs/a"]
|
||||
assert [c[1]["content"].splitlines()[0] for c in calls] == [
|
||||
"Folder: MyDocs/a",
|
||||
"Folder: MyDocs/b",
|
||||
]
|
||||
# The owner's text, stamp, and flag are untouched ...
|
||||
rows_after = _rows(db)
|
||||
assert rows_after[("MyDocs", "")] == manual_text
|
||||
@@ -555,7 +566,7 @@ def test_changed_import_never_overwrites_a_manual_row(
|
||||
assert rows_after[("MyDocs", "a")] is not None
|
||||
# The 4-field stats line carries the skip (PLAN §9 ample logging).
|
||||
assert any(
|
||||
"folder_summaries: generated=1 failed=0 pruned=0 kept_manual=1"
|
||||
"folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1"
|
||||
in r.getMessage()
|
||||
for r in records
|
||||
)
|
||||
@@ -610,10 +621,10 @@ def test_empty_table_generates_on_unchanged_walk(
|
||||
# `` (gap-fill)`` (the generated set is the full candidate set).
|
||||
assert out.rstrip().endswith(
|
||||
"overview=updated sources_version=skipped "
|
||||
"folder_summaries=2/0/0 (gap-fill)"
|
||||
"folder_summaries=3/0/0 (gap-fill)"
|
||||
)
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")}
|
||||
assert len(_folder_calls(llm2)) == 2
|
||||
assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a"), ("MyDocs", "b")}
|
||||
assert len(_folder_calls(llm2)) == 3
|
||||
assert current_sources_version(db) == 0 # the unchanged walk never bumps
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user