phase: 127_sync_incremental_folder_summaries
Build and Push Containers / build-and-push-app (push) Successful in 2m25s
Build and Push Containers / build-and-push-db (push) Successful in 15s

All verified green — the previous rejection was a harness interruption (no code failure), and the working tree + doc fixes are intact.

**Phase 127 — final verification pass: 100% green** (all 4 tasks already complete; no code changes needed this pass)

- **Re-verified implementation:** `ImportSummary.changed_paths` at all 5 increment sites + `_prune`; `affected_folder_keys` + `generate_folder_summaries(changed_docs=…)` (restricted keys, prune pass both modes, hook total = affected count); sync changed-KB branch passes `changed_docs=summary.changed_paths`; CLI keeps full-regen contract
- **Tests:** `uv run pytest --cov=app --cov-report=term-missing` → **3022 passed**, `app/` coverage **99%**; `uv run pytest tests/e2e/test_incremental_folder_summaries.py -v --no-cov` → **3 passed in isolation**; `uv run ruff check . && uv run pyright` → clean, **0 errors**
- **Completion criteria:** one-file change rewrites only its folder chain (E2E pins siblings byte-identical, text AND `updated_at`) ✅; coverage >90% ✅; isolated E2E green ✅; no completed-phase behavior change (full suite green) ✅; atomic commit — delegated to the harness per pass rules (no git commands run by me)
- **Docs:** README.md re-checked — all 7 previous-pass fixes still in place (Theme page row, drill-down `ls` + capped `read`, phase-124 answer images section + `BOR_ANSWER_IMAGES`, markdown-included summaries, 17-color palette, folder summaries in `BOR_LLM_SUMMARY_MODEL`, CLI regen note); AGENTS.md verified consistent — no changes needed
- **Deviations:** none; untracked owner file `NOTES.md` left untouched (rides in the harness commit)
- **Next pending phase:** `128_sync_label_mobile`
This commit is contained in:
2026-09-27 21:12:25 -04:00
parent f50d4c8bc3
commit 6e654df387
38 changed files with 2741 additions and 46 deletions
+380
View File
@@ -29,6 +29,7 @@ from app.rag.folder_summaries import (
FOLDER_SUMMARY_MODE,
MIN_DOCS_PER_FOLDER,
SYSTEM_PROMPT,
affected_folder_keys,
build_folder_summary_prompt,
folder_of,
generate_folder_summaries,
@@ -202,6 +203,56 @@ def test_group_by_folder_plain_file_path_is_not_a_group_key() -> None:
assert "top.md" not in [folder for _source, folder in groups]
# ---------- affected_folder_keys (phase 127, task 02) ----------
def test_affected_folder_keys_root_level_file_is_only_the_root() -> None:
"""A root-level file (``folder_of == ""``) affects ONLY the source
root — there is no folder prefix to walk."""
assert affected_folder_keys([("S", "a.md")]) == {("S", "")}
def test_affected_folder_keys_nested_file_walks_the_folder_chain() -> None:
"""A doc under ``a/b/`` affects ``a/b``, ``a``, and the source
root — the SAME prefix walk ``group_by_folder`` pass 1 uses (one
concept)."""
assert affected_folder_keys([("S", "a/b/c.md")]) == {
("S", ""),
("S", "a"),
("S", "a/b"),
}
def test_affected_folder_keys_multi_source_set_semantics() -> None:
"""Keys are per source (the PK is ``(source, folder_path)``), and
the result is a SET — a repeated chain adds no new keys, and an
unrelated source keeps its own root."""
keys = affected_folder_keys(
[
("S", "a/b/c.md"),
("T", "x.md"),
("S", "a/d.md"), # the same chain as the first entry — no new keys
]
)
assert keys == {("S", ""), ("S", "a"), ("S", "a/b"), ("T", "")}
def test_affected_folder_keys_fragment_paths_fold_into_their_parent_folder() -> None:
"""Embedded-image fragment paths (phase 126: ``x.pdf#page=…#img=…``)
get NO special case: the fragment never holds a ``/``, so
``folder_of`` folds them into the parent folder's chain (a
root-level fragment affects only the source root)."""
assert affected_folder_keys([("S", "dir/img.pdf#page=2#img=0")]) == {
("S", ""),
("S", "dir"),
}
assert affected_folder_keys([("S", "img.pdf#page=1#img=0")]) == {("S", "")}
def test_affected_folder_keys_empty_input_is_an_empty_set() -> None:
assert affected_folder_keys([]) == set()
# ---------- build_folder_summary_prompt: system ----------
@@ -1093,3 +1144,332 @@ def test_on_progress_none_is_a_zero_cost_noop(db: Session, clean_tables) -> None
("FSU", "a/b"): REPLY,
("FSU-solo", ""): REPLY,
}
# ---------- generate_folder_summaries(changed_docs=…) (phase 127, task 02) ----------
def _seed_three_sources(db: Session) -> None:
"""The restricted-mode catalogue (three sources): A is nested
(``a/`` and ``a/b/``), B is a sibling single-folder source, and C
is a single-doc root — six candidate keys total: A ×
{``""``, ``a``, ``a/b``}, B × {``""``, ``b``}, C × {``""``}.
"""
_add_doc(db, "A", "a/b/one.md", "A One", "A one lead.")
_add_doc(db, "A", "a/b/two.md", "A Two")
_add_doc(db, "A", "a/three.md", "A Three")
_add_doc(db, "B", "b/one.md", "B One")
_add_doc(db, "C", "solo.md", "C Solo")
def _stamps(
db: Session, keys: set[tuple[str, str]]
) -> dict[tuple[str, str], object]:
"""The stored rows' ``updated_at`` for *keys* (raw SQL — the
byte-identity comparisons stay honest)."""
return {
(source, folder): _updated_at(db, source, folder) for source, folder in keys
}
def _freeze(
db: Session, values: dict[tuple[str, str], str]
) -> dict[tuple[str, str], object]:
"""Overwrite the stored summaries for *values*' keys with DISTINCT
frozen text (a "regenerated" row is then distinguishable from an
"untouched" one), and return the pre-run stamps."""
for (source, folder_path), value in values.items():
db.execute(
text(
"UPDATE folder_summaries SET summary = :v"
" WHERE source = :s AND folder_path = :f"
),
{"v": value, "s": source, "f": folder_path},
)
db.commit()
return _stamps(db, set(values))
def test_changed_docs_restricted_regeneration_only_affected_keys(
db: Session, clean_tables
) -> None:
"""One changed file in A/``a/b/`` → exactly that file's folder
chain (``a/b``, ``a``, and the A root) regenerates: one lite call
per affected non-manual key, in sorted order, fresh rows. Every
OTHER stored row — same-source siblings and other sources — keeps
its summary text AND ``updated_at`` byte-identical (zero lite
calls for them)."""
_seed_three_sources(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
frozen = {
("A", ""): "A root frozen",
("A", "a"): "A/a frozen",
("A", "a/b"): "A/a/b frozen",
("B", ""): "B root frozen",
("B", "b"): "B/b frozen",
("C", ""): "C root frozen",
}
stamps = _freeze(db, frozen)
llm = _FakeLLM()
stats = asyncio.run(
generate_folder_summaries(db, llm, changed_docs=[("A", "a/b/one.md")])
)
assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 0}
assert llm.calls == 3, "one lite call per AFFECTED non-manual key — nothing else"
assert [user.splitlines()[0] for _s, user in llm.requests] == [
"Folder: A",
"Folder: A/a",
"Folder: A/a/b",
], "the affected chain in sorted (source, folder_path) order"
stored = _rows(db)
assert stored[("A", "")] == REPLY
assert stored[("A", "a")] == REPLY
assert stored[("A", "a/b")] == REPLY, "the changed file's own folder regenerates"
assert stored[("B", "")] == "B root frozen"
assert stored[("B", "b")] == "B/b frozen"
assert stored[("C", "")] == "C root frozen", (
"the unaffected rows keep their stored text"
)
for key in (("B", ""), ("B", "b"), ("C", "")):
assert _updated_at(db, *key) == stamps[key], (
"the unaffected rows are never re-stamped"
)
def test_changed_docs_manual_affected_row_is_skipped(db: Session, clean_tables) -> None:
"""A manual row INSIDE the affected chain is skipped exactly as in
full mode (phase 97): ``kept_manual``, no lite call, text AND
``updated_at`` byte-identical — the restriction changes the key
set, not the manual-row rule."""
_seed_three_sources(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
manual_text = "Owner's words about A/a."
row = db.get(FolderSummary, ("A", "a"))
assert row is not None
row.summary = manual_text
row.manually_edited = True
db.commit()
stamp_before = _updated_at(db, "A", "a")
llm = _FakeLLM()
stats = asyncio.run(
generate_folder_summaries(db, llm, changed_docs=[("A", "a/b/one.md")])
)
assert stats == {"generated": 2, "failed": 0, "pruned": 0, "kept_manual": 1}
assert llm.calls == 2, "the manual key burns zero lite calls"
assert [user.splitlines()[0] for _s, user in llm.requests] == [
"Folder: A",
"Folder: A/a/b",
]
stored = _rows(db)
assert stored[("A", "a")] == manual_text, "the owner's text is never rewritten"
assert _updated_at(db, "A", "a") == stamp_before, ("never re-stamped")
assert stored[("A", "")] == REPLY and stored[("A", "a/b")] == REPLY
assert stored[("B", "")] == REPLY and stored[("B", "b")] == REPLY
assert stored[("C", "")] == REPLY, "the unaffected rows are left as they were"
def test_changed_docs_failing_affected_folder_fails_soft(db: Session, clean_tables) -> None:
"""Per-folder fail-soft under the restriction: the failing
affected key is attempted and counted, its PREVIOUS row is kept
(an old summary is better than none), the other affected keys
still land, and the unaffected rows are untouched."""
_seed_three_sources(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
# A DISTINCT previous text on the failing folder, so "kept" is
# visible (it would otherwise be the REPLY either way).
db.execute(
text(
"UPDATE folder_summaries SET summary = 'old A/a/b'"
" WHERE (source, folder_path) = ('A', 'a/b')"
)
)
db.commit()
stamps = _stamps(db, {("B", ""), ("B", "b"), ("C", "")})
llm = _FakeLLM(fail_folders=("A/a/b",))
stats = asyncio.run(
generate_folder_summaries(db, llm, changed_docs=[("A", "a/b/one.md")])
)
assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0}
assert llm.calls == 3, "the failing key is attempted (the hook's premise)"
stored = _rows(db)
assert stored[("A", "a/b")] == "old A/a/b", "the previous row survives"
assert stored[("A", "")] == REPLY and stored[("A", "a")] == REPLY
for key, stamp in stamps.items():
assert _updated_at(db, *key) == stamp, "the unaffected rows are untouched"
def test_changed_docs_multi_source_change_set(db: Session, clean_tables) -> None:
"""A change set spanning SOURCES: the affected keys are the UNION
of each document's chain (A's nested chain + B's root and ``b``),
sorted across sources; the unaffected C root row keeps its text
AND ``updated_at``."""
_seed_three_sources(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
c_stamp = _updated_at(db, "C", "")
llm = _FakeLLM()
stats = asyncio.run(
generate_folder_summaries(
db, llm, changed_docs=[("A", "a/b/two.md"), ("B", "b/one.md")]
)
)
assert stats == {"generated": 5, "failed": 0, "pruned": 0, "kept_manual": 0}
assert llm.calls == 5
assert [user.splitlines()[0] for _s, user in llm.requests] == [
"Folder: A",
"Folder: A/a",
"Folder: A/a/b",
"Folder: B",
"Folder: B/b",
]
stored = _rows(db)
assert stored[("C", "")] == REPLY
assert _updated_at(db, "C", "") == c_stamp
def test_changed_docs_gap_in_affected_folder_is_generated(
db: Session, clean_tables
) -> None:
"""An affected folder with NO stored row (a gap in the chain) is a
candidate and gets generated like any other affected key — while a
gap in an UNAFFECTED folder stays the unchanged-walk gap probe's
job (phase 96): the restriction never widens beyond the affected
set."""
_seed_three_sources(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
db.execute(
text(
"DELETE FROM folder_summaries"
" WHERE (source, folder_path) IN (('A', 'a/b'), ('B', ''))"
)
)
db.commit()
llm = _FakeLLM()
stats = asyncio.run(
generate_folder_summaries(db, llm, changed_docs=[("A", "a/b/one.md")])
)
assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 0}
assert llm.calls == 3
stored = _rows(db)
assert stored[("A", "a/b")] == REPLY, "the gap in the affected chain is filled"
assert ("B", "") not in stored, (
"the gap in the UNAFFECTED source is left for the gap probe"
)
assert stored[("B", "b")] == REPLY
def test_changed_docs_vanished_affected_folder_is_pruned_not_generated(
db: Session, clean_tables
) -> None:
"""The ASSUMPTION pinned: an affected folder that no longer holds
any document is a PRUNE-pass job, not a generation's — the pruned
document's chain (root + its now-vanished folder) is in the change
set, only the surviving candidate (the root) generates, the
vanished folder's row is dropped, and the unaffected rows keep
their ``updated_at``."""
_add_doc(db, "A", "a/b/one.md", "A One")
_add_doc(db, "A", "a/b/two.md", "A Two")
_add_doc(db, "A", "gone/old.md", "Gone")
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
stamps = _stamps(db, {("A", "a"), ("A", "a/b")})
db.execute(text("DELETE FROM documents WHERE path = 'gone/old.md'"))
db.commit()
llm = _FakeLLM()
stats = asyncio.run(
generate_folder_summaries(db, llm, changed_docs=[("A", "gone/old.md")])
)
# affected = {("A", ""), ("A", "gone")} — the only surviving
# candidate in that set is the source root.
assert stats == {"generated": 1, "failed": 0, "pruned": 1, "kept_manual": 0}
assert llm.calls == 1
assert [user.splitlines()[0] for _s, user in llm.requests] == ["Folder: A"]
stored = _rows(db)
assert ("A", "gone") not in stored, "the vanished folder's row is pruned"
assert stored[("A", "")] == REPLY, "the root's subtree changed — regenerated"
assert stored[("A", "a")] == REPLY and stored[("A", "a/b")] == REPLY
for key, stamp in stamps.items():
assert _updated_at(db, *key) == stamp, "the unaffected rows are untouched"
def test_changed_docs_hook_total_is_the_affected_count(
db: Session, clean_tables
) -> None:
"""The progress hook (phase 98) under the restriction fires exactly
``len(affected)`` times with ``total == len(affected)`` — the sync
status's ``summaries_total`` then reports the affected count
through the existing hook — in sorted key order."""
_seed_three_sources(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
llm = _FakeLLM()
events, record = _record_progress()
stats = asyncio.run(
generate_folder_summaries(
db, llm, changed_docs=[("A", "a/b/one.md")], on_progress=record
)
)
assert events == [
(1, 3, "A", ""),
(2, 3, "A", "a"),
(3, 3, "A", "a/b"),
]
assert llm.calls == 3
assert stats["generated"] == 3
def test_changed_docs_none_is_the_pre_127_full_regeneration(
db: Session, clean_tables
) -> None:
"""The default (no ``changed_docs`` kwarg) is the pre-127 FULL
regeneration over the same three-source catalogue: every
candidate key, one call each, the 4-field stats shape unchanged —
the ``scripts/import_docs.py`` CLI contract."""
_seed_three_sources(db)
llm = _FakeLLM()
stats = asyncio.run(generate_folder_summaries(db, llm))
assert stats == {"generated": 6, "failed": 0, "pruned": 0, "kept_manual": 0}
assert llm.calls == 6, "no kwarg → every candidate burns one call"
assert set(_rows(db)) == {
("A", ""), ("A", "a"), ("A", "a/b"),
("B", ""), ("B", "b"),
("C", ""),
}
def test_changed_docs_empty_set_generates_nothing_but_still_prunes(
db: Session, clean_tables
) -> None:
"""``changed_docs=[]`` (a change set that IS given but empty)
restricts the generation to zero keys — no lite calls, no
re-stamps, the hook never fires — while the prune pass still runs
in the given mode (the sweep is mode-independent, phase 97)."""
_seed_three_sources(db)
asyncio.run(generate_folder_summaries(db, _FakeLLM()))
db.commit()
db.add(FolderSummary(source="A", folder_path="gone/old", summary="stale"))
db.commit()
stamps = _stamps(db, {("A", ""), ("A", "a"), ("A", "a/b"), ("B", ""), ("C", "")})
llm = _FakeLLM()
events, record = _record_progress()
stats = asyncio.run(
generate_folder_summaries(db, llm, changed_docs=[], on_progress=record)
)
assert stats == {"generated": 0, "failed": 0, "pruned": 1, "kept_manual": 0}
assert llm.calls == 0
assert events == [], "zero keys → the hook never fires"
for key, stamp in stamps.items():
assert _updated_at(db, *key) == stamp, "no row is re-stamped"
+236
View File
@@ -15,9 +15,11 @@ from __future__ import annotations
import asyncio
import logging
import os
from datetime import UTC, datetime
from pathlib import Path
import pymupdf as fitz
import pytest
from sqlalchemy import func, select
from sqlalchemy.orm import Session
@@ -28,6 +30,7 @@ from app.models import Chunk, Document
from app.rag.importer import (
EXCLUDED_DIRS,
ImportSummary,
_prune,
_store_summary,
import_sources,
iter_importable_files,
@@ -1194,3 +1197,236 @@ def test_progress_hook_with_limit_keeps_full_total(db, tmp_path: Path) -> None:
assert events == [(root.name, "a.md", 1, 3), (root.name, "b.md", 2, 3)]
finally:
_cleanup_source(db, root.name)
# ---------- phase 127 (task 01): ImportSummary.changed_paths ----------
class _VisionLLM(FakeEmbedder):
"""The phase-122 mock VISION client (the test_pdf pattern): the
multimodal describe call (``user`` content = a part list) gets a fixed
description; text (``lite``) calls keep the ``FakeEmbedder``
behaviour."""
DESCRIPTION = "A test image: a solid swatch of color."
async def chat(self, messages, model=None): # pyright: ignore[reportIncompatibleMethodOverride]
self.chat_calls.append(list(messages))
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
if isinstance(user, list):
return self.DESCRIPTION
first = user.split()
return "Summary of " + (first[0] if first else "<empty>")
def _png_bytes(width: int, height: int, rgb: tuple[float, float, float]) -> bytes:
"""One solid-color PNG built with PyMuPDF (the phase-126 no-binary-
fixtures convention)."""
pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height), 0)
for y in range(height):
for x in range(width):
pix.set_pixel(x, y, rgb)
return pix.tobytes("png")
def _build_one_page_pdf(path: Path, text: str) -> None:
"""One temp single-page PDF (the phase-126 convention — no binary
fixtures)."""
doc = fitz.open()
doc.new_page().insert_text((72, 72), text)
doc.save(path)
doc.close()
def test_import_summary_changed_paths_defaults_empty() -> None:
"""Completion criterion: default construction gives an empty change
set — no existing construction site migrates."""
assert ImportSummary().changed_paths == []
def test_changed_paths_records_added_updated_and_pruned_in_order(
db, tmp_path: Path
) -> None:
"""The contract (phase 127, task 01): one ``(source, path)`` entry per
ADDED, UPDATED, or PRUNED document — added/updated in walk order, the
pruned pair last (``_prune`` runs after the walk) — and
``len(changed_paths) == added + updated + pruned``."""
root = tmp_path / "changed"
root.mkdir()
for name in ("a.md", "b.md", "c.md"):
(root / name).write_text(f"# {name}\n\nv1\n")
llm = FakeEmbedder()
try:
source = root.name
first = asyncio.run(import_sources([root], llm, session=db))
assert first.added == 3 and first.pruned == 0
assert first.changed_paths == [
(source, "a.md"),
(source, "b.md"),
(source, "c.md"),
]
# Run 2: update a.md, add d.md + e.md, delete c.md (→ pruned).
(root / "a.md").write_text("# a\n\nv2 changed\n")
(root / "d.md").write_text("# d\n\nnew d\n")
(root / "e.md").write_text("# e\n\nnew e\n")
(root / "c.md").unlink()
second = asyncio.run(import_sources([root], llm, session=db, prune=True))
assert (second.added, second.updated, second.unchanged, second.pruned) == (
2, 1, 1, 1,
)
assert second.changed_paths == [
(source, "a.md"), # updated — first in walk order
(source, "d.md"), # added
(source, "e.md"), # added
(source, "c.md"), # pruned — the _prune pass runs after the walk
]
assert len(second.changed_paths) == (
second.added + second.updated + second.pruned
)
finally:
_cleanup_source(db, root.name)
def test_changed_paths_empty_on_unchanged_rerun(db, tmp_path: Path) -> None:
"""An all-unchanged re-run (prune included) records nothing."""
root = tmp_path / "nochange"
root.mkdir()
(root / "a.md").write_text("# a\n\nstable\n")
(root / "b.md").write_text("# b\n\nstable\n")
llm = FakeEmbedder()
try:
asyncio.run(import_sources([root], llm, session=db))
second = asyncio.run(import_sources([root], llm, session=db, prune=True))
assert second.unchanged == 2
assert second.changed_paths == []
finally:
_cleanup_source(db, root.name)
def test_changed_paths_empty_on_backfill_rerun(db, tmp_path: Path) -> None:
"""A backfill (phase 118: unchanged doc, NULL summary) is NOT a content
change: the add still records, the backfill re-run stays empty."""
root = tmp_path / "backfill"
root.mkdir()
(root / "a.yaml").write_text("alpha x\n")
try:
first = asyncio.run(import_sources([root], _FailingChatEmbedder(), session=db))
assert first.added == 1 and first.summary_errors == 1
assert first.changed_paths == [(root.name, "a.yaml")] # the ADD records
doc = db.scalar(
select(Document).where(Document.source == root.name, Document.path == "a.yaml")
)
assert doc is not None and doc.summary is None
second = asyncio.run(import_sources([root], FakeEmbedder(), session=db))
assert second.unchanged == 1 and second.summary_backfilled == 1
assert second.changed_paths == []
finally:
_cleanup_source(db, root.name)
def test_changed_paths_empty_on_date_only_rerun(db, tmp_path: Path) -> None:
"""A date-only refresh (phase 106, D4: content untouched, mtime moved)
is NOT a content change: the re-run records nothing."""
root = tmp_path / "dateonly"
root.mkdir()
path = root / "a.md"
path.write_text("# a\n\nstable\n")
os.utime(path, (0, datetime(2020, 1, 1, tzinfo=UTC).timestamp()))
llm = FakeEmbedder()
try:
first = asyncio.run(import_sources([root], llm, session=db))
assert first.added == 1 and first.dates_updated == 0
assert first.changed_paths == [(root.name, "a.md")]
# The content is untouched — only the mtime moved (the D4
# date-refresh on the unchanged path).
os.utime(path, (0, datetime(2021, 6, 15, tzinfo=UTC).timestamp()))
second = asyncio.run(import_sources([root], llm, session=db))
assert second.unchanged == 1 and second.dates_updated == 1
assert second.changed_paths == []
finally:
_cleanup_source(db, root.name)
def test_changed_paths_records_a_changed_image(db, tmp_path: Path) -> None:
"""The image increment site (phase 122 path): a NEW image and a
byte-CHANGED image each record their ``(source, rel)``."""
root = tmp_path / "imgsrc"
root.mkdir()
(root / "pic.png").write_bytes(_png_bytes(1, 1, (1.0, 0.0, 0.0)))
llm = _VisionLLM()
llm.settings = Settings(
_env_file=None, # pyright: ignore[reportCallIssue]
images=True,
image_dir=str(tmp_path / "images"),
)
try:
source = root.name
first = asyncio.run(import_sources([root], llm, session=db))
assert first.added == 1 and first.images_failed == 0
assert first.changed_paths == [(source, "pic.png")]
# The BYTES changed (a new color) — the image digest is over the
# bytes, so this is an UPDATE.
(root / "pic.png").write_bytes(_png_bytes(1, 1, (0.0, 1.0, 1.0)))
second = asyncio.run(import_sources([root], llm, session=db))
assert second.updated == 1
assert second.changed_paths == [(source, "pic.png")]
finally:
_cleanup_source(db, root.name)
def test_changed_paths_records_a_changed_pdf(db, tmp_path: Path) -> None:
"""The PDF increment site (phase 126 path): a NEW PDF and a text-
CHANGED PDF each record their ``(source, rel)``."""
root = tmp_path / "pdfsrc"
root.mkdir()
pdf_path = root / "doc.pdf"
_build_one_page_pdf(pdf_path, "alpha page one")
llm = _VisionLLM()
llm.settings = Settings(
_env_file=None, # pyright: ignore[reportCallIssue]
images=True,
image_dir=str(tmp_path / "images"),
import_extensions="md,txt,pdf",
)
try:
source = root.name
first = asyncio.run(import_sources([root], llm, session=db))
assert first.added == 1
assert first.changed_paths == [(source, "doc.pdf")]
# The extracted TEXT changed — the PDF digest is over it, so this
# is an UPDATE.
_build_one_page_pdf(pdf_path, "bravo page one, revised")
second = asyncio.run(import_sources([root], llm, session=db))
assert second.updated == 1
assert second.changed_paths == [(source, "doc.pdf")]
finally:
_cleanup_source(db, root.name)
def test_prune_records_changed_paths_only_when_summary_given(db, tmp_path: Path) -> None:
"""The ``_prune`` recording seam: with NO summary (the pre-phase
count-only contract) nothing is recorded; with one, each ACTUALLY
deleted doc appends its ``(source, path)`` (skipped docs never do)."""
root = tmp_path / "prunit"
root.mkdir()
(root / "gone.md").write_text("# gone\n\nbye\n")
(root / "gone2.md").write_text("# gone2\n\nbye\n")
llm = FakeEmbedder()
try:
source = root.name
asyncio.run(import_sources([root], llm, session=db))
(root / "gone.md").unlink()
(root / "gone2.md").unlink()
# Default call (summary=None): count only, no recording.
pruned = _prune(db, {source}, {(source, "gone2.md")})
assert pruned == 1
assert db.scalar(
select(Document).where(Document.source == source, Document.path == "gone.md")
) is None
# With a summary: the deleted doc records its pair.
counters = ImportSummary()
pruned = _prune(db, {source}, set(), summary=counters)
assert pruned == 1
assert counters.changed_paths == [(source, "gone2.md")]
finally:
_cleanup_source(db, root.name)
+5 -2
View File
@@ -40,7 +40,7 @@ from __future__ import annotations
import asyncio
import re
import threading
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterator, Sequence
from datetime import datetime
from pathlib import Path
@@ -819,12 +819,15 @@ def _patch_sync_seams(
*,
skip: bool = False,
only_missing: bool = False,
changed_docs: Sequence[tuple[str, str]] | None = None, # phase 127
on_progress: Callable[[int, int, str, str], None] | None = None,
) -> dict[str, int]:
# Phase 98 (task 01): the runner wires the progress hook into
# the generation branches — the fake accepts it (the state-
# machine tests' canned run never fires it: zero summary
# counters, like a no-gap skip).
# counters, like a no-gap skip). Phase 127 (task 03): the
# changed-KB branch passes the change set as changed_docs= —
# the fake accepts it (these canned runs record nothing).
return {"generated": 0, "failed": 0, "pruned": 0}
monkeypatch.setattr(sync_api, "generate_folder_summaries", fake_folder_summaries)