feat(sources): real-time file progress for sync and upload — background upload with success toast

tests/unit/test_importer.py

@@ -18,6 +18,7 @@ from pathlib import Path
 import pytest
 from sqlalchemy import func, select
 
+import app.rag.importer as importer
 from app.config import Settings
 from app.models import Chunk, Document
 from app.rag.importer import (
@@ -708,3 +709,140 @@ def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -
         ) is not None
     finally:
         _cleanup_source(db, root.name)
+
+
+# ---------- phase 64 (task 01): optional per-file progress hook ----------
+
+
+def test_progress_hook_reports_every_file_in_order_across_roots(
+    db, tmp_path: Path
+) -> None:
+    """Multi-root, multi-file: the hook receives the exact
+    ``(source, rel, done, total)`` sequence — roots in *sources* order,
+    ``rel`` the same POSIX path the doc rows use, ``done`` the 1-based
+    index across **all** sources, ``total`` the combined count."""
+    root_a = tmp_path / "Alpha"
+    root_b = tmp_path / "Beta"
+    root_a.mkdir()
+    (root_b / "sub").mkdir(parents=True)
+    (root_a / "a1.md").write_text("# A1\n\na one\n")
+    (root_a / "a2.md").write_text("# A2\n\na two\n")
+    (root_a / "a1.md").write_text("# A1\n\na one\n")
+    (root_b / "sub" / "b1.md").write_text("# B1\n\nb one\n")
+    events: list[tuple[str, str, int, int]] = []
+
+    def progress(source: str, rel: str, done: int, total: int) -> None:
+        events.append((source, rel, done, total))
+
+    try:
+        summary = asyncio.run(
+            import_sources([root_a, root_b], FakeEmbedder(), session=db, progress=progress)
+        )
+        assert summary.files == 3 and summary.added == 3
+        assert events == [
+            ("Alpha", "a1.md", 1, 3),
+            ("Alpha", "a2.md", 2, 3),
+            ("Beta", "sub/b1.md", 3, 3),  # POSIX rel, sorted within the root
+        ]
+    finally:
+        _cleanup_source(db, "Alpha")
+        _cleanup_source(db, "Beta")
+
+
+def test_no_progress_means_no_prewalk(
+    db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """``progress=None`` callers pay no extra pass: the real walker is hit
+    exactly once per source root (one pass — as before phase 64), proven
+    with a counting sentinel; with the hook it is hit twice (pre-walk for
+    ``total`` + the processing pass)."""
+    root = tmp_path / "nowalk"
+    root.mkdir()
+    (root / "a.md").write_text("# A\n\na\n")
+    real_walker = importer.iter_importable_files
+    walk_calls = 0
+
+    def counting(
+        r: Path, extensions: frozenset[str], excluded: frozenset[str] = EXCLUDED_DIRS
+    ) -> list[Path]:
+        nonlocal walk_calls
+        walk_calls += 1
+        return real_walker(r, extensions, excluded)
+
+    monkeypatch.setattr(importer, "iter_importable_files", counting)
+    try:
+        summary = asyncio.run(import_sources([root], FakeEmbedder(), session=db))
+        assert summary.files == 1
+        assert walk_calls == 1  # exactly one pass — the pre-change behaviour
+        walk_calls = 0
+        events: list[tuple[str, str, int, int]] = []
+        summary2 = asyncio.run(
+            import_sources(
+                [root], FakeEmbedder(), session=db,
+                progress=lambda s, r, d, t: events.append((s, r, d, t)),
+            )
+        )
+        assert summary2.files == 1
+        assert walk_calls == 2  # pre-walk (total) + processing pass
+        assert events == [(root.name, "a.md", 1, 1)]
+    finally:
+        _cleanup_source(db, root.name)
+
+
+def test_progress_hook_counts_unchanged_and_error_files(db, tmp_path: Path) -> None:
+    """The hook fires *before* ``_index_file``: a file whose embedding
+    fails (and one that re-imports as unchanged) is still reported as the
+    current file — the sequence covers every importable file."""
+    root = tmp_path / "progress-mixed"
+    root.mkdir()
+    (root / "bad.md").write_text("# Bad\n\npoison content the endpoint refuses\n")
+    (root / "good.md").write_text("# Good\n\nperfectly fine content\n")
+    expected = [(root.name, "bad.md", 1, 2), (root.name, "good.md", 2, 2)]
+    try:
+        events: list[tuple[str, str, int, int]] = []
+        first = asyncio.run(
+            import_sources(
+                [root], _PoisonEmbedder(), session=db,
+                progress=lambda s, r, d, t: events.append((s, r, d, t)),
+            )
+        )
+        # bad.md was already reported (done=1) when its embed raised —
+        # no file silently disappears from the sequence.
+        assert events == expected
+        assert first.errors == 1 and first.added == 1
+        # Re-run: good.md is now unchanged, bad.md is retried and fails
+        # again — both still count in the sequence.
+        events.clear()
+        second = asyncio.run(
+            import_sources(
+                [root], _PoisonEmbedder(), session=db,
+                progress=lambda s, r, d, t: events.append((s, r, d, t)),
+            )
+        )
+        assert events == expected
+        assert second.errors == 1 and second.unchanged == 1
+    finally:
+        _cleanup_source(db, root.name)
+
+
+def test_progress_hook_with_limit_keeps_full_total(db, tmp_path: Path) -> None:
+    """The debug ``limit`` path is unchanged for the hook: it fires only
+    for processed files (``done`` never exceeds the limit), while
+    ``total`` stays the FULL pre-walk count — an incomplete walk must not
+    misreport the denominator."""
+    root = tmp_path / "progress-limited"
+    root.mkdir()
+    for name in ("a.md", "b.md", "c.md"):
+        (root / name).write_text(f"# {name}\n\nbody {name}\n")
+    events: list[tuple[str, str, int, int]] = []
+    try:
+        summary = asyncio.run(
+            import_sources(
+                [root], FakeEmbedder(), limit=2, session=db,
+                progress=lambda s, r, d, t: events.append((s, r, d, t)),
+            )
+        )
+        assert summary.files == 2
+        assert events == [(root.name, "a.md", 1, 3), (root.name, "b.md", 2, 3)]
+    finally:
+        _cleanup_source(db, root.name)