"""Integration: the admin archive-upload API (phase 49, task 02; backgrounded in phase 64, task 03). Real Postgres (``podman compose up -d db``); the upload dir is pointed at a fresh tmp dir per test by monkeypatching the router's ``get_settings`` (the ``test_git_sources_api.py`` pattern — the dev ``.env`` never leaks in), and the scan uses the deterministic in-process ``FakeEmbedder`` (``test_sync_api.py``'s ``_real_llm`` pattern — real import, no network). Phase 64 (task 03) contract under test: ``POST`` answers **202** the moment the archive is safely on disk (an ``UploadAccepted`` body naming the safe source), and the rest — unpack → swap → row upsert → model check → scan → change-gated overview — runs in a **background task** behind ``GET /api/git-sources/upload/status`` (the phase-32 ``SyncStatus`` shape, incl. the phase-64 ``current_file`` / ``files_done`` / ``files_total``). Every post-receive scenario observes the **status endpoint, polled until terminal** — never the HTTP response; post-202 failures are status states (``failed`` + sanitized error, A5), never HTTP errors. Navigating away mid-scan no longer aborts anything: the task is created before the 202 and its outcome lands in the module-level status. The inline gates are unchanged: * anonymous → 403 ``{"detail": "admin only"}`` (the router's ``require_admin`` covers the new status route too); * name/format gate → 422: a non-archive extension names the accepted formats; a ``..`` / separator / empty-stem name (including a bare ``tar.gz``) is rejected with the task-01 message — and the upload dir is never created for a rejected name (control characters never reach the app: the multipart transport percent-encodes them — the task-01 branch for them is covered in ``test_archive_upload.py``); * one upload at a time → 409 ``an upload is already in progress`` — the gate is the module-level **flag** (checked and set with no await in between, BEFORE the receive — the handler now awaits before the background task exists), held through the whole background run; * streaming cap → 413 naming the ``upload_max_mb`` cap; the temp ``.upload`` file is removed and the flag is released (no stray ``.`` files in the upload dir; the next upload is not 409). Background branches (A5): unpack failure (zip-slip, tar symlink escape, corrupt archive), zero-entry archive, swap failure, the concurrent-insert ``IntegrityError`` backstop, dead models, a cancelled run (the ``finally`` cleans both temps + releases the flag), and the success path — each lands in the status as ``failed`` (with the sanitized error) or ``success`` (with the ``UploadOut`` fields in ``detail``); a FAILED upload **never** touches the previous folder, row, or KB of an earlier good upload (the no-partial-state locked decision, asserted explicitly). ``git_sources`` / ``documents`` / ``chunks`` / ``kb_overview`` are global state: truncated around every test. """ from __future__ import annotations import asyncio import io import logging import os import re import tarfile import threading import time import zipfile from collections.abc import Callable, Iterator from pathlib import Path from typing import Any, Literal import pytest from fastapi.testclient import TestClient from sqlalchemy import select, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from app.api import git_sources as git_sources_api from app.config import Settings from app.db import SessionLocal from app.main import app as fastapi_app from app.models import GitSource from app.rag.archive_upload import ArchiveUploadError, unpack_archive from app.rag.importer import ImportSummary from app.rag.llm import EmbeddingError, ModelUnavailableError from tests.conftest import ADMIN_PASSWORD from tests.fakes import FakeEmbedder @pytest.fixture(autouse=True) def clean_git_sources(db: Session) -> Iterator[None]: """The stored list is global state: reset around every test.""" db.execute(text("TRUNCATE git_sources")) db.commit() yield db.execute(text("TRUNCATE git_sources")) db.commit() @pytest.fixture(autouse=True) def clean_documents(db: Session) -> Iterator[None]: """The happy path writes ``documents``/``chunks`` and (via the change-gated overview refresh) ``kb_overview`` — global, truncated around every test.""" db.execute(text("TRUNCATE chunks, documents, kb_overview")) db.commit() yield db.execute(text("TRUNCATE chunks, documents, kb_overview")) db.commit() @pytest.fixture(autouse=True) def fresh_upload_status() -> Iterator[None]: """The module-level upload status + flag are process-global: reset them before AND after every test (the ``fresh_sync_status`` pattern from ``tests/unit/test_sync_button.py``) — a previous test's terminal state must never leak into the next one's idle-shape assertions or its 409 expectations.""" git_sources_api._upload_status = git_sources_api.UploadStatus() git_sources_api._upload_in_progress = False yield git_sources_api._upload_status = git_sources_api.UploadStatus() git_sources_api._upload_in_progress = False @pytest.fixture() def upload_client() -> Iterator[TestClient]: """Admin-signed client (the context-manager form keeps one event loop for the whole test — the background task the 202 creates runs on it, so the client must stay open until the run is observed in a terminal state).""" with TestClient(fastapi_app) as client: r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}" yield client def _point_at(monkeypatch: pytest.MonkeyPatch, upload_dir: Path, upload_max_mb: int = 512) -> None: """Fresh settings on the router's module: the tmp upload dir and (optionally) a shrunk cap — the dev ``.env`` never leaks in.""" monkeypatch.setattr( git_sources_api, "get_settings", lambda: Settings( _env_file=None, # pyright: ignore[reportCallIssue] upload_dir=str(upload_dir), upload_max_mb=upload_max_mb, ), ) def _real_llm(monkeypatch: pytest.MonkeyPatch) -> None: """The pipeline's ``LLMClient`` becomes the deterministic in-process ``FakeEmbedder`` (real import, no network); it also implements ``embed_one``/``chat``, so the phase-41 probe passes.""" monkeypatch.setattr(git_sources_api, "LLMClient", lambda: FakeEmbedder()) #: tarfile write modes used by the tests (uncompressed + gzip). _TAR_WRITE_MODES = Literal["w", "w:gz"] def _tarball(path: Path, files: dict[str, str], compress: _TAR_WRITE_MODES = "w:gz") -> Path: with tarfile.open(path, compress) as tf: for rel, content in files.items(): data = content.encode("utf-8") info = tarfile.TarInfo(rel) info.size = len(data) info.mtime = 0 tf.addfile(info, io.BytesIO(data)) return path def _zip(path: Path, files: dict[str, str]) -> Path: with zipfile.ZipFile(path, "w") as zf: for rel, content in files.items(): zf.writestr(rel, content) return path def _targz_bytes(files: dict[str, str]) -> bytes: """A tarball in memory (no temp file on disk).""" buf = io.BytesIO() with tarfile.open(fileobj=buf, mode="w:gz") as tf: for rel, content in files.items(): data = content.encode("utf-8") info = tarfile.TarInfo(rel) info.size = len(data) info.mtime = 0 tf.addfile(info, io.BytesIO(data)) return buf.getvalue() def _post(client: TestClient, filename: str, payload: bytes): """One upload request (multipart, the page's exact shape).""" return client.post( "/api/git-sources/upload", files={"file": (filename, payload, "application/octet-stream")}, ) def _wait_status(client: TestClient, timeout: float = 30.0) -> dict[str, Any]: """Poll ``GET /api/git-sources/upload/status`` until the run is terminal (``success`` / ``failed``) and return the terminal body. Phase 64: the 202 answers BEFORE the scan — every post-receive assertion observes the status endpoint, not the HTTP response.""" deadline = time.monotonic() + timeout body: dict[str, Any] = {} while True: r = client.get("/api/git-sources/upload/status") assert r.status_code == 200, r.text body = r.json() if body["state"] in ("success", "failed"): return body if time.monotonic() >= deadline: raise AssertionError(f"upload never reached a terminal state: {body}") time.sleep(0.05) def _row(db: Session, path: str) -> GitSource | None: db.expire_all() return db.scalar(select(GitSource).where(GitSource.path == path)) def _docs(client: TestClient) -> list[tuple[str, str]]: body = client.get("/api/docs").json() return [(d["source"], d["path"]) for d in body["documents"]] def _count_rows(db: Session) -> int: return db.execute(text("SELECT count(*) FROM git_sources")).scalar_one() def _good_upload( client: TestClient, name: str = "safe", files: dict[str, str] | None = None ) -> None: """A successful upload of a two-sentinel tarball (202 → status ``success``) — the baseline state the no-partial-state tests protect.""" payload_files = files or { "alpha.md": "# Alpha\noriginal sentinel one\n", "bravo.md": "# Bravo\noriginal sentinel two\n", } r = _post(client, f"{name}.tar.gz", _targz_bytes(payload_files)) assert r.status_code == 202, r.text assert _wait_status(client)["state"] == "success" def _assert_previous_intact( client: TestClient, db: Session, upload_dir: Path, name: str, files: dict[str, str] ) -> None: """The no-partial-state locked decision, asserted explicitly: after a FAILED upload the previous folder's content, the row (same ``added_at``), and the KB are all exactly as the good upload left them — and no temp file survived.""" folder = upload_dir / name assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == files assert _row(db, str(folder)) is not None # the row is still there assert _count_rows(db) == 1 # …and no second row appeared assert _docs(client) == [(name, "alpha.md"), (name, "bravo.md")] assert [p.name for p in upload_dir.iterdir()] == [name] # no stray temp class _GatedImport: """The mock import (the task-02 seam from ``tests/unit/test_sync_button.py``): fires the runner's OWN progress hook once (the progress-shaped call goes through the real hook closure — the closure under test), parks on a threading gate so the test can read the status mid-run, then returns the canned summary (or raises ``fail``).""" def __init__( self, summary: ImportSummary, started: threading.Event, release: threading.Event, fail: BaseException | None = None, ) -> None: self.summary = summary self.started = started self.release = release self.fail = fail self.hook_calls: list[tuple[str, str, int, int]] = [] self.prune_flags: list[bool] = [] async def __call__( self, sources: list[Path], llm: object, *, prune: bool = False, limit: int | None = None, session: object = None, progress: Callable[[str, str, int, int], None] | None = None, ignore_by_root: dict[str, list[str]] | None = None, # phase 89 ) -> ImportSummary: self.prune_flags.append(prune) if progress is not None: progress("homelab", "notes/deep.md", 1, 3) self.hook_calls.append(("homelab", "notes/deep.md", 1, 3)) self.started.set() await asyncio.to_thread(self.release.wait, 30.0) if self.fail is not None: raise self.fail return self.summary class _GatedUnpack: """Parks inside ``unpack_archive`` — a SYNCHRONOUS seam (the app loop blocks while parked — only usable when the run is driven directly on a worker loop, never through HTTP, where the blocked loop would also stall the 202's response delivery) — then runs the REAL unpack so the run completes for real.""" def __init__(self, started: threading.Event, release: threading.Event) -> None: self.started = started self.release = release def __call__(self, archive: Path, target: Path, max_bytes: int) -> None: self.started.set() self.release.wait(30.0) unpack_archive(archive, target, max_bytes) class _ImmediateImport: """A mock import that returns the canned summary immediately (no gate — the gate for that test lives in unpack instead).""" def __init__(self, summary: ImportSummary) -> None: self.summary = summary async def __call__(self, sources: list[Path], llm: object, **kwargs: object) -> ImportSummary: return self.summary # --- anonymous ------------------------------------------------------------- def test_anonymous_upload_and_status_get_403(client: TestClient, db: Session) -> None: r = _post(client, "homelab.tar.gz", b"not an archive at all") assert r.status_code == 403 assert r.json() == {"detail": "admin only"} # The status route sits behind the same router-level dependency. r = client.get("/api/git-sources/upload/status") assert r.status_code == 403 assert r.json() == {"detail": "admin only"} assert _count_rows(db) == 0 def test_idle_upload_status_pins_full_shape(upload_client: TestClient) -> None: """Idle: the full response dict is pinned — identical key set to ``GET /api/sync/status`` (phase 32), progress keys null/0/0.""" assert upload_client.get("/api/git-sources/upload/status").json() == { "state": "idle", "started_at": None, "finished_at": None, "detail": {}, "error": None, "current_file": None, "files_done": 0, "files_total": 0, } # --- name / format gate ----------------------------------------------------- def test_non_archive_extension_gets_422_naming_formats( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) r = _post(upload_client, "notes.txt", b"hello world") assert r.status_code == 422 assert r.json()["detail"] == "only .tar, .tar.gz, .tgz or .zip archives are accepted" assert _count_rows(db) == 0 # The name gate runs before the dir is created — nothing on disk. assert not uploads.exists() @pytest.mark.parametrize( "filename", [ "../evil.zip", # ``..`` + separator "sub/dir.tar.gz", # separator "tar.gz", # bare suffix → empty stem ".tar.gz", # empty stem # (control characters in the filename are percent-encoded by the # multipart transport before the app ever sees them — the # ``archive_source_name`` branch for them is covered in # tests/unit/test_archive_upload.py) ], ) def test_unsafe_name_gets_422( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path, filename: str, ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) r = _post(upload_client, filename, b"content") assert r.status_code == 422, f"{filename!r} must be rejected: {r.text}" assert r.json()["detail"] != "only .tar, .tar.gz, .tgz or .zip archives are accepted" assert _count_rows(db) == 0 assert not uploads.exists() # --- one at a time (409) ---------------------------------------------------- def test_second_upload_while_one_is_in_flight_returns_409( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """The 202 answers BEFORE the scan: while the first run is in flight (parked inside its blocked scan), the second upload is 409 — the module-level flag, held from the first receive, is the gate — and after the first run lands in a terminal state, uploads are accepted again (a full real run this time).""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"}) started = threading.Event() release = threading.Event() fake_import = _GatedImport(ImportSummary(files=1, unchanged=1), started, release) monkeypatch.setattr(git_sources_api, "import_sources", fake_import) with TestClient(fastapi_app) as client: assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204 r = _post(client, "homelab.tar.gz", archive.read_bytes()) assert r.status_code == 202, r.text assert r.json() == {"detail": "upload received", "name": "homelab"} assert started.wait(15.0), "background run did not reach its scan" # The run is in flight: the flag (not the task's done-ness) is # held — the second upload is 409. r = _post(client, "homelab.tar.gz", archive.read_bytes()) assert r.status_code == 409 assert r.json() == {"detail": "an upload is already in progress"} release.set() status = _wait_status(client) assert status["state"] == "success", status # The flag was released: the next upload goes through for real. with TestClient(fastapi_app) as third: assert third.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204 r = _post(third, "homelab.tar.gz", archive.read_bytes()) assert r.status_code == 202, r.text assert _wait_status(third)["state"] == "success" def test_upload_refused_while_flag_held( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: """The smallest test seam: the module-level flag itself. Held → 409 and nothing happens (no dir, no row) — even though the status is still ``idle`` (no run exists at all): the flag, not a task/state check, is the gate. That is what the receive window needs — the handler awaits the (1 MiB-chunk) streaming receive BEFORE the background task exists, so only the flag can stop a concurrent POST from starting a second run.""" _point_at(monkeypatch, tmp_path / "uploads") monkeypatch.setattr(git_sources_api, "_upload_in_progress", True) r = _post(upload_client, "a.tar.gz", b"x") assert r.status_code == 409 assert r.json() == {"detail": "an upload is already in progress"} # The flag — not the status state — is the gate: assert upload_client.get("/api/git-sources/upload/status").json()["state"] == "idle" assert _count_rows(db) == 0 assert not (tmp_path / "uploads").exists() # --- the 202 lands with the archive on disk (A2) ----------------------------- def test_202_answers_before_the_scan_completes( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """The 202 is the "successfully uploaded" moment (A2), not the scan result: the response lands while the scan is still IN FLIGHT (parked inside its gated import) — the status is ``running``, the outcome not yet there — and only later does the run complete (the outcome lands in the status, which the UI polls). This is the navigate-away contract, unit-level: the background task is created before the response, and its outcome lands in ``_upload_status`` — never in the response.""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"}) started = threading.Event() release = threading.Event() fake_import = _GatedImport(ImportSummary(files=1, added=1), started, release) monkeypatch.setattr(git_sources_api, "import_sources", fake_import) with TestClient(fastapi_app) as client: assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204 r = _post(client, "homelab.tar.gz", archive.read_bytes()) assert r.status_code == 202, r.text assert r.json() == {"detail": "upload received", "name": "homelab"} # The response landed BEFORE the scan: the run is in flight on # this client's loop and has not completed (the scan is parked). assert started.wait(15.0), "background run did not reach its scan" s = client.get("/api/git-sources/upload/status").json() assert s["state"] == "running" assert s["detail"] == {} and s["error"] is None assert s["started_at"] is not None and s["finished_at"] is None # Mid-flight: unpack + swap already landed the folder (the # archive was safely on disk at the 202 moment), the scan has # not run yet. assert (uploads / "homelab").is_dir() release.set() s = _wait_status(client) assert s["state"] == "success", s assert s["detail"]["added"] == 1 # No dotfile temp survived the run. assert [p.name for p in uploads.iterdir()] == ["homelab"] def test_background_run_starts_with_the_archive_on_disk( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """A2 pinned at the runner level: the background run receives the archive SAFELY ON DISK — parked inside unpack (before the real unpack and the post-unpack unlink), the dotfile temp upload file still exists, and the module-level status (what ``GET /upload/status`` reads) is ``running`` with no file yet (A4: "Processing…"). After release the run completes for real (real unpack + swap) and no dotfile temp survives. ``_run_upload`` is driven directly on a worker loop (the house background-task pattern from ``test_sync_button.py``) — the park is a synchronous block, which would also stall the 202's response delivery through HTTP, so the runner-level observation is the deterministic one.""" uploads = tmp_path / "uploads" uploads.mkdir() _point_at(monkeypatch, uploads) async def _no_probe(llm: object) -> None: pass monkeypatch.setattr(git_sources_api, "check_models", _no_probe) monkeypatch.setattr(git_sources_api, "LLMClient", lambda: object()) monkeypatch.setattr(git_sources_api, "import_sources", _ImmediateImport( ImportSummary(files=1, unchanged=1) )) class _DummySession: def scalar(self, statement: object) -> None: return None def add(self, row: object) -> None: pass def commit(self) -> None: pass def close(self) -> None: pass monkeypatch.setattr(git_sources_api, "SessionLocal", _DummySession) # A real, valid archive in the temp upload file — exactly what the # handler leaves behind before the 202. real_archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"}) temp_upload = uploads / ".homelab.0.upload" temp_unpack = uploads / ".homelab.0.unpack" temp_upload.write_bytes(real_archive.read_bytes()) # The handler holds the flag when it creates the task — simulate. git_sources_api._upload_in_progress = True started = threading.Event() release = threading.Event() monkeypatch.setattr(git_sources_api, "unpack_archive", _GatedUnpack(started, release)) errors: list[BaseException] = [] def _run() -> None: try: asyncio.run( git_sources_api._run_upload( "homelab", "homelab.tar.gz", len(real_archive.read_bytes()), uploads, temp_upload, temp_unpack, ) ) except BaseException as e: # noqa: BLE001 — surfaced to the test errors.append(e) thread = threading.Thread(target=_run, daemon=True) thread.start() try: assert started.wait(15.0), "the run never reached its unpack" # The archive is on disk while the run is in flight (A2): the # temp upload file exists — the run is parked before the real # unpack and the post-unpack unlink. assert temp_upload.is_file() # The module-level status (what GET /upload/status reads) is # running, with no file yet (A4). status = git_sources_api._upload_status assert status.state == "running" assert status.current_file is None assert status.files_done == 0 and status.files_total == 0 assert status.detail == {} and status.error is None assert status.started_at is not None and status.finished_at is None release.set() thread.join(20) finally: release.set() assert not thread.is_alive() assert errors == [], f"the run raised: {errors!r}" terminal = git_sources_api._upload_status assert terminal.state == "success" assert terminal.detail["source"] == "homelab" # The folder is in place and no dotfile temp survived the run. assert [p.name for p in uploads.iterdir()] == ["homelab"] assert git_sources_api._upload_in_progress is False # --- mid-run progress: current_file + counts (phase 64) ---------------------- @pytest.mark.parametrize( "fail", [ pytest.param(None, id="success-terminal"), pytest.param( EmbeddingError( "embeddings request to https://u:p@aipi.example.com/v1 " "failed: connection refused" ), id="failed-terminal", ), ], ) def test_mid_run_status_reports_current_file_then_terminal_clears_it( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, db: Session, fail: BaseException | None, ) -> None: """Mid-run: the status carries the file the (mock) scan is processing — assigned through the runner's own hook closure (the task-02 seam: the progress-shaped call goes through the real hook). Terminal states — BOTH ``success`` and ``failed`` — clear ``current_file`` but keep the run's final counts; the failure error is sanitized (credentials masked).""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"}) started = threading.Event() release = threading.Event() fake_import = _GatedImport( ImportSummary(files=3, added=1, updated=1, unchanged=1), started, release, fail ) monkeypatch.setattr(git_sources_api, "import_sources", fake_import) with TestClient(fastapi_app) as client: assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204 r = _post(client, "homelab.tar.gz", archive.read_bytes()) assert r.status_code == 202, r.text assert started.wait(15.0), "the run never reached its scan" # While the scan is parked: the hook's file is live on the status. s = client.get("/api/git-sources/upload/status").json() assert s["state"] == "running" assert s["current_file"] == "homelab/notes/deep.md" assert s["files_done"] == 1 assert s["files_total"] == 3 assert s["error"] is None assert s["started_at"] is not None and s["finished_at"] is None release.set() s = _wait_status(client) # Wiring: prune=True is preserved, and the hook fired through the # runner's own closure (the recorded call is what the closure # assigned to the status above). assert fake_import.prune_flags == [True] assert fake_import.hook_calls == [("homelab", "notes/deep.md", 1, 3)] # Terminal: current_file null, final counts retained. assert s["current_file"] is None assert s["files_done"] == 1 and s["files_total"] == 3 assert s["finished_at"] is not None if fail is None: assert s["state"] == "success", s assert s["error"] is None assert s["detail"]["source"] == "homelab" assert s["detail"]["files"] == 3 else: assert s["state"] == "failed", s assert s["detail"] == {} error = s["error"] or "" assert "*****@aipi.example.com" in error # credentials masked assert "u:p" not in error # A5: a post-swap failure keeps folder + row — the next # sync/re-upload retries idempotently. folder = uploads / "homelab" assert folder.is_dir() assert _row(db, str(folder)) is not None # --- cancellation: the finally cleans temps + releases the flag -------------- def test_cancelled_run_cleans_temps_and_releases_the_flag( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """App shutdown cancels the background task: ``CancelledError`` is deliberately NOT caught (the ``_run_sync`` rule — swallowing it would mask a real stop), but the ``finally`` still runs: both temps are gone and the flag is released, so nothing lingers after the stop. ``_run_upload`` is driven directly on a worker loop (the house background-task pattern from ``test_sync_button.py``); every seam is faked, so no DB and no HTTP are needed.""" uploads = tmp_path / "uploads" uploads.mkdir() _point_at(monkeypatch, uploads) async def _no_probe(llm: object) -> None: pass monkeypatch.setattr(git_sources_api, "check_models", _no_probe) monkeypatch.setattr(git_sources_api, "LLMClient", lambda: object()) monkeypatch.setattr(git_sources_api, "unpack_archive", lambda a, t, cap: None) monkeypatch.setattr(git_sources_api, "swap_in", lambda new_dir, final_dir: None) started = threading.Event() release = threading.Event() monkeypatch.setattr( git_sources_api, "import_sources", _GatedImport(ImportSummary(), started, release), ) class _DummySession: def scalar(self, statement: object) -> None: return None def add(self, row: object) -> None: pass def commit(self) -> None: pass def close(self) -> None: pass monkeypatch.setattr(git_sources_api, "SessionLocal", _DummySession) # The handler holds the flag when it creates the task — simulate. git_sources_api._upload_in_progress = True temp_upload = uploads / ".homelab.0.upload" temp_unpack = uploads / ".homelab.0.unpack" temp_upload.write_bytes(b"compressed bytes") temp_unpack.mkdir() (temp_unpack / "alpha.md").write_text("# A\n", encoding="utf-8") task_holder: list[asyncio.Task[None]] = [] errors: list[BaseException] = [] def _run() -> None: async def _main() -> None: task = asyncio.create_task( git_sources_api._run_upload( "homelab", "homelab.tar.gz", 4, uploads, temp_upload, temp_unpack ) ) task_holder.append(task) # The app-shutdown shape: cancel the task on the loop once # the run has reached its (gated) scan. while not started.is_set(): await asyncio.sleep(0.01) task.cancel() # Free the gated import's worker thread AFTER the cancel: # a to_thread future cannot be cancelled while its thread # runs, so the task only lands in the cancelled state once # the worker is free. release.set() try: await task except asyncio.CancelledError as e: errors.append(e) try: asyncio.run(_main()) except BaseException as e: # noqa: BLE001 — surfaced to the test errors.append(e) thread = threading.Thread(target=_run, daemon=True) thread.start() try: thread.join(20) finally: release.set() # backstop — the loop already released on cancel assert not thread.is_alive() # The CancelledError propagated (not swallowed) — and the finally # did its cleanup: no temp survives, the flag is released. assert len(errors) == 1 assert isinstance(errors[0], asyncio.CancelledError) assert not temp_upload.exists() assert not temp_unpack.exists() assert git_sources_api._upload_in_progress is False # --- in-flight run must not block a concurrent TRUNCATE (regression, -------- # --- phase 49 task 03; now structural in phase 64) --------------------------- def test_in_flight_upload_does_not_block_a_concurrent_truncate( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """While the background run is in flight, a concurrent TRUNCATE of the KB + registry tables (the E2E isolation-fixture pattern) must complete. Regression for the phase-49 task-03 deadlock, now structural (phase 64): the handler no longer touches the DB at all, and the row upsert inside the run commits in its own short-lived session (closed before the scan) — so no session can hold ``git_sources`` locks across the scan. This pins it: with the scan held in flight, the TRUNCATE completes well inside its 5 s ``lock_timeout`` (without the fix it times out with a lock-not-available error).""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"}) started = threading.Event() release = threading.Event() fake_import = _GatedImport(ImportSummary(files=1, unchanged=1), started, release) monkeypatch.setattr(git_sources_api, "import_sources", fake_import) with TestClient(fastapi_app) as client: assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204 r = _post(client, "homelab.tar.gz", archive.read_bytes()) assert r.status_code == 202, r.text assert started.wait(15.0), "background run did not reach its scan" # The E2E isolation TRUNCATE, exactly as the story fixtures run # it — must complete while the scan is held in flight. with SessionLocal() as tr, tr.begin(): tr.execute(text("SET LOCAL lock_timeout = '5s'")) tr.execute( text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources") ) release.set() status = _wait_status(client) assert status["state"] == "success", status # --- streaming cap (413) ---------------------------------------------------- def test_oversized_upload_gets_413_and_leaves_no_temp( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads, upload_max_mb=1) # 2 MiB of incompressible bytes with a 1 MiB cap → 413 naming the # cap (inline — the run never starts). big = os.urandom(2 * 1024 * 1024) r = _post(upload_client, "big.zip", big) assert r.status_code == 413 assert "1 MiB" in r.json()["detail"] assert _count_rows(db) == 0 assert uploads.exists() # the dir was created before streaming assert list(uploads.iterdir()) == [] # the temp .upload file is gone # The inline exception path released the flag: the next upload is # NOT 409. Boundary: EXACTLY the cap is not 413 (the check is # strictly >) — the stream completes, 202 lands, and the garbage # bytes fail at the BACKGROUND unpack instead (a status state, A5 — # never an HTTP error post-202). r = _post(upload_client, "exact.zip", os.urandom(1024 * 1024)) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert "could not unpack the archive" in (status["error"] or "") assert list(uploads.iterdir()) == [] # --- unpack safety: failed runs leave the previous state intact --------------- def test_zip_slip_archive_fails_the_run_and_previous_state_is_intact( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = { "alpha.md": "# Alpha\noriginal sentinel one\n", "bravo.md": "# Bravo\noriginal sentinel two\n", } _good_upload(upload_client, "safe", files) evil = _zip(tmp_path / "safe.zip", {"../evil.txt": "pwned"}) r = _post(upload_client, "safe.zip", evil.read_bytes()) assert r.status_code == 202, r.text # on disk → the failure is a status state status = _wait_status(upload_client) assert status["state"] == "failed" assert "traversal" in (status["error"] or "") assert not (tmp_path / "evil.txt").exists() # the escape never landed _assert_previous_intact(upload_client, db, uploads, "safe", files) def test_tar_symlink_escape_fails_the_run_and_previous_state_is_intact( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = { "alpha.md": "# Alpha\noriginal sentinel one\n", "bravo.md": "# Bravo\noriginal sentinel two\n", } _good_upload(upload_client, "safe", files) evil = tmp_path / "safe.tar" with tarfile.open(evil, "w") as tf: info = tarfile.TarInfo("link") info.type = tarfile.SYMTYPE info.linkname = "/etc/passwd" tf.addfile(info) r = _post(upload_client, "safe.tar", evil.read_bytes()) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert "escape" in (status["error"] or "") _assert_previous_intact(upload_client, db, uploads, "safe", files) def test_corrupt_archive_fails_the_run_and_previous_state_is_intact( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = { "alpha.md": "# Alpha\noriginal sentinel one\n", "bravo.md": "# Bravo\noriginal sentinel two\n", } _good_upload(upload_client, "safe", files) # A truncated zip (the EOCD is cut off) is not a zip and not a tar. good_zip = _zip(tmp_path / "good.zip", {"alpha.md": "# Alpha\nx\n"}) good_bytes = good_zip.read_bytes() truncated = good_bytes[: len(good_bytes) // 2] r = _post(upload_client, "safe.zip", truncated) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert "could not unpack the archive" in (status["error"] or "") # A zero-byte "archive" fails the same way. r = _post(upload_client, "safe.tar", b"") assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert "could not unpack the archive" in (status["error"] or "") _assert_previous_intact(upload_client, db, uploads, "safe", files) def test_swap_failure_fails_the_run_and_previous_state_is_intact( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: """A rename failure in ``swap_in`` (OS error) → status ``failed`` with its message; the previous folder/row/KB are untouched and no temp survives (the run's finally cleans the temp sibling). A failure here leaves the previous folder/row/KB untouched (A5).""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = { "alpha.md": "# Alpha\noriginal sentinel one\n", "bravo.md": "# Bravo\noriginal sentinel two\n", } _good_upload(upload_client, "safe", files) def failing_swap(new_dir: Path, final_dir: Path) -> None: raise ArchiveUploadError("could not replace the previous folder") monkeypatch.setattr(git_sources_api, "swap_in", failing_swap) r = _post(upload_client, "safe.tar.gz", _targz_bytes({"alpha.md": "# A\nx\n"})) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert status["error"] == "could not replace the previous folder" _assert_previous_intact(upload_client, db, uploads, "safe", files) def test_zero_entry_archive_fails_the_run( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: """A completely empty archive (zero entries) → status ``failed`` ``the archive contains no files`` — both containers — with no folder, row, or temp file left behind (a pre-swap failure leaves the KB/folders/rows untouched, A5).""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) empty_targz = tmp_path / "empty.tar.gz" with tarfile.open(empty_targz, "w:gz"): pass r = _post(upload_client, "empty.tar.gz", empty_targz.read_bytes()) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert status["error"] == "the archive contains no files" empty_zip = tmp_path / "empty.zip" with zipfile.ZipFile(empty_zip, "w"): pass r = _post(upload_client, "empty.zip", empty_zip.read_bytes()) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert status["error"] == "the archive contains no files" assert _count_rows(db) == 0 assert not (uploads / "empty").exists() assert list(uploads.iterdir()) == [] def test_concurrent_row_insert_fails_the_run_with_the_path_named( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """The upsert's unique-index backstop: the pre-check sees no row, but the commit hits the unique constraint (a concurrent insert the pre-check missed) → status ``failed`` naming the path. The folder stays — the row exists (the concurrent one) and the next sync/re-upload sees it (A5); no temp survives and the short-lived session was closed.""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) class _ConcurrentInsertSession: """The short-lived upsert session, faked: the pre-check misses (no row), the commit hits the unique index.""" def __init__(self) -> None: self.closed = False self.added: list[object] = [] def scalar(self, statement: object) -> None: return None def add(self, row: object) -> None: self.added.append(row) def commit(self) -> None: raise IntegrityError("INSERT INTO git_sources", {}, Exception("duplicate key")) def rollback(self) -> None: pass def close(self) -> None: self.closed = True sessions: list[_ConcurrentInsertSession] = [] def _fake_session_factory() -> _ConcurrentInsertSession: s = _ConcurrentInsertSession() sessions.append(s) return s monkeypatch.setattr(git_sources_api, "SessionLocal", _fake_session_factory) r = _post( upload_client, "homelab.tar.gz", _tarball(tmp_path / "h.tar.gz", {"alpha.md": "# Alpha\nx\n"}).read_bytes(), ) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed" assert status["error"] == ( f"a local source with this path already exists: {uploads / 'homelab'}" ) # The folder stays (the row exists — the concurrent insert — and # the next sync sees it); no temp survived; the session was closed. assert (uploads / "homelab").is_dir() assert [p.name for p in uploads.iterdir()] == ["homelab"] assert len(sessions) == 1 assert sessions[0].added, "the run attempted the insert (the pre-check missed)" assert sessions[0].closed # --- happy path ------------------------------------------------------------- def test_happy_path_tar_gz( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path, caplog: pytest.LogCaptureFixture, ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = {"alpha.md": "# Alpha\nfirst sentinel\n", "bravo.md": "# Bravo\nsecond sentinel\n"} payload = _tarball(tmp_path / "homelab.tar.gz", files).read_bytes() with caplog.at_level(logging.INFO, logger="app.api.git_sources"): r = _post(upload_client, "homelab.tar.gz", payload) assert r.status_code == 202, r.text # 202 = the archive is on disk (A2) — the safe name rides back # in the ``UploadAccepted`` body (the toast moment). assert r.json() == {"detail": "upload received", "name": "homelab"} status = _wait_status(upload_client) assert status["state"] == "success", status assert status["error"] is None # ``detail`` carries the ``UploadOut`` fields (the pre-phase-64 # response shape — now the status's success detail). detail = status["detail"] assert set(detail) == { "source", "files", "added", "updated", "unchanged", "pruned", "errors", "chunks", "overview", } assert detail["source"] == "homelab" # filename minus the archive suffix assert detail["files"] == 2 assert detail["added"] == 2 assert detail["updated"] == 0 assert detail["unchanged"] == 0 assert detail["pruned"] == 0 assert detail["errors"] == 0 assert detail["chunks"] >= 2 assert detail["overview"] is True # the KB changed → the overview refreshed # Terminal progress state: current_file null, timestamps set. assert status["current_file"] is None assert status["started_at"] is not None and status["finished_at"] is not None # Unpacked under the tmp upload dir, dotfile temps cleaned up. folder = uploads / "homelab" assert folder.is_dir() assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == files assert [p.name for p in uploads.iterdir()] == ["homelab"] # One kind=local row; the NOT-NULL url column carries the path. row = _row(db, str(folder)) assert row is not None assert row.kind == "local" assert row.url == str(folder) assert row.path == str(folder) assert _count_rows(db) == 1 # The KB lists both files under the source name. assert _docs(upload_client) == [("homelab", "alpha.md"), ("homelab", "bravo.md")] # The per-upload log line (PLAN §9 / AGENTS.md rule 10) — moved # with the scan into the background task (``total_ms`` is the run's # duration). lines = [ rec.getMessage() for rec in caplog.records if rec.getMessage().startswith("upload: ") ] assert len(lines) == 1, lines match = re.match( r"^upload: name=homelab file=homelab\.tar\.gz bytes_in=(\d+) files=2 added=2 " r"updated=0 unchanged=0 pruned=0 errors=0 overview=True total_ms=\d+$", lines[0], ) assert match, lines[0] assert int(match.group(1)) == len(payload) # the compressed bytes in @pytest.mark.parametrize( ("filename", "builder"), [ ("notes.zip", "zip"), ("plain.tar", "tar"), ("tgz.tgz", "targz"), ], ) def test_happy_path_other_accepted_formats( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path, filename: str, builder: str, ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = {"alpha.md": "# Alpha\nsentinel\n"} make = {"zip": _zip, "tar": lambda p, f: _tarball(p, f, "w"), "targz": _tarball}[builder] payload = make(tmp_path / filename, files).read_bytes() r = _post(upload_client, filename, payload) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "success", status detail = status["detail"] expected_name = {"notes.zip": "notes", "plain.tar": "plain", "tgz.tgz": "tgz"}[filename] assert r.json()["name"] == expected_name # the safe name in the 202 body assert detail["source"] == expected_name assert detail["added"] == 1 assert _docs(upload_client) == [(expected_name, "alpha.md")] assert _count_rows(db) == 1 # --- re-upload, same name: in-place replace --------------------------------- def test_reupload_same_name_replaces_in_place( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) v1 = { "alpha.md": "# Alpha\nv1 content\n", "bravo.md": "# Bravo\nv1 content\n", "charlie.md": "# Charlie\nv1 content\n", } r = _post(upload_client, "homelab.tar.gz", _tarball(tmp_path / "v1.tar.gz", v1).read_bytes()) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "success", status assert status["detail"]["added"] == 3 row_before = _row(db, str(uploads / "homelab")) assert row_before is not None added_at_before = row_before.added_at # v2: alpha changed, bravo dropped, delta new — the in-place-replace # scenario, now observed via the SECOND run's status. v2 = {"alpha.md": "# Alpha\nv2 CHANGED content\n", "delta.md": "# Delta\nbrand new\n"} r = _post(upload_client, "homelab.tar.gz", _tarball(tmp_path / "v2.tar.gz", v2).read_bytes()) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "success", status detail = status["detail"] assert detail["source"] == "homelab" assert detail["files"] == 2 assert detail["added"] == 1 # delta.md assert detail["updated"] == 1 # alpha.md (hash changed) assert detail["unchanged"] == 0 assert detail["pruned"] == 2 # bravo.md + charlie.md left the folder → pruned assert detail["overview"] is True # Exactly ONE row for the path, and its added_at survived (the # second run's upsert left the existing row alone). assert _count_rows(db) == 1 row_after = _row(db, str(uploads / "homelab")) assert row_after is not None assert row_after.id == row_before.id assert row_after.added_at == added_at_before # The folder holds ONLY the new archive's files (full replacement — # no stale files from v1), and the KB mirrors it. folder = uploads / "homelab" assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == v2 assert [p.name for p in uploads.iterdir()] == ["homelab"] assert _docs(upload_client) == [("homelab", "alpha.md"), ("homelab", "delta.md")] def test_non_a9_archive_is_a_valid_replacement( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: """An archive with only non-A9 files is a valid replacement: the swap happens, the scan indexes nothing, prune removes the source's docs, and the overview is NOT refreshed (no KB change).""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) r = _post( upload_client, "notes.tar.gz", _tarball(tmp_path / "a.tar.gz", {"readme.md": "# Readme\nv1\n"}).read_bytes(), ) assert r.status_code == 202, r.text assert _wait_status(upload_client)["state"] == "success" assert _docs(upload_client) == [("notes", "readme.md")] r = _post( upload_client, "notes.tar.gz", _tarball(tmp_path / "b.tar.gz", {"binary.bin": "not an importable format"}).read_bytes(), ) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "success", status detail = status["detail"] assert detail["files"] == 0 # nothing matches the A9 filter assert detail["added"] == 0 assert detail["updated"] == 0 assert detail["pruned"] == 1 # readme.md left the KB assert detail["overview"] is False # added + updated == 0 → no refresh assert {p.name for p in (uploads / "notes").iterdir()} == {"binary.bin"} assert _count_rows(db) == 1 assert _docs(upload_client) == [] # --- fail-fast models (the phase-49 503 becomes a status state, A5) ----------- def test_models_down_fails_the_run_and_leaves_folder_and_row( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: """The folder and row are committed BEFORE the model probe: a dead endpoint lands in the status as ``failed`` with the sanitized model-unavailable message (the phase-49 503 becomes a status state, A5) and the next sync/re-upload retries idempotently; the scan never ran.""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) async def dead_probe(llm: object) -> None: raise ModelUnavailableError( "The embedding model ('embed') is not available — check the model " "endpoint and retry. (embeddings request to " "https://user:secret@aipi.reeseapps.com/v1 failed: connection refused)" ) monkeypatch.setattr(git_sources_api, "check_models", dead_probe) r = _post( upload_client, "homelab.tar.gz", _tarball(tmp_path / "h.tar.gz", {"alpha.md": "# Alpha\nx\n"}).read_bytes(), ) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "failed", status error = status["error"] or "" assert "The embedding model ('embed') is not available" in error assert "*****@aipi.reeseapps.com" in error # the sanitizer masked the credentials assert "user:secret" not in error assert "connection refused" in error # the reason survives # The folder and row are already committed (idempotent retry path). folder = uploads / "homelab" assert folder.is_dir() row = _row(db, str(folder)) assert row is not None assert row.kind == "local" assert row.url == str(folder) # The scan never ran: no docs, no stray temps. assert _docs(upload_client) == [] assert [p.name for p in uploads.iterdir()] == ["homelab"] # --- phase 89: re-uploads honor the row's saved ignore list ------------------- def test_reupload_honors_saved_ignore_list( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: """Phase 89: a re-upload of an EXISTING source honors the ignore list saved on its row. First scan (fresh row, no list) indexes the ignored file too; once the list is saved via the API, re-uploading the same archive again scans only the kept file — the previously indexed ignored file leaves the KB (prune), and the run lands ``success`` with counts that exclude it. The upload itself keeps every file on disk: the ignore is about the index, not the folder.""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = { "keep.md": "# Keep\nin scope\n", "ignore/secret.md": "# Secret\nignored\n", } r = _post(upload_client, "docs.tar.gz", _targz_bytes(files)) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "success", status # First scan: the fresh row has no list → both files are indexed. assert status["detail"]["files"] == 2 assert _docs(upload_client) == [("docs", "ignore/secret.md"), ("docs", "keep.md")] # Save the ignore list on the row via the API (phase 89, task 03). folder = uploads / "docs" row = _row(db, str(folder)) assert row is not None r = upload_client.patch( f"/api/git-sources/{row.id}", json={"ignore_paths": ["ignore/"]} ) assert r.status_code == 200, r.text assert r.json()["ignore_paths"] == ["ignore"] # stored normalized # Re-upload the SAME archive (same name → in-place replace). r = _post(upload_client, "docs.tar.gz", _targz_bytes(files)) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "success", status detail = status["detail"] # The scan walked only keep.md (unchanged); the ignored file never # entered the walk, so prune dropped the first scan's row for it. assert detail["files"] == 1 assert detail["added"] == 0 assert detail["updated"] == 0 assert detail["unchanged"] == 1 assert detail["pruned"] == 1 assert _docs(upload_client) == [("docs", "keep.md")] # The re-upload left the row's list in place (the existing row is # untouched by the upsert), and the folder keeps every file. row_after = _row(db, str(folder)) assert row_after is not None assert row_after.ignore_paths == ["ignore"] assert {p.name for p in folder.iterdir()} == {"keep.md", "ignore"} def test_upload_new_source_without_list_imports_everything( upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path ) -> None: """Phase 89 regression: an upload of a NEW source name (no row yet, hence no ignore list) imports everything in the archive — including nested files — exactly as before phase 89.""" uploads = tmp_path / "uploads" _point_at(monkeypatch, uploads) _real_llm(monkeypatch) files = { "alpha.md": "# Alpha\nroot file\n", "sub/deep.md": "# Deep\nnested file\n", } r = _post(upload_client, "fresh.tar.gz", _targz_bytes(files)) assert r.status_code == 202, r.text status = _wait_status(upload_client) assert status["state"] == "success", status detail = status["detail"] assert detail["files"] == 2 assert detail["added"] == 2 # The fresh row carries the server-default empty list: nothing was # ignored. row = _row(db, str(uploads / "fresh")) assert row is not None assert (row.ignore_paths or []) == [] assert _docs(upload_client) == [("fresh", "alpha.md"), ("fresh", "sub/deep.md")]