brain-of-reese/tests/integration/test_git_sources_upload.py

"""Integration: the admin archive-upload API (phase 49, task 02).

Real Postgres (``podman compose up -d db``); the upload dir is pointed
at a fresh tmp dir per test by monkeypatching the router's
``get_settings`` (the ``test_git_sources_api.py`` pattern — the dev
``.env`` never leaks in), and the scan uses the deterministic
in-process ``FakeEmbedder`` (``test_sync_api.py``'s ``_real_llm``
pattern — real import, no network).

Contract under test:

* anonymous → 403 ``{"detail": "admin only"}`` (the router's
  ``require_admin`` covers the new route);
* name/format gate → 422: a non-archive extension names the accepted
  formats; a ``..`` / separator / empty-stem name (including a bare
  ``tar.gz``) is rejected with the task-01 message — and the upload
  dir is never created for a rejected name (control characters never
  reach the app: the multipart transport percent-encodes them — the
  task-01 branch for them is covered in ``test_archive_upload.py``);
* one upload at a time → 409 ``an upload is already in progress``
  (while a run is in flight — the first request holds the flag through
  its scan — and while the module-level flag seam is held);
* streaming cap → 413 naming the ``upload_max_mb`` cap; the temp
  ``.upload`` file is removed (no stray ``.`` files in the upload dir);
* unpack safety → 422 (zip-slip member, tar symlink escape, corrupt
  archive, zero-entry archive) — and a failed upload **never** touches
  the previous folder, row, or KB of an earlier good upload (the
  no-partial-state locked decision, asserted explicitly);
* happy path → 200 with the sync-detail count keys (``source`` +
  ``files/added/updated/unchanged/pruned/errors/chunks/overview``), a
  ``kind=local`` row under the tmp ``upload_dir`` (the NOT-NULL
  ``url`` column carries the path — the phase-38 convention), the
  unpacked folder, the KB via ``GET /api/docs``, and the per-upload
  log line (PLAN §9 / AGENTS.md rule 10);
* re-upload, same name → the swap replaces the folder in place, the
  row is NOT duplicated (``added_at`` preserved), dropped files are
  pruned from the KB, added/changed files are indexed;
* an archive with only non-A9 files is a VALID replacement (indexes
  nothing, prunes the previous docs, no overview refresh);
* dead models → 503 with the sanitized model-unavailable message; the
  folder and row are already committed (the next sync/re-upload
  retries idempotently).

``git_sources`` / ``documents`` / ``chunks`` / ``kb_overview`` are
global state: truncated around every test.
"""
from __future__ import annotations

import asyncio
import io
import logging
import os
import re
import tarfile
import threading
import zipfile
from collections.abc import Iterator
from pathlib import Path
from typing import Literal

import pytest
from fastapi.testclient import TestClient
from sqlalchemy import select, text
from sqlalchemy.orm import Session

from app.api import git_sources as git_sources_api
from app.config import Settings
from app.db import SessionLocal
from app.main import app as fastapi_app
from app.models import GitSource
from app.rag.archive_upload import ArchiveUploadError
from app.rag.importer import ImportSummary
from app.rag.llm import ModelUnavailableError
from tests.conftest import ADMIN_PASSWORD
from tests.fakes import FakeEmbedder


@pytest.fixture(autouse=True)
def clean_git_sources(db: Session) -> Iterator[None]:
    """The stored list is global state: reset around every test."""
    db.execute(text("TRUNCATE git_sources"))
    db.commit()
    yield
    db.execute(text("TRUNCATE git_sources"))
    db.commit()


@pytest.fixture(autouse=True)
def clean_documents(db: Session) -> Iterator[None]:
    """The happy path writes ``documents``/``chunks`` and (via the
    change-gated overview refresh) ``kb_overview`` — global, truncated
    around every test."""
    db.execute(text("TRUNCATE chunks, documents, kb_overview"))
    db.commit()
    yield
    db.execute(text("TRUNCATE chunks, documents, kb_overview"))
    db.commit()


@pytest.fixture()
def upload_client() -> Iterator[TestClient]:
    """Admin-signed client (the context-manager form keeps one event
    loop across requests — the in-flight 409 test needs the first
    request's run to survive while the second lands)."""
    with TestClient(fastapi_app) as client:
        r = client.post("/api/login", json={"password": ADMIN_PASSWORD})
        assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}"
        yield client


def _point_at(monkeypatch: pytest.MonkeyPatch, upload_dir: Path, upload_max_mb: int = 512) -> None:
    """Fresh settings on the router's module: the tmp upload dir and
    (optionally) a shrunk cap — the dev ``.env`` never leaks in."""
    monkeypatch.setattr(
        git_sources_api,
        "get_settings",
        lambda: Settings(
            _env_file=None,  # pyright: ignore[reportCallIssue]
            upload_dir=str(upload_dir),
            upload_max_mb=upload_max_mb,
        ),
    )


def _real_llm(monkeypatch: pytest.MonkeyPatch) -> None:
    """The pipeline's ``LLMClient`` becomes the deterministic
    in-process ``FakeEmbedder`` (real import, no network); it also
    implements ``embed_one``/``chat``, so the phase-41 probe passes."""
    monkeypatch.setattr(git_sources_api, "LLMClient", lambda: FakeEmbedder())


#: tarfile write modes used by the tests (uncompressed + gzip).
_TAR_WRITE_MODES = Literal["w", "w:gz"]


def _tarball(path: Path, files: dict[str, str], compress: _TAR_WRITE_MODES = "w:gz") -> Path:
    with tarfile.open(path, compress) as tf:
        for rel, content in files.items():
            data = content.encode("utf-8")
            info = tarfile.TarInfo(rel)
            info.size = len(data)
            info.mtime = 0
            tf.addfile(info, io.BytesIO(data))
    return path


def _zip(path: Path, files: dict[str, str]) -> Path:
    with zipfile.ZipFile(path, "w") as zf:
        for rel, content in files.items():
            zf.writestr(rel, content)
    return path


def _targz_bytes(files: dict[str, str]) -> bytes:
    """A tarball in memory (no temp file on disk)."""
    buf = io.BytesIO()
    with tarfile.open(fileobj=buf, mode="w:gz") as tf:
        for rel, content in files.items():
            data = content.encode("utf-8")
            info = tarfile.TarInfo(rel)
            info.size = len(data)
            info.mtime = 0
            tf.addfile(info, io.BytesIO(data))
    return buf.getvalue()


def _post(client: TestClient, filename: str, payload: bytes):
    """One upload request (multipart, the page's exact shape)."""
    return client.post(
        "/api/git-sources/upload",
        files={"file": (filename, payload, "application/octet-stream")},
    )


def _row(db: Session, path: str) -> GitSource | None:
    db.expire_all()
    return db.scalar(select(GitSource).where(GitSource.path == path))


def _docs(client: TestClient) -> list[tuple[str, str]]:
    body = client.get("/api/docs").json()
    return [(d["source"], d["path"]) for d in body["documents"]]


def _count_rows(db: Session) -> int:
    return db.execute(text("SELECT count(*) FROM git_sources")).scalar_one()


def _good_upload(
    client: TestClient, name: str = "safe", files: dict[str, str] | None = None
) -> None:
    """A 200 upload of a two-sentinel tarball — the baseline state the
    no-partial-state tests protect."""
    payload_files = files or {
        "alpha.md": "# Alpha\noriginal sentinel one\n",
        "bravo.md": "# Bravo\noriginal sentinel two\n",
    }
    r = _post(client, f"{name}.tar.gz", _targz_bytes(payload_files))
    assert r.status_code == 200, r.text


def _assert_previous_intact(
    client: TestClient, db: Session, upload_dir: Path, name: str, files: dict[str, str]
) -> None:
    """The no-partial-state locked decision, asserted explicitly: after a
    FAILED upload the previous folder's content, the row (same
    ``added_at``), and the KB are all exactly as the good upload left
    them — and no temp file survived."""
    folder = upload_dir / name
    assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == files
    assert _row(db, str(folder)) is not None  # the row is still there
    assert _count_rows(db) == 1  # …and no second row appeared
    assert _docs(client) == [(name, "alpha.md"), (name, "bravo.md")]
    assert [p.name for p in upload_dir.iterdir()] == [name]  # no stray temp


# --- anonymous -------------------------------------------------------------


def test_anonymous_upload_gets_403(client: TestClient, db: Session) -> None:
    r = _post(client, "homelab.tar.gz", b"not an archive at all")
    assert r.status_code == 403
    assert r.json() == {"detail": "admin only"}
    assert _count_rows(db) == 0


# --- name / format gate -----------------------------------------------------


def test_non_archive_extension_gets_422_naming_formats(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)

    r = _post(upload_client, "notes.txt", b"hello world")
    assert r.status_code == 422
    assert r.json()["detail"] == "only .tar, .tar.gz, .tgz or .zip archives are accepted"
    assert _count_rows(db) == 0
    # The name gate runs before the dir is created — nothing on disk.
    assert not uploads.exists()


@pytest.mark.parametrize(
    "filename",
    [
        "../evil.zip",  # ``..`` + separator
        "sub/dir.tar.gz",  # separator
        "tar.gz",  # bare suffix → empty stem
        ".tar.gz",  # empty stem
        # (control characters in the filename are percent-encoded by the
        # multipart transport before the app ever sees them — the
        # ``archive_source_name`` branch for them is covered in
        # tests/unit/test_archive_upload.py)
    ],
)
def test_unsafe_name_gets_422(
    upload_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    tmp_path: Path,
    filename: str,
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)

    r = _post(upload_client, filename, b"content")
    assert r.status_code == 422, f"{filename!r} must be rejected: {r.text}"
    assert r.json()["detail"] != "only .tar, .tar.gz, .tgz or .zip archives are accepted"
    assert _count_rows(db) == 0
    assert not uploads.exists()


# --- one at a time (409) ----------------------------------------------------


def test_second_upload_while_one_is_in_flight_returns_409(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """The flag is held from the name gate through the scan response:
    while the first run is inside its (blocked) scan, the second upload
    is 409 — and after the first finishes, uploads are accepted again."""
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"})
    started = threading.Event()
    release = threading.Event()

    class BlockingImport:
        async def __call__(self, sources, llm, **kwargs) -> ImportSummary:
            started.set()  # the run is in flight (the flag was set earlier)
            await asyncio.to_thread(release.wait, 15.0)
            return ImportSummary(files=1, unchanged=1)

    monkeypatch.setattr(git_sources_api, "import_sources", BlockingImport())

    first = {}

    def do_first() -> None:
        # A separate client/cookie jar: the two requests run on separate
        # TestClient portals; only the module-level flag is shared.
        with TestClient(fastapi_app) as c:
            assert c.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
            first["r"] = _post(c, "homelab.tar.gz", archive.read_bytes())

    thread = threading.Thread(target=do_first)
    thread.start()
    try:
        assert started.wait(15.0), "first upload did not reach its scan"
        with TestClient(fastapi_app) as second:
            assert second.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
            r = _post(second, "homelab.tar.gz", archive.read_bytes())
        assert r.status_code == 409
        assert r.json() == {"detail": "an upload is already in progress"}
    finally:
        release.set()
        thread.join(20)

    assert first["r"].status_code == 200, first["r"].text
    # The flag was released: the next upload goes through for real.
    with TestClient(fastapi_app) as third:
        assert third.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
        r = _post(third, "homelab.tar.gz", archive.read_bytes())
    assert r.status_code == 200, r.text


def test_upload_refused_while_flag_held(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    """The smallest test seam: the module-level flag itself. Held →
    409, nothing happens (no dir, no row)."""
    _point_at(monkeypatch, tmp_path / "uploads")
    monkeypatch.setattr(git_sources_api, "_upload_in_progress", True)

    r = _post(upload_client, "a.tar.gz", b"x")
    assert r.status_code == 409
    assert r.json() == {"detail": "an upload is already in progress"}
    assert _count_rows(db) == 0
    assert not (tmp_path / "uploads").exists()


# --- in-flight upload must not hold the request session (regression, -----
# --- phase 49 task 03) ----------------------------------------------------


def test_in_flight_upload_does_not_block_a_concurrent_truncate(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    """While the scan is in flight, a concurrent TRUNCATE of the KB +
    registry tables (the E2E isolation-fixture pattern) must complete.

    Regression for the phase-49 task-03 deadlock: the handler used to
    keep its request ``db`` session open across the scan, and the
    uncommitted ``_commit_new`` refresh transaction held
    ``git_sources`` locks for the whole scan. A concurrent TRUNCATE
    (documents locked, git_sources pending) then deadlocked with the
    scan's own document locks — a cycle spanning three connections
    that Postgres's detector cannot see, hanging the app and the test
    run forever. The handler now releases the session before the scan;
    this pins it: with the scan held in flight, the TRUNCATE completes
    well inside its 5 s ``lock_timeout`` (without the fix it times out
    with a lock-not-available error)."""
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"})
    started = threading.Event()
    release = threading.Event()

    class BlockingImport:
        async def __call__(self, sources, llm, **kwargs) -> ImportSummary:
            started.set()  # the run is in flight (the flag was set earlier)
            await asyncio.to_thread(release.wait, 15.0)
            return ImportSummary(files=1, unchanged=1)

    monkeypatch.setattr(git_sources_api, "import_sources", BlockingImport())

    first = {}

    def do_first() -> None:
        with TestClient(fastapi_app) as c:
            assert c.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
            first["r"] = _post(c, "homelab.tar.gz", archive.read_bytes())

    thread = threading.Thread(target=do_first)
    thread.start()
    try:
        assert started.wait(15.0), "first upload did not reach its scan"
        # The E2E isolation TRUNCATE, exactly as the story fixtures run
        # it — must complete while the scan is held in flight.
        with SessionLocal() as tr, tr.begin():
            tr.execute(text("SET LOCAL lock_timeout = '5s'"))
            tr.execute(
                text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources")
            )
    finally:
        release.set()
        thread.join(20)

    # The scan finished once released — the 200 stands (the TRUNCATE ran
    # after the row upsert and dropped it; the next re-upload is
    # idempotent, and the autouse fixtures reset the tables anyway).
    assert first["r"].status_code == 200, first["r"].text


# --- streaming cap (413) ----------------------------------------------------


def test_oversized_upload_gets_413_and_leaves_no_temp(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads, upload_max_mb=1)

    # 2 MiB of incompressible bytes with a 1 MiB cap → 413 naming the cap.
    big = os.urandom(2 * 1024 * 1024)
    r = _post(upload_client, "big.zip", big)
    assert r.status_code == 413
    assert "1 MiB" in r.json()["detail"]
    assert _count_rows(db) == 0
    assert uploads.exists()  # the dir was created before streaming
    assert list(uploads.iterdir()) == []  # the temp .upload file is gone

    # Boundary: EXACTLY the cap is not 413 (the check is strictly >) —
    # the stream completes and the garbage bytes fail at unpack instead.
    r = _post(upload_client, "exact.zip", os.urandom(1024 * 1024))
    assert r.status_code == 422
    assert "could not unpack the archive" in r.json()["detail"]
    assert list(uploads.iterdir()) == []


# --- unpack safety: failed uploads leave the previous state intact ----------


def test_zip_slip_archive_gets_422_and_previous_state_is_intact(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    files = {
        "alpha.md": "# Alpha\noriginal sentinel one\n",
        "bravo.md": "# Bravo\noriginal sentinel two\n",
    }
    _good_upload(upload_client, "safe", files)

    evil = _zip(tmp_path / "safe.zip", {"../evil.txt": "pwned"})
    r = _post(upload_client, "safe.zip", evil.read_bytes())
    assert r.status_code == 422
    assert "traversal" in r.json()["detail"]
    assert not (tmp_path / "evil.txt").exists()  # the escape never landed

    _assert_previous_intact(upload_client, db, uploads, "safe", files)


def test_tar_symlink_escape_gets_422_and_previous_state_is_intact(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    files = {
        "alpha.md": "# Alpha\noriginal sentinel one\n",
        "bravo.md": "# Bravo\noriginal sentinel two\n",
    }
    _good_upload(upload_client, "safe", files)

    evil = tmp_path / "safe.tar"
    with tarfile.open(evil, "w") as tf:
        info = tarfile.TarInfo("link")
        info.type = tarfile.SYMTYPE
        info.linkname = "/etc/passwd"
        tf.addfile(info)
    r = _post(upload_client, "safe.tar", evil.read_bytes())
    assert r.status_code == 422
    assert "escape" in r.json()["detail"]

    _assert_previous_intact(upload_client, db, uploads, "safe", files)


def test_corrupt_archive_gets_422_and_previous_state_is_intact(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    files = {
        "alpha.md": "# Alpha\noriginal sentinel one\n",
        "bravo.md": "# Bravo\noriginal sentinel two\n",
    }
    _good_upload(upload_client, "safe", files)

    # A truncated zip (the EOCD is cut off) is not a zip and not a tar.
    good_zip = _zip(tmp_path / "good.zip", {"alpha.md": "# Alpha\nx\n"})
    good_bytes = good_zip.read_bytes()
    truncated = good_bytes[: len(good_bytes) // 2]
    r = _post(upload_client, "safe.zip", truncated)
    assert r.status_code == 422
    assert "could not unpack the archive" in r.json()["detail"]

    # A zero-byte "archive" fails the same way.
    r = _post(upload_client, "safe.tar", b"")
    assert r.status_code == 422
    assert "could not unpack the archive" in r.json()["detail"]

    _assert_previous_intact(upload_client, db, uploads, "safe", files)


def test_swap_failure_gets_422_and_previous_state_is_intact(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    """A rename failure in ``swap_in`` (OS error) → 422 with its
    message; the previous folder/row/KB are untouched and no temp
    survives (the handler's finally cleans the temp sibling)."""
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    files = {
        "alpha.md": "# Alpha\noriginal sentinel one\n",
        "bravo.md": "# Bravo\noriginal sentinel two\n",
    }
    _good_upload(upload_client, "safe", files)

    def failing_swap(new_dir: Path, final_dir: Path) -> None:
        raise ArchiveUploadError("could not replace the previous folder")

    monkeypatch.setattr(git_sources_api, "swap_in", failing_swap)
    r = _post(upload_client, "safe.tar.gz", _targz_bytes({"alpha.md": "# A\nx\n"}))
    assert r.status_code == 422
    assert r.json()["detail"] == "could not replace the previous folder"

    _assert_previous_intact(upload_client, db, uploads, "safe", files)


def test_zero_entry_archive_gets_422(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    """A completely empty archive (zero entries) is 422 — both
    containers — with no folder, row, or temp file left behind."""
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)

    empty_targz = tmp_path / "empty.tar.gz"
    with tarfile.open(empty_targz, "w:gz"):
        pass
    r = _post(upload_client, "empty.tar.gz", empty_targz.read_bytes())
    assert r.status_code == 422
    assert r.json()["detail"] == "the archive contains no files"

    empty_zip = tmp_path / "empty.zip"
    with zipfile.ZipFile(empty_zip, "w"):
        pass
    r = _post(upload_client, "empty.zip", empty_zip.read_bytes())
    assert r.status_code == 422
    assert r.json()["detail"] == "the archive contains no files"

    assert _count_rows(db) == 0
    assert not (uploads / "empty").exists()
    assert list(uploads.iterdir()) == []


# --- happy path -------------------------------------------------------------


def test_happy_path_tar_gz(
    upload_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    files = {"alpha.md": "# Alpha\nfirst sentinel\n", "bravo.md": "# Bravo\nsecond sentinel\n"}
    payload = _tarball(tmp_path / "homelab.tar.gz", files).read_bytes()

    with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
        r = _post(upload_client, "homelab.tar.gz", payload)
    assert r.status_code == 200, r.text
    body = r.json()
    assert set(body) == {
        "source", "files", "added", "updated", "unchanged", "pruned", "errors", "chunks",
        "overview",
    }
    assert body["source"] == "homelab"  # filename minus the archive suffix
    assert body["files"] == 2
    assert body["added"] == 2
    assert body["updated"] == 0
    assert body["unchanged"] == 0
    assert body["pruned"] == 0
    assert body["errors"] == 0
    assert body["chunks"] >= 2
    assert body["overview"] is True  # the KB changed → the overview refreshed

    # Unpacked under the tmp upload dir, dotfile temps cleaned up.
    folder = uploads / "homelab"
    assert folder.is_dir()
    assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == files
    assert [p.name for p in uploads.iterdir()] == ["homelab"]

    # One kind=local row; the NOT-NULL url column carries the path.
    row = _row(db, str(folder))
    assert row is not None
    assert row.kind == "local"
    assert row.url == str(folder)
    assert row.path == str(folder)
    assert _count_rows(db) == 1

    # The KB lists both files under the source name.
    assert _docs(upload_client) == [("homelab", "alpha.md"), ("homelab", "bravo.md")]

    # The per-upload log line (PLAN §9 / AGENTS.md rule 10).
    lines = [rec.getMessage() for rec in caplog.records if rec.getMessage().startswith("upload: ")]
    assert len(lines) == 1, lines
    match = re.match(
        r"^upload: name=homelab file=homelab\.tar\.gz bytes_in=(\d+) files=2 added=2 "
        r"updated=0 unchanged=0 pruned=0 errors=0 overview=True total_ms=\d+$",
        lines[0],
    )
    assert match, lines[0]
    assert int(match.group(1)) == len(payload)  # the compressed bytes in


@pytest.mark.parametrize(
    ("filename", "builder"),
    [
        ("notes.zip", "zip"),
        ("plain.tar", "tar"),
        ("tgz.tgz", "targz"),
    ],
)
def test_happy_path_other_accepted_formats(
    upload_client: TestClient,
    monkeypatch: pytest.MonkeyPatch,
    db: Session,
    tmp_path: Path,
    filename: str,
    builder: str,
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    files = {"alpha.md": "# Alpha\nsentinel\n"}
    make = {"zip": _zip, "tar": lambda p, f: _tarball(p, f, "w"), "targz": _tarball}[builder]
    payload = make(tmp_path / filename, files).read_bytes()

    r = _post(upload_client, filename, payload)
    assert r.status_code == 200, r.text
    body = r.json()
    expected_name = {"notes.zip": "notes", "plain.tar": "plain", "tgz.tgz": "tgz"}[filename]
    assert body["source"] == expected_name
    assert body["added"] == 1
    assert _docs(upload_client) == [(expected_name, "alpha.md")]
    assert _count_rows(db) == 1


# --- re-upload, same name: in-place replace ---------------------------------


def test_reupload_same_name_replaces_in_place(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)
    v1 = {
        "alpha.md": "# Alpha\nv1 content\n",
        "bravo.md": "# Bravo\nv1 content\n",
        "charlie.md": "# Charlie\nv1 content\n",
    }
    r = _post(upload_client, "homelab.tar.gz", _tarball(tmp_path / "v1.tar.gz", v1).read_bytes())
    assert r.status_code == 200, r.text
    assert r.json()["added"] == 3
    row_before = _row(db, str(uploads / "homelab"))
    assert row_before is not None
    added_at_before = row_before.added_at

    # v2: alpha changed, bravo dropped, delta new.
    v2 = {"alpha.md": "# Alpha\nv2 CHANGED content\n", "delta.md": "# Delta\nbrand new\n"}
    r = _post(upload_client, "homelab.tar.gz", _tarball(tmp_path / "v2.tar.gz", v2).read_bytes())
    assert r.status_code == 200, r.text
    body = r.json()
    assert body["source"] == "homelab"
    assert body["files"] == 2
    assert body["added"] == 1  # delta.md
    assert body["updated"] == 1  # alpha.md (hash changed)
    assert body["unchanged"] == 0
    assert body["pruned"] == 2  # bravo.md + charlie.md left the folder → pruned
    assert body["overview"] is True

    # Exactly ONE row for the path, and its added_at survived (the
    # upsert left the existing row alone).
    assert _count_rows(db) == 1
    row_after = _row(db, str(uploads / "homelab"))
    assert row_after is not None
    assert row_after.id == row_before.id
    assert row_after.added_at == added_at_before

    # The folder holds ONLY the new archive's files (full replacement —
    # no stale files from v1), and the KB mirrors it.
    folder = uploads / "homelab"
    assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == v2
    assert [p.name for p in uploads.iterdir()] == ["homelab"]
    assert _docs(upload_client) == [("homelab", "alpha.md"), ("homelab", "delta.md")]


def test_non_a9_archive_is_a_valid_replacement(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    """An archive with only non-A9 files is a valid replacement: the
    swap happens, the scan indexes nothing, prune removes the source's
    docs, and the overview is NOT refreshed (no KB change)."""
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)

    r = _post(
        upload_client,
        "notes.tar.gz",
        _tarball(tmp_path / "a.tar.gz", {"readme.md": "# Readme\nv1\n"}).read_bytes(),
    )
    assert r.status_code == 200, r.text
    assert _docs(upload_client) == [("notes", "readme.md")]

    r = _post(
        upload_client,
        "notes.tar.gz",
        _tarball(tmp_path / "b.tar.gz", {"binary.bin": "not an importable format"}).read_bytes(),
    )
    assert r.status_code == 200, r.text
    body = r.json()
    assert body["files"] == 0  # nothing matches the A9 filter
    assert body["added"] == 0
    assert body["updated"] == 0
    assert body["pruned"] == 1  # readme.md left the KB
    assert body["overview"] is False  # added + updated == 0 → no refresh

    assert {p.name for p in (uploads / "notes").iterdir()} == {"binary.bin"}
    assert _count_rows(db) == 1
    assert _docs(upload_client) == []


# --- fail-fast models (503) ---------------------------------------------------


def test_models_down_gets_503_and_leaves_folder_and_row(
    upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
    """The folder and row are committed BEFORE the model probe: a dead
    endpoint answers 503 (sanitized — credentials masked) and the next
    sync/re-upload retries idempotently; the scan never ran."""
    uploads = tmp_path / "uploads"
    _point_at(monkeypatch, uploads)
    _real_llm(monkeypatch)

    async def dead_probe(llm: object) -> None:
        raise ModelUnavailableError(
            "The embedding model ('embed') is not available — check the model "
            "endpoint and retry. (embeddings request to "
            "https://user:secret@aipi.reeseapps.com/v1 failed: connection refused)"
        )

    monkeypatch.setattr(git_sources_api, "check_models", dead_probe)
    r = _post(
        upload_client,
        "homelab.tar.gz",
        _tarball(tmp_path / "h.tar.gz", {"alpha.md": "# Alpha\nx\n"}).read_bytes(),
    )
    assert r.status_code == 503
    detail = r.json()["detail"]
    assert "The embedding model ('embed') is not available" in detail
    assert "*****@aipi.reeseapps.com" in detail  # the sanitizer masked the credentials
    assert "user:secret" not in detail
    assert "connection refused" in detail  # the reason survives

    # The folder and row are already committed (idempotent retry path).
    folder = uploads / "homelab"
    assert folder.is_dir()
    row = _row(db, str(folder))
    assert row is not None
    assert row.kind == "local"
    assert row.url == str(folder)
    # The scan never ran: no docs, no stray temps.
    assert _docs(upload_client) == []
    assert [p.name for p in uploads.iterdir()] == ["homelab"]