1355 lines
53 KiB
Python
1355 lines
53 KiB
Python
"""Integration: the admin archive-upload API (phase 49, task 02;
|
|
backgrounded in phase 64, task 03).
|
|
|
|
Real Postgres (``podman compose up -d db``); the upload dir is pointed
|
|
at a fresh tmp dir per test by monkeypatching the router's
|
|
``get_settings`` (the ``test_git_sources_api.py`` pattern — the dev
|
|
``.env`` never leaks in), and the scan uses the deterministic
|
|
in-process ``FakeEmbedder`` (``test_sync_api.py``'s ``_real_llm``
|
|
pattern — real import, no network).
|
|
|
|
Phase 64 (task 03) contract under test: ``POST`` answers **202** the
|
|
moment the archive is safely on disk (an ``UploadAccepted`` body naming
|
|
the safe source), and the rest — unpack → swap → row upsert → model
|
|
check → scan → change-gated overview — runs in a **background task**
|
|
behind ``GET /api/git-sources/upload/status`` (the phase-32
|
|
``SyncStatus`` shape, incl. the phase-64 ``current_file`` /
|
|
``files_done`` / ``files_total``). Every post-receive scenario observes
|
|
the **status endpoint, polled until terminal** — never the HTTP
|
|
response; post-202 failures are status states (``failed`` + sanitized
|
|
error, A5), never HTTP errors. Navigating away mid-scan no longer
|
|
aborts anything: the task is created before the 202 and its outcome
|
|
lands in the module-level status.
|
|
|
|
The inline gates are unchanged:
|
|
|
|
* anonymous → 403 ``{"detail": "admin only"}`` (the router's
|
|
``require_admin`` covers the new status route too);
|
|
* name/format gate → 422: a non-archive extension names the accepted
|
|
formats; a ``..`` / separator / empty-stem name (including a bare
|
|
``tar.gz``) is rejected with the task-01 message — and the upload
|
|
dir is never created for a rejected name (control characters never
|
|
reach the app: the multipart transport percent-encodes them — the
|
|
task-01 branch for them is covered in ``test_archive_upload.py``);
|
|
* one upload at a time → 409 ``an upload is already in progress`` —
|
|
the gate is the module-level **flag** (checked and set with no await
|
|
in between, BEFORE the receive — the handler now awaits before the
|
|
background task exists), held through the whole background run;
|
|
* streaming cap → 413 naming the ``upload_max_mb`` cap; the temp
|
|
``.upload`` file is removed and the flag is released (no stray
|
|
``.`` files in the upload dir; the next upload is not 409).
|
|
|
|
Background branches (A5): unpack failure (zip-slip, tar symlink
|
|
escape, corrupt archive), zero-entry archive, swap failure, the
|
|
concurrent-insert ``IntegrityError`` backstop, dead models, a
|
|
cancelled run (the ``finally`` cleans both temps + releases the flag),
|
|
and the success path — each lands in the status as ``failed`` (with
|
|
the sanitized error) or ``success`` (with the ``UploadOut`` fields in
|
|
``detail``); a FAILED upload **never** touches the previous folder,
|
|
row, or KB of an earlier good upload (the no-partial-state locked
|
|
decision, asserted explicitly).
|
|
|
|
``git_sources`` / ``documents`` / ``chunks`` / ``kb_overview`` are
|
|
global state: truncated around every test.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import io
|
|
import logging
|
|
import os
|
|
import re
|
|
import tarfile
|
|
import threading
|
|
import time
|
|
import zipfile
|
|
from collections.abc import Callable, Iterator
|
|
from pathlib import Path
|
|
from typing import Any, Literal
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import select, text
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.api import git_sources as git_sources_api
|
|
from app.config import Settings
|
|
from app.db import SessionLocal
|
|
from app.main import app as fastapi_app
|
|
from app.models import GitSource
|
|
from app.rag.archive_upload import ArchiveUploadError, unpack_archive
|
|
from app.rag.importer import ImportSummary
|
|
from app.rag.llm import EmbeddingError, ModelUnavailableError
|
|
from tests.conftest import ADMIN_PASSWORD
|
|
from tests.fakes import FakeEmbedder
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clean_git_sources(db: Session) -> Iterator[None]:
|
|
"""The stored list is global state: reset around every test."""
|
|
db.execute(text("TRUNCATE git_sources"))
|
|
db.commit()
|
|
yield
|
|
db.execute(text("TRUNCATE git_sources"))
|
|
db.commit()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clean_documents(db: Session) -> Iterator[None]:
|
|
"""The happy path writes ``documents``/``chunks`` and (via the
|
|
change-gated overview refresh) ``kb_overview`` — global, truncated
|
|
around every test."""
|
|
db.execute(text("TRUNCATE chunks, documents, kb_overview"))
|
|
db.commit()
|
|
yield
|
|
db.execute(text("TRUNCATE chunks, documents, kb_overview"))
|
|
db.commit()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def fresh_upload_status() -> Iterator[None]:
|
|
"""The module-level upload status + flag are process-global: reset
|
|
them before AND after every test (the ``fresh_sync_status`` pattern
|
|
from ``tests/unit/test_sync_button.py``) — a previous test's
|
|
terminal state must never leak into the next one's idle-shape
|
|
assertions or its 409 expectations."""
|
|
git_sources_api._upload_status = git_sources_api.UploadStatus()
|
|
git_sources_api._upload_in_progress = False
|
|
yield
|
|
git_sources_api._upload_status = git_sources_api.UploadStatus()
|
|
git_sources_api._upload_in_progress = False
|
|
|
|
|
|
@pytest.fixture()
|
|
def upload_client() -> Iterator[TestClient]:
|
|
"""Admin-signed client (the context-manager form keeps one event
|
|
loop for the whole test — the background task the 202 creates runs
|
|
on it, so the client must stay open until the run is observed in a
|
|
terminal state)."""
|
|
with TestClient(fastapi_app) as client:
|
|
r = client.post("/api/login", json={"password": ADMIN_PASSWORD})
|
|
assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}"
|
|
yield client
|
|
|
|
|
|
def _point_at(monkeypatch: pytest.MonkeyPatch, upload_dir: Path, upload_max_mb: int = 512) -> None:
|
|
"""Fresh settings on the router's module: the tmp upload dir and
|
|
(optionally) a shrunk cap — the dev ``.env`` never leaks in."""
|
|
monkeypatch.setattr(
|
|
git_sources_api,
|
|
"get_settings",
|
|
lambda: Settings(
|
|
_env_file=None, # pyright: ignore[reportCallIssue]
|
|
upload_dir=str(upload_dir),
|
|
upload_max_mb=upload_max_mb,
|
|
),
|
|
)
|
|
|
|
|
|
def _real_llm(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The pipeline's ``LLMClient`` becomes the deterministic
|
|
in-process ``FakeEmbedder`` (real import, no network); it also
|
|
implements ``embed_one``/``chat``, so the phase-41 probe passes."""
|
|
monkeypatch.setattr(git_sources_api, "LLMClient", lambda: FakeEmbedder())
|
|
|
|
|
|
#: tarfile write modes used by the tests (uncompressed + gzip).
|
|
_TAR_WRITE_MODES = Literal["w", "w:gz"]
|
|
|
|
|
|
def _tarball(path: Path, files: dict[str, str], compress: _TAR_WRITE_MODES = "w:gz") -> Path:
|
|
with tarfile.open(path, compress) as tf:
|
|
for rel, content in files.items():
|
|
data = content.encode("utf-8")
|
|
info = tarfile.TarInfo(rel)
|
|
info.size = len(data)
|
|
info.mtime = 0
|
|
tf.addfile(info, io.BytesIO(data))
|
|
return path
|
|
|
|
|
|
def _zip(path: Path, files: dict[str, str]) -> Path:
|
|
with zipfile.ZipFile(path, "w") as zf:
|
|
for rel, content in files.items():
|
|
zf.writestr(rel, content)
|
|
return path
|
|
|
|
|
|
def _targz_bytes(files: dict[str, str]) -> bytes:
|
|
"""A tarball in memory (no temp file on disk)."""
|
|
buf = io.BytesIO()
|
|
with tarfile.open(fileobj=buf, mode="w:gz") as tf:
|
|
for rel, content in files.items():
|
|
data = content.encode("utf-8")
|
|
info = tarfile.TarInfo(rel)
|
|
info.size = len(data)
|
|
info.mtime = 0
|
|
tf.addfile(info, io.BytesIO(data))
|
|
return buf.getvalue()
|
|
|
|
|
|
def _post(client: TestClient, filename: str, payload: bytes):
|
|
"""One upload request (multipart, the page's exact shape)."""
|
|
return client.post(
|
|
"/api/git-sources/upload",
|
|
files={"file": (filename, payload, "application/octet-stream")},
|
|
)
|
|
|
|
|
|
def _wait_status(client: TestClient, timeout: float = 30.0) -> dict[str, Any]:
|
|
"""Poll ``GET /api/git-sources/upload/status`` until the run is
|
|
terminal (``success`` / ``failed``) and return the terminal body.
|
|
|
|
Phase 64: the 202 answers BEFORE the scan — every post-receive
|
|
assertion observes the status endpoint, not the HTTP response."""
|
|
deadline = time.monotonic() + timeout
|
|
body: dict[str, Any] = {}
|
|
while True:
|
|
r = client.get("/api/git-sources/upload/status")
|
|
assert r.status_code == 200, r.text
|
|
body = r.json()
|
|
if body["state"] in ("success", "failed"):
|
|
return body
|
|
if time.monotonic() >= deadline:
|
|
raise AssertionError(f"upload never reached a terminal state: {body}")
|
|
time.sleep(0.05)
|
|
|
|
|
|
def _row(db: Session, path: str) -> GitSource | None:
|
|
db.expire_all()
|
|
return db.scalar(select(GitSource).where(GitSource.path == path))
|
|
|
|
|
|
def _docs(client: TestClient) -> list[tuple[str, str]]:
|
|
body = client.get("/api/docs").json()
|
|
return [(d["source"], d["path"]) for d in body["documents"]]
|
|
|
|
|
|
def _count_rows(db: Session) -> int:
|
|
return db.execute(text("SELECT count(*) FROM git_sources")).scalar_one()
|
|
|
|
|
|
def _good_upload(
|
|
client: TestClient, name: str = "safe", files: dict[str, str] | None = None
|
|
) -> None:
|
|
"""A successful upload of a two-sentinel tarball (202 → status
|
|
``success``) — the baseline state the no-partial-state tests
|
|
protect."""
|
|
payload_files = files or {
|
|
"alpha.md": "# Alpha\noriginal sentinel one\n",
|
|
"bravo.md": "# Bravo\noriginal sentinel two\n",
|
|
}
|
|
r = _post(client, f"{name}.tar.gz", _targz_bytes(payload_files))
|
|
assert r.status_code == 202, r.text
|
|
assert _wait_status(client)["state"] == "success"
|
|
|
|
|
|
def _assert_previous_intact(
|
|
client: TestClient, db: Session, upload_dir: Path, name: str, files: dict[str, str]
|
|
) -> None:
|
|
"""The no-partial-state locked decision, asserted explicitly: after a
|
|
FAILED upload the previous folder's content, the row (same
|
|
``added_at``), and the KB are all exactly as the good upload left
|
|
them — and no temp file survived."""
|
|
folder = upload_dir / name
|
|
assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == files
|
|
assert _row(db, str(folder)) is not None # the row is still there
|
|
assert _count_rows(db) == 1 # …and no second row appeared
|
|
assert _docs(client) == [(name, "alpha.md"), (name, "bravo.md")]
|
|
assert [p.name for p in upload_dir.iterdir()] == [name] # no stray temp
|
|
|
|
|
|
class _GatedImport:
|
|
"""The mock import (the task-02 seam from
|
|
``tests/unit/test_sync_button.py``): fires the runner's OWN
|
|
progress hook once (the progress-shaped call goes through the real
|
|
hook closure — the closure under test), parks on a threading gate
|
|
so the test can read the status mid-run, then returns the canned
|
|
summary (or raises ``fail``)."""
|
|
|
|
def __init__(
|
|
self,
|
|
summary: ImportSummary,
|
|
started: threading.Event,
|
|
release: threading.Event,
|
|
fail: BaseException | None = None,
|
|
) -> None:
|
|
self.summary = summary
|
|
self.started = started
|
|
self.release = release
|
|
self.fail = fail
|
|
self.hook_calls: list[tuple[str, str, int, int]] = []
|
|
self.prune_flags: list[bool] = []
|
|
|
|
async def __call__(
|
|
self,
|
|
sources: list[Path],
|
|
llm: object,
|
|
*,
|
|
prune: bool = False,
|
|
limit: int | None = None,
|
|
session: object = None,
|
|
progress: Callable[[str, str, int, int], None] | None = None,
|
|
) -> ImportSummary:
|
|
self.prune_flags.append(prune)
|
|
if progress is not None:
|
|
progress("homelab", "notes/deep.md", 1, 3)
|
|
self.hook_calls.append(("homelab", "notes/deep.md", 1, 3))
|
|
self.started.set()
|
|
await asyncio.to_thread(self.release.wait, 30.0)
|
|
if self.fail is not None:
|
|
raise self.fail
|
|
return self.summary
|
|
|
|
|
|
class _GatedUnpack:
|
|
"""Parks inside ``unpack_archive`` — a SYNCHRONOUS seam (the app
|
|
loop blocks while parked — only usable when the run is driven
|
|
directly on a worker loop, never through HTTP, where the blocked
|
|
loop would also stall the 202's response delivery) — then runs the
|
|
REAL unpack so the run completes for real."""
|
|
|
|
def __init__(self, started: threading.Event, release: threading.Event) -> None:
|
|
self.started = started
|
|
self.release = release
|
|
|
|
def __call__(self, archive: Path, target: Path, max_bytes: int) -> None:
|
|
self.started.set()
|
|
self.release.wait(30.0)
|
|
unpack_archive(archive, target, max_bytes)
|
|
|
|
|
|
class _ImmediateImport:
|
|
"""A mock import that returns the canned summary immediately (no
|
|
gate — the gate for that test lives in unpack instead)."""
|
|
|
|
def __init__(self, summary: ImportSummary) -> None:
|
|
self.summary = summary
|
|
|
|
async def __call__(self, sources: list[Path], llm: object, **kwargs: object) -> ImportSummary:
|
|
return self.summary
|
|
|
|
|
|
# --- anonymous -------------------------------------------------------------
|
|
|
|
|
|
def test_anonymous_upload_and_status_get_403(client: TestClient, db: Session) -> None:
|
|
r = _post(client, "homelab.tar.gz", b"not an archive at all")
|
|
assert r.status_code == 403
|
|
assert r.json() == {"detail": "admin only"}
|
|
# The status route sits behind the same router-level dependency.
|
|
r = client.get("/api/git-sources/upload/status")
|
|
assert r.status_code == 403
|
|
assert r.json() == {"detail": "admin only"}
|
|
assert _count_rows(db) == 0
|
|
|
|
|
|
def test_idle_upload_status_pins_full_shape(upload_client: TestClient) -> None:
|
|
"""Idle: the full response dict is pinned — identical key set to
|
|
``GET /api/sync/status`` (phase 32), progress keys null/0/0."""
|
|
assert upload_client.get("/api/git-sources/upload/status").json() == {
|
|
"state": "idle",
|
|
"started_at": None,
|
|
"finished_at": None,
|
|
"detail": {},
|
|
"error": None,
|
|
"current_file": None,
|
|
"files_done": 0,
|
|
"files_total": 0,
|
|
}
|
|
|
|
|
|
# --- name / format gate -----------------------------------------------------
|
|
|
|
|
|
def test_non_archive_extension_gets_422_naming_formats(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
|
|
r = _post(upload_client, "notes.txt", b"hello world")
|
|
assert r.status_code == 422
|
|
assert r.json()["detail"] == "only .tar, .tar.gz, .tgz or .zip archives are accepted"
|
|
assert _count_rows(db) == 0
|
|
# The name gate runs before the dir is created — nothing on disk.
|
|
assert not uploads.exists()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"filename",
|
|
[
|
|
"../evil.zip", # ``..`` + separator
|
|
"sub/dir.tar.gz", # separator
|
|
"tar.gz", # bare suffix → empty stem
|
|
".tar.gz", # empty stem
|
|
# (control characters in the filename are percent-encoded by the
|
|
# multipart transport before the app ever sees them — the
|
|
# ``archive_source_name`` branch for them is covered in
|
|
# tests/unit/test_archive_upload.py)
|
|
],
|
|
)
|
|
def test_unsafe_name_gets_422(
|
|
upload_client: TestClient,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
db: Session,
|
|
tmp_path: Path,
|
|
filename: str,
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
|
|
r = _post(upload_client, filename, b"content")
|
|
assert r.status_code == 422, f"{filename!r} must be rejected: {r.text}"
|
|
assert r.json()["detail"] != "only .tar, .tar.gz, .tgz or .zip archives are accepted"
|
|
assert _count_rows(db) == 0
|
|
assert not uploads.exists()
|
|
|
|
|
|
# --- one at a time (409) ----------------------------------------------------
|
|
|
|
|
|
def test_second_upload_while_one_is_in_flight_returns_409(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""The 202 answers BEFORE the scan: while the first run is in
|
|
flight (parked inside its blocked scan), the second upload is 409
|
|
— the module-level flag, held from the first receive, is the gate —
|
|
and after the first run lands in a terminal state, uploads are
|
|
accepted again (a full real run this time)."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"})
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
fake_import = _GatedImport(ImportSummary(files=1, unchanged=1), started, release)
|
|
monkeypatch.setattr(git_sources_api, "import_sources", fake_import)
|
|
|
|
with TestClient(fastapi_app) as client:
|
|
assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
|
|
r = _post(client, "homelab.tar.gz", archive.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
assert r.json() == {"detail": "upload received", "name": "homelab"}
|
|
assert started.wait(15.0), "background run did not reach its scan"
|
|
# The run is in flight: the flag (not the task's done-ness) is
|
|
# held — the second upload is 409.
|
|
r = _post(client, "homelab.tar.gz", archive.read_bytes())
|
|
assert r.status_code == 409
|
|
assert r.json() == {"detail": "an upload is already in progress"}
|
|
release.set()
|
|
status = _wait_status(client)
|
|
assert status["state"] == "success", status
|
|
|
|
# The flag was released: the next upload goes through for real.
|
|
with TestClient(fastapi_app) as third:
|
|
assert third.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
|
|
r = _post(third, "homelab.tar.gz", archive.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
assert _wait_status(third)["state"] == "success"
|
|
|
|
|
|
def test_upload_refused_while_flag_held(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
"""The smallest test seam: the module-level flag itself. Held →
|
|
409 and nothing happens (no dir, no row) — even though the status
|
|
is still ``idle`` (no run exists at all): the flag, not a
|
|
task/state check, is the gate. That is what the receive window
|
|
needs — the handler awaits the (1 MiB-chunk) streaming receive
|
|
BEFORE the background task exists, so only the flag can stop a
|
|
concurrent POST from starting a second run."""
|
|
_point_at(monkeypatch, tmp_path / "uploads")
|
|
monkeypatch.setattr(git_sources_api, "_upload_in_progress", True)
|
|
|
|
r = _post(upload_client, "a.tar.gz", b"x")
|
|
assert r.status_code == 409
|
|
assert r.json() == {"detail": "an upload is already in progress"}
|
|
# The flag — not the status state — is the gate:
|
|
assert upload_client.get("/api/git-sources/upload/status").json()["state"] == "idle"
|
|
assert _count_rows(db) == 0
|
|
assert not (tmp_path / "uploads").exists()
|
|
|
|
|
|
# --- the 202 lands with the archive on disk (A2) -----------------------------
|
|
|
|
|
|
def test_202_answers_before_the_scan_completes(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""The 202 is the "successfully uploaded" moment (A2), not the scan
|
|
result: the response lands while the scan is still IN FLIGHT (parked
|
|
inside its gated import) — the status is ``running``, the outcome
|
|
not yet there — and only later does the run complete (the outcome
|
|
lands in the status, which the UI polls). This is the navigate-away
|
|
contract, unit-level: the background task is created before the
|
|
response, and its outcome lands in ``_upload_status`` — never in
|
|
the response."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"})
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
fake_import = _GatedImport(ImportSummary(files=1, added=1), started, release)
|
|
monkeypatch.setattr(git_sources_api, "import_sources", fake_import)
|
|
|
|
with TestClient(fastapi_app) as client:
|
|
assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
|
|
r = _post(client, "homelab.tar.gz", archive.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
assert r.json() == {"detail": "upload received", "name": "homelab"}
|
|
# The response landed BEFORE the scan: the run is in flight on
|
|
# this client's loop and has not completed (the scan is parked).
|
|
assert started.wait(15.0), "background run did not reach its scan"
|
|
s = client.get("/api/git-sources/upload/status").json()
|
|
assert s["state"] == "running"
|
|
assert s["detail"] == {} and s["error"] is None
|
|
assert s["started_at"] is not None and s["finished_at"] is None
|
|
# Mid-flight: unpack + swap already landed the folder (the
|
|
# archive was safely on disk at the 202 moment), the scan has
|
|
# not run yet.
|
|
assert (uploads / "homelab").is_dir()
|
|
release.set()
|
|
s = _wait_status(client)
|
|
assert s["state"] == "success", s
|
|
assert s["detail"]["added"] == 1
|
|
# No dotfile temp survived the run.
|
|
assert [p.name for p in uploads.iterdir()] == ["homelab"]
|
|
|
|
|
|
def test_background_run_starts_with_the_archive_on_disk(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""A2 pinned at the runner level: the background run receives the
|
|
archive SAFELY ON DISK — parked inside unpack (before the real
|
|
unpack and the post-unpack unlink), the dotfile temp upload file
|
|
still exists, and the module-level status (what
|
|
``GET /upload/status`` reads) is ``running`` with no file yet (A4:
|
|
"Processing…"). After release the run completes for real (real
|
|
unpack + swap) and no dotfile temp survives.
|
|
|
|
``_run_upload`` is driven directly on a worker loop (the house
|
|
background-task pattern from ``test_sync_button.py``) — the park is
|
|
a synchronous block, which would also stall the 202's response
|
|
delivery through HTTP, so the runner-level observation is the
|
|
deterministic one."""
|
|
uploads = tmp_path / "uploads"
|
|
uploads.mkdir()
|
|
_point_at(monkeypatch, uploads)
|
|
|
|
async def _no_probe(llm: object) -> None:
|
|
pass
|
|
|
|
monkeypatch.setattr(git_sources_api, "check_models", _no_probe)
|
|
monkeypatch.setattr(git_sources_api, "LLMClient", lambda: object())
|
|
monkeypatch.setattr(git_sources_api, "import_sources", _ImmediateImport(
|
|
ImportSummary(files=1, unchanged=1)
|
|
))
|
|
|
|
class _DummySession:
|
|
def scalar(self, statement: object) -> None:
|
|
return None
|
|
|
|
def add(self, row: object) -> None:
|
|
pass
|
|
|
|
def commit(self) -> None:
|
|
pass
|
|
|
|
def close(self) -> None:
|
|
pass
|
|
|
|
monkeypatch.setattr(git_sources_api, "SessionLocal", _DummySession)
|
|
|
|
# A real, valid archive in the temp upload file — exactly what the
|
|
# handler leaves behind before the 202.
|
|
real_archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"})
|
|
temp_upload = uploads / ".homelab.0.upload"
|
|
temp_unpack = uploads / ".homelab.0.unpack"
|
|
temp_upload.write_bytes(real_archive.read_bytes())
|
|
|
|
# The handler holds the flag when it creates the task — simulate.
|
|
git_sources_api._upload_in_progress = True
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
monkeypatch.setattr(git_sources_api, "unpack_archive", _GatedUnpack(started, release))
|
|
|
|
errors: list[BaseException] = []
|
|
|
|
def _run() -> None:
|
|
try:
|
|
asyncio.run(
|
|
git_sources_api._run_upload(
|
|
"homelab",
|
|
"homelab.tar.gz",
|
|
len(real_archive.read_bytes()),
|
|
uploads,
|
|
temp_upload,
|
|
temp_unpack,
|
|
)
|
|
)
|
|
except BaseException as e: # noqa: BLE001 — surfaced to the test
|
|
errors.append(e)
|
|
|
|
thread = threading.Thread(target=_run, daemon=True)
|
|
thread.start()
|
|
try:
|
|
assert started.wait(15.0), "the run never reached its unpack"
|
|
# The archive is on disk while the run is in flight (A2): the
|
|
# temp upload file exists — the run is parked before the real
|
|
# unpack and the post-unpack unlink.
|
|
assert temp_upload.is_file()
|
|
# The module-level status (what GET /upload/status reads) is
|
|
# running, with no file yet (A4).
|
|
status = git_sources_api._upload_status
|
|
assert status.state == "running"
|
|
assert status.current_file is None
|
|
assert status.files_done == 0 and status.files_total == 0
|
|
assert status.detail == {} and status.error is None
|
|
assert status.started_at is not None and status.finished_at is None
|
|
release.set()
|
|
thread.join(20)
|
|
finally:
|
|
release.set()
|
|
assert not thread.is_alive()
|
|
assert errors == [], f"the run raised: {errors!r}"
|
|
terminal = git_sources_api._upload_status
|
|
assert terminal.state == "success"
|
|
assert terminal.detail["source"] == "homelab"
|
|
# The folder is in place and no dotfile temp survived the run.
|
|
assert [p.name for p in uploads.iterdir()] == ["homelab"]
|
|
assert git_sources_api._upload_in_progress is False
|
|
|
|
|
|
# --- mid-run progress: current_file + counts (phase 64) ----------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"fail",
|
|
[
|
|
pytest.param(None, id="success-terminal"),
|
|
pytest.param(
|
|
EmbeddingError(
|
|
"embeddings request to https://u:p@aipi.example.com/v1 "
|
|
"failed: connection refused"
|
|
),
|
|
id="failed-terminal",
|
|
),
|
|
],
|
|
)
|
|
def test_mid_run_status_reports_current_file_then_terminal_clears_it(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
db: Session,
|
|
fail: BaseException | None,
|
|
) -> None:
|
|
"""Mid-run: the status carries the file the (mock) scan is
|
|
processing — assigned through the runner's own hook closure (the
|
|
task-02 seam: the progress-shaped call goes through the real hook).
|
|
Terminal states — BOTH ``success`` and ``failed`` — clear
|
|
``current_file`` but keep the run's final counts; the failure error
|
|
is sanitized (credentials masked)."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"})
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
fake_import = _GatedImport(
|
|
ImportSummary(files=3, added=1, updated=1, unchanged=1), started, release, fail
|
|
)
|
|
monkeypatch.setattr(git_sources_api, "import_sources", fake_import)
|
|
|
|
with TestClient(fastapi_app) as client:
|
|
assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
|
|
r = _post(client, "homelab.tar.gz", archive.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
assert started.wait(15.0), "the run never reached its scan"
|
|
# While the scan is parked: the hook's file is live on the status.
|
|
s = client.get("/api/git-sources/upload/status").json()
|
|
assert s["state"] == "running"
|
|
assert s["current_file"] == "homelab/notes/deep.md"
|
|
assert s["files_done"] == 1
|
|
assert s["files_total"] == 3
|
|
assert s["error"] is None
|
|
assert s["started_at"] is not None and s["finished_at"] is None
|
|
release.set()
|
|
s = _wait_status(client)
|
|
|
|
# Wiring: prune=True is preserved, and the hook fired through the
|
|
# runner's own closure (the recorded call is what the closure
|
|
# assigned to the status above).
|
|
assert fake_import.prune_flags == [True]
|
|
assert fake_import.hook_calls == [("homelab", "notes/deep.md", 1, 3)]
|
|
|
|
# Terminal: current_file null, final counts retained.
|
|
assert s["current_file"] is None
|
|
assert s["files_done"] == 1 and s["files_total"] == 3
|
|
assert s["finished_at"] is not None
|
|
if fail is None:
|
|
assert s["state"] == "success", s
|
|
assert s["error"] is None
|
|
assert s["detail"]["source"] == "homelab"
|
|
assert s["detail"]["files"] == 3
|
|
else:
|
|
assert s["state"] == "failed", s
|
|
assert s["detail"] == {}
|
|
error = s["error"] or ""
|
|
assert "*****@aipi.example.com" in error # credentials masked
|
|
assert "u:p" not in error
|
|
# A5: a post-swap failure keeps folder + row — the next
|
|
# sync/re-upload retries idempotently.
|
|
folder = uploads / "homelab"
|
|
assert folder.is_dir()
|
|
assert _row(db, str(folder)) is not None
|
|
|
|
|
|
# --- cancellation: the finally cleans temps + releases the flag --------------
|
|
|
|
|
|
def test_cancelled_run_cleans_temps_and_releases_the_flag(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""App shutdown cancels the background task: ``CancelledError`` is
|
|
deliberately NOT caught (the ``_run_sync`` rule — swallowing it
|
|
would mask a real stop), but the ``finally`` still runs: both temps
|
|
are gone and the flag is released, so nothing lingers after the
|
|
stop. ``_run_upload`` is driven directly on a worker loop (the
|
|
house background-task pattern from ``test_sync_button.py``); every
|
|
seam is faked, so no DB and no HTTP are needed."""
|
|
uploads = tmp_path / "uploads"
|
|
uploads.mkdir()
|
|
_point_at(monkeypatch, uploads)
|
|
|
|
async def _no_probe(llm: object) -> None:
|
|
pass
|
|
|
|
monkeypatch.setattr(git_sources_api, "check_models", _no_probe)
|
|
monkeypatch.setattr(git_sources_api, "LLMClient", lambda: object())
|
|
monkeypatch.setattr(git_sources_api, "unpack_archive", lambda a, t, cap: None)
|
|
monkeypatch.setattr(git_sources_api, "swap_in", lambda new_dir, final_dir: None)
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
monkeypatch.setattr(
|
|
git_sources_api,
|
|
"import_sources",
|
|
_GatedImport(ImportSummary(), started, release),
|
|
)
|
|
|
|
class _DummySession:
|
|
def scalar(self, statement: object) -> None:
|
|
return None
|
|
|
|
def add(self, row: object) -> None:
|
|
pass
|
|
|
|
def commit(self) -> None:
|
|
pass
|
|
|
|
def close(self) -> None:
|
|
pass
|
|
|
|
monkeypatch.setattr(git_sources_api, "SessionLocal", _DummySession)
|
|
|
|
# The handler holds the flag when it creates the task — simulate.
|
|
git_sources_api._upload_in_progress = True
|
|
temp_upload = uploads / ".homelab.0.upload"
|
|
temp_unpack = uploads / ".homelab.0.unpack"
|
|
temp_upload.write_bytes(b"compressed bytes")
|
|
temp_unpack.mkdir()
|
|
(temp_unpack / "alpha.md").write_text("# A\n", encoding="utf-8")
|
|
|
|
task_holder: list[asyncio.Task[None]] = []
|
|
errors: list[BaseException] = []
|
|
|
|
def _run() -> None:
|
|
async def _main() -> None:
|
|
task = asyncio.create_task(
|
|
git_sources_api._run_upload(
|
|
"homelab", "homelab.tar.gz", 4, uploads, temp_upload, temp_unpack
|
|
)
|
|
)
|
|
task_holder.append(task)
|
|
# The app-shutdown shape: cancel the task on the loop once
|
|
# the run has reached its (gated) scan.
|
|
while not started.is_set():
|
|
await asyncio.sleep(0.01)
|
|
task.cancel()
|
|
# Free the gated import's worker thread AFTER the cancel:
|
|
# a to_thread future cannot be cancelled while its thread
|
|
# runs, so the task only lands in the cancelled state once
|
|
# the worker is free.
|
|
release.set()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError as e:
|
|
errors.append(e)
|
|
|
|
try:
|
|
asyncio.run(_main())
|
|
except BaseException as e: # noqa: BLE001 — surfaced to the test
|
|
errors.append(e)
|
|
|
|
thread = threading.Thread(target=_run, daemon=True)
|
|
thread.start()
|
|
try:
|
|
thread.join(20)
|
|
finally:
|
|
release.set() # backstop — the loop already released on cancel
|
|
assert not thread.is_alive()
|
|
# The CancelledError propagated (not swallowed) — and the finally
|
|
# did its cleanup: no temp survives, the flag is released.
|
|
assert len(errors) == 1
|
|
assert isinstance(errors[0], asyncio.CancelledError)
|
|
assert not temp_upload.exists()
|
|
assert not temp_unpack.exists()
|
|
assert git_sources_api._upload_in_progress is False
|
|
|
|
|
|
# --- in-flight run must not block a concurrent TRUNCATE (regression, --------
|
|
# --- phase 49 task 03; now structural in phase 64) ---------------------------
|
|
|
|
|
|
def test_in_flight_upload_does_not_block_a_concurrent_truncate(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""While the background run is in flight, a concurrent TRUNCATE of
|
|
the KB + registry tables (the E2E isolation-fixture pattern) must
|
|
complete.
|
|
|
|
Regression for the phase-49 task-03 deadlock, now structural (phase
|
|
64): the handler no longer touches the DB at all, and the row
|
|
upsert inside the run commits in its own short-lived session
|
|
(closed before the scan) — so no session can hold ``git_sources``
|
|
locks across the scan. This pins it: with the scan held in flight,
|
|
the TRUNCATE completes well inside its 5 s ``lock_timeout``
|
|
(without the fix it times out with a lock-not-available error)."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
archive = _tarball(tmp_path / "homelab.tar.gz", {"alpha.md": "# Alpha\nx\n"})
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
fake_import = _GatedImport(ImportSummary(files=1, unchanged=1), started, release)
|
|
monkeypatch.setattr(git_sources_api, "import_sources", fake_import)
|
|
|
|
with TestClient(fastapi_app) as client:
|
|
assert client.post("/api/login", json={"password": ADMIN_PASSWORD}).status_code == 204
|
|
r = _post(client, "homelab.tar.gz", archive.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
assert started.wait(15.0), "background run did not reach its scan"
|
|
# The E2E isolation TRUNCATE, exactly as the story fixtures run
|
|
# it — must complete while the scan is held in flight.
|
|
with SessionLocal() as tr, tr.begin():
|
|
tr.execute(text("SET LOCAL lock_timeout = '5s'"))
|
|
tr.execute(
|
|
text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources")
|
|
)
|
|
release.set()
|
|
status = _wait_status(client)
|
|
assert status["state"] == "success", status
|
|
|
|
|
|
# --- streaming cap (413) ----------------------------------------------------
|
|
|
|
|
|
def test_oversized_upload_gets_413_and_leaves_no_temp(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads, upload_max_mb=1)
|
|
|
|
# 2 MiB of incompressible bytes with a 1 MiB cap → 413 naming the
|
|
# cap (inline — the run never starts).
|
|
big = os.urandom(2 * 1024 * 1024)
|
|
r = _post(upload_client, "big.zip", big)
|
|
assert r.status_code == 413
|
|
assert "1 MiB" in r.json()["detail"]
|
|
assert _count_rows(db) == 0
|
|
assert uploads.exists() # the dir was created before streaming
|
|
assert list(uploads.iterdir()) == [] # the temp .upload file is gone
|
|
|
|
# The inline exception path released the flag: the next upload is
|
|
# NOT 409. Boundary: EXACTLY the cap is not 413 (the check is
|
|
# strictly >) — the stream completes, 202 lands, and the garbage
|
|
# bytes fail at the BACKGROUND unpack instead (a status state, A5 —
|
|
# never an HTTP error post-202).
|
|
r = _post(upload_client, "exact.zip", os.urandom(1024 * 1024))
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert "could not unpack the archive" in (status["error"] or "")
|
|
assert list(uploads.iterdir()) == []
|
|
|
|
|
|
# --- unpack safety: failed runs leave the previous state intact ---------------
|
|
|
|
|
|
def test_zip_slip_archive_fails_the_run_and_previous_state_is_intact(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
files = {
|
|
"alpha.md": "# Alpha\noriginal sentinel one\n",
|
|
"bravo.md": "# Bravo\noriginal sentinel two\n",
|
|
}
|
|
_good_upload(upload_client, "safe", files)
|
|
|
|
evil = _zip(tmp_path / "safe.zip", {"../evil.txt": "pwned"})
|
|
r = _post(upload_client, "safe.zip", evil.read_bytes())
|
|
assert r.status_code == 202, r.text # on disk → the failure is a status state
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert "traversal" in (status["error"] or "")
|
|
assert not (tmp_path / "evil.txt").exists() # the escape never landed
|
|
|
|
_assert_previous_intact(upload_client, db, uploads, "safe", files)
|
|
|
|
|
|
def test_tar_symlink_escape_fails_the_run_and_previous_state_is_intact(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
files = {
|
|
"alpha.md": "# Alpha\noriginal sentinel one\n",
|
|
"bravo.md": "# Bravo\noriginal sentinel two\n",
|
|
}
|
|
_good_upload(upload_client, "safe", files)
|
|
|
|
evil = tmp_path / "safe.tar"
|
|
with tarfile.open(evil, "w") as tf:
|
|
info = tarfile.TarInfo("link")
|
|
info.type = tarfile.SYMTYPE
|
|
info.linkname = "/etc/passwd"
|
|
tf.addfile(info)
|
|
r = _post(upload_client, "safe.tar", evil.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert "escape" in (status["error"] or "")
|
|
|
|
_assert_previous_intact(upload_client, db, uploads, "safe", files)
|
|
|
|
|
|
def test_corrupt_archive_fails_the_run_and_previous_state_is_intact(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
files = {
|
|
"alpha.md": "# Alpha\noriginal sentinel one\n",
|
|
"bravo.md": "# Bravo\noriginal sentinel two\n",
|
|
}
|
|
_good_upload(upload_client, "safe", files)
|
|
|
|
# A truncated zip (the EOCD is cut off) is not a zip and not a tar.
|
|
good_zip = _zip(tmp_path / "good.zip", {"alpha.md": "# Alpha\nx\n"})
|
|
good_bytes = good_zip.read_bytes()
|
|
truncated = good_bytes[: len(good_bytes) // 2]
|
|
r = _post(upload_client, "safe.zip", truncated)
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert "could not unpack the archive" in (status["error"] or "")
|
|
|
|
# A zero-byte "archive" fails the same way.
|
|
r = _post(upload_client, "safe.tar", b"")
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert "could not unpack the archive" in (status["error"] or "")
|
|
|
|
_assert_previous_intact(upload_client, db, uploads, "safe", files)
|
|
|
|
|
|
def test_swap_failure_fails_the_run_and_previous_state_is_intact(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
"""A rename failure in ``swap_in`` (OS error) → status ``failed``
|
|
with its message; the previous folder/row/KB are untouched and no
|
|
temp survives (the run's finally cleans the temp sibling). A
|
|
failure here leaves the previous folder/row/KB untouched (A5)."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
files = {
|
|
"alpha.md": "# Alpha\noriginal sentinel one\n",
|
|
"bravo.md": "# Bravo\noriginal sentinel two\n",
|
|
}
|
|
_good_upload(upload_client, "safe", files)
|
|
|
|
def failing_swap(new_dir: Path, final_dir: Path) -> None:
|
|
raise ArchiveUploadError("could not replace the previous folder")
|
|
|
|
monkeypatch.setattr(git_sources_api, "swap_in", failing_swap)
|
|
r = _post(upload_client, "safe.tar.gz", _targz_bytes({"alpha.md": "# A\nx\n"}))
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert status["error"] == "could not replace the previous folder"
|
|
|
|
_assert_previous_intact(upload_client, db, uploads, "safe", files)
|
|
|
|
|
|
def test_zero_entry_archive_fails_the_run(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
"""A completely empty archive (zero entries) → status ``failed``
|
|
``the archive contains no files`` — both containers — with no
|
|
folder, row, or temp file left behind (a pre-swap failure leaves
|
|
the KB/folders/rows untouched, A5)."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
|
|
empty_targz = tmp_path / "empty.tar.gz"
|
|
with tarfile.open(empty_targz, "w:gz"):
|
|
pass
|
|
r = _post(upload_client, "empty.tar.gz", empty_targz.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert status["error"] == "the archive contains no files"
|
|
|
|
empty_zip = tmp_path / "empty.zip"
|
|
with zipfile.ZipFile(empty_zip, "w"):
|
|
pass
|
|
r = _post(upload_client, "empty.zip", empty_zip.read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert status["error"] == "the archive contains no files"
|
|
|
|
assert _count_rows(db) == 0
|
|
assert not (uploads / "empty").exists()
|
|
assert list(uploads.iterdir()) == []
|
|
|
|
|
|
def test_concurrent_row_insert_fails_the_run_with_the_path_named(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""The upsert's unique-index backstop: the pre-check sees no row,
|
|
but the commit hits the unique constraint (a concurrent insert the
|
|
pre-check missed) → status ``failed`` naming the path. The folder
|
|
stays — the row exists (the concurrent one) and the next
|
|
sync/re-upload sees it (A5); no temp survives and the short-lived
|
|
session was closed."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
|
|
class _ConcurrentInsertSession:
|
|
"""The short-lived upsert session, faked: the pre-check misses
|
|
(no row), the commit hits the unique index."""
|
|
|
|
def __init__(self) -> None:
|
|
self.closed = False
|
|
self.added: list[object] = []
|
|
|
|
def scalar(self, statement: object) -> None:
|
|
return None
|
|
|
|
def add(self, row: object) -> None:
|
|
self.added.append(row)
|
|
|
|
def commit(self) -> None:
|
|
raise IntegrityError("INSERT INTO git_sources", {}, Exception("duplicate key"))
|
|
|
|
def rollback(self) -> None:
|
|
pass
|
|
|
|
def close(self) -> None:
|
|
self.closed = True
|
|
|
|
sessions: list[_ConcurrentInsertSession] = []
|
|
|
|
def _fake_session_factory() -> _ConcurrentInsertSession:
|
|
s = _ConcurrentInsertSession()
|
|
sessions.append(s)
|
|
return s
|
|
|
|
monkeypatch.setattr(git_sources_api, "SessionLocal", _fake_session_factory)
|
|
r = _post(
|
|
upload_client,
|
|
"homelab.tar.gz",
|
|
_tarball(tmp_path / "h.tar.gz", {"alpha.md": "# Alpha\nx\n"}).read_bytes(),
|
|
)
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed"
|
|
assert status["error"] == (
|
|
f"a local source with this path already exists: {uploads / 'homelab'}"
|
|
)
|
|
# The folder stays (the row exists — the concurrent insert — and
|
|
# the next sync sees it); no temp survived; the session was closed.
|
|
assert (uploads / "homelab").is_dir()
|
|
assert [p.name for p in uploads.iterdir()] == ["homelab"]
|
|
assert len(sessions) == 1
|
|
assert sessions[0].added, "the run attempted the insert (the pre-check missed)"
|
|
assert sessions[0].closed
|
|
|
|
|
|
# --- happy path -------------------------------------------------------------
|
|
|
|
|
|
def test_happy_path_tar_gz(
|
|
upload_client: TestClient,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
db: Session,
|
|
tmp_path: Path,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
files = {"alpha.md": "# Alpha\nfirst sentinel\n", "bravo.md": "# Bravo\nsecond sentinel\n"}
|
|
payload = _tarball(tmp_path / "homelab.tar.gz", files).read_bytes()
|
|
|
|
with caplog.at_level(logging.INFO, logger="app.api.git_sources"):
|
|
r = _post(upload_client, "homelab.tar.gz", payload)
|
|
assert r.status_code == 202, r.text
|
|
# 202 = the archive is on disk (A2) — the safe name rides back
|
|
# in the ``UploadAccepted`` body (the toast moment).
|
|
assert r.json() == {"detail": "upload received", "name": "homelab"}
|
|
status = _wait_status(upload_client)
|
|
|
|
assert status["state"] == "success", status
|
|
assert status["error"] is None
|
|
# ``detail`` carries the ``UploadOut`` fields (the pre-phase-64
|
|
# response shape — now the status's success detail).
|
|
detail = status["detail"]
|
|
assert set(detail) == {
|
|
"source", "files", "added", "updated", "unchanged", "pruned", "errors", "chunks",
|
|
"overview",
|
|
}
|
|
assert detail["source"] == "homelab" # filename minus the archive suffix
|
|
assert detail["files"] == 2
|
|
assert detail["added"] == 2
|
|
assert detail["updated"] == 0
|
|
assert detail["unchanged"] == 0
|
|
assert detail["pruned"] == 0
|
|
assert detail["errors"] == 0
|
|
assert detail["chunks"] >= 2
|
|
assert detail["overview"] is True # the KB changed → the overview refreshed
|
|
# Terminal progress state: current_file null, timestamps set.
|
|
assert status["current_file"] is None
|
|
assert status["started_at"] is not None and status["finished_at"] is not None
|
|
|
|
# Unpacked under the tmp upload dir, dotfile temps cleaned up.
|
|
folder = uploads / "homelab"
|
|
assert folder.is_dir()
|
|
assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == files
|
|
assert [p.name for p in uploads.iterdir()] == ["homelab"]
|
|
|
|
# One kind=local row; the NOT-NULL url column carries the path.
|
|
row = _row(db, str(folder))
|
|
assert row is not None
|
|
assert row.kind == "local"
|
|
assert row.url == str(folder)
|
|
assert row.path == str(folder)
|
|
assert _count_rows(db) == 1
|
|
|
|
# The KB lists both files under the source name.
|
|
assert _docs(upload_client) == [("homelab", "alpha.md"), ("homelab", "bravo.md")]
|
|
|
|
# The per-upload log line (PLAN §9 / AGENTS.md rule 10) — moved
|
|
# with the scan into the background task (``total_ms`` is the run's
|
|
# duration).
|
|
lines = [
|
|
rec.getMessage() for rec in caplog.records if rec.getMessage().startswith("upload: ")
|
|
]
|
|
assert len(lines) == 1, lines
|
|
match = re.match(
|
|
r"^upload: name=homelab file=homelab\.tar\.gz bytes_in=(\d+) files=2 added=2 "
|
|
r"updated=0 unchanged=0 pruned=0 errors=0 overview=True total_ms=\d+$",
|
|
lines[0],
|
|
)
|
|
assert match, lines[0]
|
|
assert int(match.group(1)) == len(payload) # the compressed bytes in
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("filename", "builder"),
|
|
[
|
|
("notes.zip", "zip"),
|
|
("plain.tar", "tar"),
|
|
("tgz.tgz", "targz"),
|
|
],
|
|
)
|
|
def test_happy_path_other_accepted_formats(
|
|
upload_client: TestClient,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
db: Session,
|
|
tmp_path: Path,
|
|
filename: str,
|
|
builder: str,
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
files = {"alpha.md": "# Alpha\nsentinel\n"}
|
|
make = {"zip": _zip, "tar": lambda p, f: _tarball(p, f, "w"), "targz": _tarball}[builder]
|
|
payload = make(tmp_path / filename, files).read_bytes()
|
|
|
|
r = _post(upload_client, filename, payload)
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "success", status
|
|
detail = status["detail"]
|
|
expected_name = {"notes.zip": "notes", "plain.tar": "plain", "tgz.tgz": "tgz"}[filename]
|
|
assert r.json()["name"] == expected_name # the safe name in the 202 body
|
|
assert detail["source"] == expected_name
|
|
assert detail["added"] == 1
|
|
assert _docs(upload_client) == [(expected_name, "alpha.md")]
|
|
assert _count_rows(db) == 1
|
|
|
|
|
|
# --- re-upload, same name: in-place replace ---------------------------------
|
|
|
|
|
|
def test_reupload_same_name_replaces_in_place(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
v1 = {
|
|
"alpha.md": "# Alpha\nv1 content\n",
|
|
"bravo.md": "# Bravo\nv1 content\n",
|
|
"charlie.md": "# Charlie\nv1 content\n",
|
|
}
|
|
r = _post(upload_client, "homelab.tar.gz", _tarball(tmp_path / "v1.tar.gz", v1).read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "success", status
|
|
assert status["detail"]["added"] == 3
|
|
row_before = _row(db, str(uploads / "homelab"))
|
|
assert row_before is not None
|
|
added_at_before = row_before.added_at
|
|
|
|
# v2: alpha changed, bravo dropped, delta new — the in-place-replace
|
|
# scenario, now observed via the SECOND run's status.
|
|
v2 = {"alpha.md": "# Alpha\nv2 CHANGED content\n", "delta.md": "# Delta\nbrand new\n"}
|
|
r = _post(upload_client, "homelab.tar.gz", _tarball(tmp_path / "v2.tar.gz", v2).read_bytes())
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "success", status
|
|
detail = status["detail"]
|
|
assert detail["source"] == "homelab"
|
|
assert detail["files"] == 2
|
|
assert detail["added"] == 1 # delta.md
|
|
assert detail["updated"] == 1 # alpha.md (hash changed)
|
|
assert detail["unchanged"] == 0
|
|
assert detail["pruned"] == 2 # bravo.md + charlie.md left the folder → pruned
|
|
assert detail["overview"] is True
|
|
|
|
# Exactly ONE row for the path, and its added_at survived (the
|
|
# second run's upsert left the existing row alone).
|
|
assert _count_rows(db) == 1
|
|
row_after = _row(db, str(uploads / "homelab"))
|
|
assert row_after is not None
|
|
assert row_after.id == row_before.id
|
|
assert row_after.added_at == added_at_before
|
|
|
|
# The folder holds ONLY the new archive's files (full replacement —
|
|
# no stale files from v1), and the KB mirrors it.
|
|
folder = uploads / "homelab"
|
|
assert {p.name: p.read_text(encoding="utf-8") for p in folder.iterdir()} == v2
|
|
assert [p.name for p in uploads.iterdir()] == ["homelab"]
|
|
assert _docs(upload_client) == [("homelab", "alpha.md"), ("homelab", "delta.md")]
|
|
|
|
|
|
def test_non_a9_archive_is_a_valid_replacement(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
"""An archive with only non-A9 files is a valid replacement: the
|
|
swap happens, the scan indexes nothing, prune removes the source's
|
|
docs, and the overview is NOT refreshed (no KB change)."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
|
|
r = _post(
|
|
upload_client,
|
|
"notes.tar.gz",
|
|
_tarball(tmp_path / "a.tar.gz", {"readme.md": "# Readme\nv1\n"}).read_bytes(),
|
|
)
|
|
assert r.status_code == 202, r.text
|
|
assert _wait_status(upload_client)["state"] == "success"
|
|
assert _docs(upload_client) == [("notes", "readme.md")]
|
|
|
|
r = _post(
|
|
upload_client,
|
|
"notes.tar.gz",
|
|
_tarball(tmp_path / "b.tar.gz", {"binary.bin": "not an importable format"}).read_bytes(),
|
|
)
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "success", status
|
|
detail = status["detail"]
|
|
assert detail["files"] == 0 # nothing matches the A9 filter
|
|
assert detail["added"] == 0
|
|
assert detail["updated"] == 0
|
|
assert detail["pruned"] == 1 # readme.md left the KB
|
|
assert detail["overview"] is False # added + updated == 0 → no refresh
|
|
|
|
assert {p.name for p in (uploads / "notes").iterdir()} == {"binary.bin"}
|
|
assert _count_rows(db) == 1
|
|
assert _docs(upload_client) == []
|
|
|
|
|
|
# --- fail-fast models (the phase-49 503 becomes a status state, A5) -----------
|
|
|
|
|
|
def test_models_down_fails_the_run_and_leaves_folder_and_row(
|
|
upload_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
|
|
) -> None:
|
|
"""The folder and row are committed BEFORE the model probe: a dead
|
|
endpoint lands in the status as ``failed`` with the sanitized
|
|
model-unavailable message (the phase-49 503 becomes a status state,
|
|
A5) and the next sync/re-upload retries idempotently; the scan
|
|
never ran."""
|
|
uploads = tmp_path / "uploads"
|
|
_point_at(monkeypatch, uploads)
|
|
_real_llm(monkeypatch)
|
|
|
|
async def dead_probe(llm: object) -> None:
|
|
raise ModelUnavailableError(
|
|
"The embedding model ('embed') is not available — check the model "
|
|
"endpoint and retry. (embeddings request to "
|
|
"https://user:secret@aipi.reeseapps.com/v1 failed: connection refused)"
|
|
)
|
|
|
|
monkeypatch.setattr(git_sources_api, "check_models", dead_probe)
|
|
r = _post(
|
|
upload_client,
|
|
"homelab.tar.gz",
|
|
_tarball(tmp_path / "h.tar.gz", {"alpha.md": "# Alpha\nx\n"}).read_bytes(),
|
|
)
|
|
assert r.status_code == 202, r.text
|
|
status = _wait_status(upload_client)
|
|
assert status["state"] == "failed", status
|
|
error = status["error"] or ""
|
|
assert "The embedding model ('embed') is not available" in error
|
|
assert "*****@aipi.reeseapps.com" in error # the sanitizer masked the credentials
|
|
assert "user:secret" not in error
|
|
assert "connection refused" in error # the reason survives
|
|
|
|
# The folder and row are already committed (idempotent retry path).
|
|
folder = uploads / "homelab"
|
|
assert folder.is_dir()
|
|
row = _row(db, str(folder))
|
|
assert row is not None
|
|
assert row.kind == "local"
|
|
assert row.url == str(folder)
|
|
# The scan never ran: no docs, no stray temps.
|
|
assert _docs(upload_client) == []
|
|
assert [p.name for p in uploads.iterdir()] == ["homelab"]
|