brain-of-reese/tests/integration/test_document_content.py

"""Integration tests: the document-content API — the viewer's data source.

Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient:
* GET: 200 with the full field set for a seeded document (all formats);
* GET: ``summary`` surfaced for summarized docs, ``null`` for markdown
  (phase 36);
* GET: user-gated (phase 79 — the phase-16 "public viewer" soft rule is
  superseded; the shared chats are the anonymous surface now), so the
  shared ``client`` is signed in as the admin for the module's contract
  tests and the anonymous 403/401 pins build their own client;
* GET: 404 for an unknown (source, path) pair;
* GET: 404 for traversal-style ``path`` values (no filesystem access → no
  leak).

PATCH /api/documents/summary (phase 57, task 01) — the admin summary
editor, on the same DB-backed fixtures:
* update: ``documents.summary`` + the ``is_summary`` chunk's content
  replaced, fresh embedding, content chunks untouched (count unchanged —
  D4 re-embed scope);
* clear: empty/whitespace text → ``summary`` NULL + ``is_summary`` chunk
  deleted (idempotent, no embed call);
* markdown doc (no prior ``is_summary`` chunk) → one created at
  position −1 with the new embedding;
* 404 unknown (source, path) (incl. traversal strings); 403 anonymous;
* embed failure → 503 sanitized detail, DB byte-for-byte untouched
  (fail-before-write — embed before any mutation).
"""
from __future__ import annotations

from datetime import UTC, datetime

import pytest
from fastapi.testclient import TestClient
from sqlalchemy import select, text
from sqlalchemy.orm import Session

import app.api.docs as docs_api
from app.config import get_settings
from app.main import app as fastapi_app
from app.models import Chunk, Document
from app.rag.llm import EmbeddingError
from tests.conftest import ADMIN_PASSWORD
from tests.fakes import FakeEmbedder


@pytest.fixture(autouse=True)
def _user_signed_in(client: TestClient) -> None:
    """Phase 79 (task 03): the document-content endpoint is user-gated —
    the shared ``client`` signs in as the admin for this module's GET /
    PATCH contract tests. The ONE test that pins the anonymous PATCH 403
    builds its own client (it must stay unsigned)."""
    r = client.post("/api/login", json={"password": ADMIN_PASSWORD})
    assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}"


def _seed_doc(
    db,
    source: str = "Homelab",
    path: str = "kubernetes.md",
    title: str = "Kubernetes Homelab Cluster",
    content: str = "# Kubernetes\n\nTalos on 3 nodes.",
    chunks: int = 2,
    summary: str | None = None,
    summary_chunk: bool = False,
) -> None:
    """Truncate the KB and insert one document with ``chunks`` chunk rows.

    ``summary_chunk`` additionally indexes the phase-30 ``is_summary``
    chunk at position −1 with the seeded vector (requires ``summary`` —
    the phase-30 invariant: the chunk mirrors ``documents.summary``).
    """
    db.execute(text("TRUNCATE chunks, documents"))
    db.commit()
    doc = Document(
        source=source,
        path=path,
        full_path=f"/tmp/{path}",
        title=title,
        content=content,
        content_hash="a" * 64,
        indexed_at=datetime.now(UTC),
        # Phase 106: pin the creation date explicitly — the endpoint
        # serves it verbatim (the column is NOT NULL; the server
        # default would make the pin time-dependent).
        created_at=datetime(2020, 5, 4, 8, 30, 0, tzinfo=UTC),
        summary=summary,
    )
    db.add(doc)
    db.flush()
    if chunks:
        db.add_all(
            Chunk(document_id=doc.id, position=i, content=f"chunk {i}", embedding=[0.01] * 768)
            for i in range(chunks)
        )
    if summary_chunk:
        assert summary is not None
        db.add(
            Chunk(
                document_id=doc.id,
                position=-1,
                content=summary,
                embedding=[0.01] * 768,
                is_summary=True,
            )
        )
    db.commit()


# ---------------------------------------------------------------------------
# PATCH /api/documents/summary (phase 57, task 01)
# ---------------------------------------------------------------------------


class _DeadEmbedder:
    """An ``LLMClient`` stand-in whose ``embed`` always fails — the
    dead-endpoint path (phase 57 fail-before-write). The message mirrors
    ``LLMClient.embed``'s transport wrap, with embedded credentials that
    the sanitizer must mask."""

    def __init__(self) -> None:
        self.calls: list[list[str]] = []

    async def embed(self, texts: list[str]) -> list[list[float]]:
        self.calls.append(list(texts))
        raise EmbeddingError(
            "embeddings request to https://user:secret@aipi.example.com/v1 "
            "failed: connection refused"
        )


def _seed_yaml_doc(db, *, summary: str, summary_chunk: bool) -> None:
    """One phase-30-style non-markdown doc: 2 content chunks + (optionally)
    its ``is_summary`` chunk."""
    _seed_doc(
        db,
        path="container_gitlab/gitlab-compose.yaml",
        title="gitlab-compose",
        content="services:\n  gitlab:\n    image: gitlab/gitlab-ce",
        summary=summary,
        summary_chunk=summary_chunk,
    )


def _doc_id(db, path: str):
    return db.scalar(select(Document.id).where(Document.path == path))


def _chunk_snapshots(db, doc_id) -> dict:
    """``{chunk id: (position, content, embedding, is_summary)}``.

    Before/after comparisons prove exactly which rows a PATCH touched.
    Embeddings are compared read-back to read-back: pgvector stores
    float4, so raw Python floats do not round-trip exactly (the house
    style elsewhere is ``is not None`` + dimension checks)."""
    rows = db.scalars(select(Chunk).where(Chunk.document_id == doc_id)).all()
    return {
        c.id: (
            c.position,
            c.content,
            list(c.embedding) if c.embedding is not None else None,
            c.is_summary,
        )
        for c in rows
    }


def test_summary_patch_update_reembeds_summary_chunk(
    admin_client: TestClient, client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session
) -> None:
    """Admin PATCH with new text (phase 57, D4): ``documents.summary`` and
    the ``is_summary`` chunk's content are replaced and the chunk gets a
    fresh embedding — in place (same row id); the content chunks are
    untouched (same ids, positions, contents, vectors) and the total
    count is unchanged. One ``embed`` call, only with the new text."""
    old = "GitLab CE runs in a Podman compose stack on the homelab NAS."
    new = "GitLab CE is now backed by external PostgreSQL and MinIO."
    _seed_yaml_doc(db, summary=old, summary_chunk=True)
    doc_id = _doc_id(db, "container_gitlab/gitlab-compose.yaml")
    db.expire_all()
    before = _chunk_snapshots(db, doc_id)
    assert len(before) == 3
    try:
        fake = FakeEmbedder()
        monkeypatch.setattr(docs_api, "LLMClient", lambda: fake)
        r = admin_client.patch(
            "/api/documents/summary",
            json={
                "source": "Homelab",
                "path": "container_gitlab/gitlab-compose.yaml",
                "summary": new,
            },
        )
        assert r.status_code == 200, r.text
        body = r.json()
        assert set(body) == {"source", "path", "summary", "chunks"}
        assert body["source"] == "Homelab"
        assert body["path"] == "container_gitlab/gitlab-compose.yaml"
        assert body["summary"] == new
        assert body["chunks"] == 3  # 2 content + 1 is_summary — unchanged by an update

        db.expire_all()
        row = db.get(Document, doc_id)
        assert row is not None
        assert row.summary == new
        after = _chunk_snapshots(db, doc_id)
        assert set(after) == set(before)  # same rows: nothing added or deleted
        sc = next(cid for cid, v in after.items() if v[3])
        assert sc in before  # replaced in place — the same row id
        pos, content, vec, _ = after[sc]
        assert pos == -1
        assert content == new
        assert vec is not None and len(vec) == 768
        assert vec != before[sc][2]  # fresh embedding, not the seeded one
        for cid, v in after.items():
            if not v[3]:
                assert v == before[cid]  # content chunks untouched, byte for byte
        assert fake.calls == [[new]]  # one embed call, the new text only
        # The viewer's data source now carries the new summary verbatim
        # (the ``client`` is signed in — phase 79 gated the viewer).
        g = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "container_gitlab/gitlab-compose.yaml"},
        )
        assert g.status_code == 200
        assert g.json()["summary"] == new
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_summary_patch_clear(
    admin_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session
) -> None:
    """PATCH with empty/whitespace text clears (phase 57, D4):
    ``documents.summary`` → NULL and the ``is_summary`` chunk is deleted;
    the content chunks are untouched. A second clear is an idempotent 200
    and clearing never calls the embedder."""
    old = "GitLab CE runs in a Podman compose stack on the homelab NAS."
    _seed_yaml_doc(db, summary=old, summary_chunk=True)
    doc_id = _doc_id(db, "container_gitlab/gitlab-compose.yaml")
    db.expire_all()
    before = _chunk_snapshots(db, doc_id)
    assert len(before) == 3
    try:
        fake = FakeEmbedder()
        monkeypatch.setattr(docs_api, "LLMClient", lambda: fake)
        for body_summary in ("   ", ""):  # whitespace, then a true empty string
            r = admin_client.patch(
                "/api/documents/summary",
                json={
                    "source": "Homelab",
                    "path": "container_gitlab/gitlab-compose.yaml",
                    "summary": body_summary,
                },
            )
            assert r.status_code == 200, r.text
            assert r.json() == {
                "source": "Homelab",
                "path": "container_gitlab/gitlab-compose.yaml",
                "summary": None,
                "chunks": 2,
            }
        db.expire_all()
        row = db.get(Document, doc_id)
        assert row is not None
        assert row.summary is None
        after = _chunk_snapshots(db, doc_id)
        assert len(after) == 2  # the is_summary chunk row is gone (count −1)
        for cid, v in after.items():
            assert not v[3]
            assert v == before[cid]  # content chunks untouched, byte for byte
        assert fake.calls == []  # clearing never embeds
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_summary_patch_markdown_doc_creates_summary_chunk(
    admin_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session
) -> None:
    """A markdown doc (no ``is_summary`` chunk by construction — phase 30)
    gets exactly one created at position −1 with the new embedding; the
    content chunks are untouched and the count goes 2 → 3. This is also
    the recovery path for a phase-30 fail-soft import (document indexed
    without its summary chunk)."""
    new = "Talos Kubernetes on 3 nodes with a CNI of choice."
    _seed_doc(db, summary=None, summary_chunk=False)  # the default kubernetes.md
    doc_id = _doc_id(db, "kubernetes.md")
    db.expire_all()
    before = _chunk_snapshots(db, doc_id)
    assert len(before) == 2
    try:
        fake = FakeEmbedder()
        monkeypatch.setattr(docs_api, "LLMClient", lambda: fake)
        r = admin_client.patch(
            "/api/documents/summary",
            json={"source": "Homelab", "path": "kubernetes.md", "summary": new},
        )
        assert r.status_code == 200, r.text
        body = r.json()
        assert body["source"] == "Homelab"
        assert body["path"] == "kubernetes.md"
        assert body["summary"] == new
        assert body["chunks"] == 3  # 2 content + the new is_summary
        db.expire_all()
        row = db.get(Document, doc_id)
        assert row is not None
        assert row.summary == new
        after = _chunk_snapshots(db, doc_id)
        assert len(after) == 3
        for cid, v in before.items():
            assert after[cid] == v  # content chunks untouched, byte for byte
        sc = next(cid for cid in after if cid not in before)
        pos, content, vec, is_summary = after[sc]
        assert (pos, is_summary) == (-1, True)
        assert content == new
        assert vec is not None and len(vec) == 768
        assert fake.calls == [[new]]
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_summary_patch_404_unknown_pair(admin_client: TestClient, db: Session) -> None:
    """Unknown (source, path) pairs — including traversal strings — are
    just missing rows: 404 ``document not found`` (the same shape as the
    public GET). A pair that exists under a DIFFERENT source is 404 too."""
    _seed_yaml_doc(db, summary="old", summary_chunk=True)
    try:
        for source, path in (
            ("Homelab", "nope/missing.md"),
            ("Deployments", "container_gitlab/gitlab-compose.yaml"),
            ("Homelab", "../../etc/passwd"),
        ):
            r = admin_client.patch(
                "/api/documents/summary",
                json={"source": source, "path": path, "summary": "whatever"},
            )
            assert r.status_code == 404, (source, path)
            assert r.json() == {"detail": "document not found"}, (source, path)
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_summary_patch_403_anonymous(db: Session) -> None:
    """The edit affordance is admin-only (phase 57, D4 — the viewer is
    user-gated since phase 79): an anonymous PATCH gets 403 ``admin
    only`` and touches nothing. A FRESH client — the module's autouse
    fixture signed the shared one in."""
    old = "GitLab CE runs in a Podman compose stack on the homelab NAS."
    _seed_yaml_doc(db, summary=old, summary_chunk=True)
    doc_id = _doc_id(db, "container_gitlab/gitlab-compose.yaml")
    anonymous = TestClient(fastapi_app)
    try:
        r = anonymous.patch(
            "/api/documents/summary",
            json={
                "source": "Homelab",
                "path": "container_gitlab/gitlab-compose.yaml",
                "summary": "not allowed",
            },
        )
        assert r.status_code == 403
        assert r.json() == {"detail": "admin only"}
        db.expire_all()
        row = db.get(Document, doc_id)
        assert row is not None
        assert row.summary == old  # untouched
        assert len(_chunk_snapshots(db, doc_id)) == 3
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_summary_patch_embed_failure_503_db_untouched(
    admin_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session
) -> None:
    """Fail-before-write (phase 57 locked decision): a dead embedding
    endpoint → 503 with a sanitized detail (credentials masked, the
    reason survives — the ``git_sources.py`` ``ModelUnavailableError``
    style) and the row + every chunk byte-for-byte as before."""
    old = "GitLab CE runs in a Podman compose stack on the homelab NAS."
    _seed_yaml_doc(db, summary=old, summary_chunk=True)
    doc_id = _doc_id(db, "container_gitlab/gitlab-compose.yaml")
    db.expire_all()
    before = _chunk_snapshots(db, doc_id)
    try:
        dead = _DeadEmbedder()
        monkeypatch.setattr(docs_api, "LLMClient", lambda: dead)
        r = admin_client.patch(
            "/api/documents/summary",
            json={
                "source": "Homelab",
                "path": "container_gitlab/gitlab-compose.yaml",
                "summary": "new text",
            },
        )
        assert r.status_code == 503
        detail = r.json()["detail"]
        assert "*****@aipi.example.com" in detail  # credentials masked
        assert "user:secret" not in detail
        assert "connection refused" in detail  # the reason survives
        assert dead.calls == [["new text"]]  # the embed was attempted…
        db.expire_all()
        row = db.get(Document, doc_id)
        assert row is not None
        assert row.summary == old  # …and failed before any mutation
        assert _chunk_snapshots(db, doc_id) == before  # every row, byte for byte
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_content_200_all_fields(client, db) -> None:
    _seed_doc(db)
    try:
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "kubernetes.md"},
        )
        assert r.status_code == 200
        body = r.json()
        # Wire-additive (phase 106, task 05): ``created_at`` joins the
        # content shape (after ``summary``, before ``content``) — and
        # (phase 122, task 04) ``is_image`` joins it ALWAYS present
        # (text docs: false); ``image_url`` is ABSENT for a text doc
        # (never null — the ``DocContent`` omission rule).
        assert set(body) == {
            "source", "path", "title", "format", "summary", "created_at",
            "content", "indexed_at", "chunks", "is_image",
        }
        assert body["is_image"] is False
        assert "image_url" not in body  # absent — never null (text doc)
        datetime.fromisoformat(body["created_at"])  # raises if not ISO-8601
        assert body["source"] == "Homelab"
        assert body["path"] == "kubernetes.md"
        assert body["title"] == "Kubernetes Homelab Cluster"
        assert body["format"] == "md"
        assert body["summary"] is None  # markdown doc → no summary (phase 36)
        assert body["content"] == "# Kubernetes\n\nTalos on 3 nodes."
        assert body["chunks"] == 2
        datetime.fromisoformat(body["indexed_at"])  # raises if not ISO-8601
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_content_summary_surfaced_for_summarized_doc(client, db) -> None:
    """A non-markdown document with a phase-30 summary returns it verbatim
    (phase 36 — the viewer's data contract gains the nullable field),
    for a signed-in caller (phase 79 — the viewer is token-or-admin; the
    anonymous 401 contract is pinned in ``test_auth_api.py``)."""
    summary = (
        "GitLab CE runs in a Podman compose stack on the homelab NAS with a "
        "persistent volume for data and a backup job."
    )
    _seed_doc(
        db,
        path="container_gitlab/gitlab-compose.yaml",
        title="gitlab-compose",
        content="services:\n  gitlab:\n    image: gitlab/gitlab-ce",
        summary=summary,
    )
    try:
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "container_gitlab/gitlab-compose.yaml"},
        )
        assert r.status_code == 200
        body = r.json()
        assert body["summary"] == summary  # verbatim, no wrapping
        assert body["content"] == "services:\n  gitlab:\n    image: gitlab/gitlab-ce"
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_content_summary_null_for_markdown_doc(client, db) -> None:
    """Markdown documents carry no summary (phase 30) → JSON ``null``
    (signed-in caller — phase 79 gated the viewer)."""
    _seed_doc(
        db,
        path="kubernetes.md",
        content="# Kubernetes\n\nTalos on 3 nodes.",
        summary=None,
    )
    try:
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "kubernetes.md"},
        )
        assert r.status_code == 200
        body = r.json()
        assert "summary" in body
        assert body["summary"] is None
        assert body["content"] == "# Kubernetes\n\nTalos on 3 nodes."
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_content_404_unknown_path(client, db) -> None:
    _seed_doc(db)
    try:
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "nope/missing.md"},
        )
        assert r.status_code == 404
        assert r.json() == {"detail": "document not found"}
        # A pair that exists under a DIFFERENT source is also 404 — both
        # values must match the row.
        r = client.get(
            "/api/documents/content",
            params={"source": "Deployments", "path": "kubernetes.md"},
        )
        assert r.status_code == 404
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_content_404_traversal_style_path_no_leak(client, db) -> None:
    """DB-only lookup: traversal strings are just non-existent rows — 404,
    and the response must not carry anything from the filesystem."""
    _seed_doc(db)
    try:
        for path in ("../../etc/passwd", "../kubernetes.md", "..%2F..%2Fetc%2Fpasswd"):
            r = client.get(
                "/api/documents/content",
                params={"source": "Homelab", "path": path},
            )
            assert r.status_code == 404, path
            assert r.json() == {"detail": "document not found"}, path
            assert "root:" not in r.text, path
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_content_format_from_suffix(client, db) -> None:
    """format = lowercased path suffix: yaml documents (phase 09 corpus) and
    the no-suffix fallback both flow through the same endpoint."""
    try:
        _seed_doc(
            db,
            path="container_gitlab/gitlab-compose.yaml",
            title="gitlab-compose",
            content="services:\n  gitlab:\n    image: gitlab/gitlab-ce",
            chunks=0,
        )
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "container_gitlab/gitlab-compose.yaml"},
        )
        assert r.status_code == 200
        body = r.json()
        assert body["format"] == "yaml"
        assert body["chunks"] == 0  # outerjoin → zero, not missing

        _seed_doc(db, path="README", title="README", content="plain text, no suffix", chunks=0)
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "README"},
        )
        assert r.status_code == 200
        assert r.json()["format"] == "text"  # no-suffix fallback
    finally:
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()


def test_content_format_name_token_extensionless(
    client, db, monkeypatch
) -> None:
    """Phase 102: the endpoint's ``format`` carries the name token for an
    extensionless document whose lowercased full filename is a
    ``BOR_IMPORT_EXTENSIONS`` token (``Dockerfile`` → ``dockerfile``),
    ``text`` for an extensionless name that is NOT configured, and every
    suffixed value is unchanged (display never depends on the import
    list — ``notes/README.dev`` badges ``dev`` even though ``dev`` is not
    a configured token). The endpoint reads the real ``get_settings()``,
    so the token lands via the house env-override pattern (the default
    list does not contain ``dockerfile``)."""
    get_settings.cache_clear()
    try:
        monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,dockerfile")

        _seed_doc(
            db,
            path="services/api/Dockerfile",
            title="Dockerfile",
            content="FROM alpine\nCMD [\"/bin/sh\"]",
            chunks=0,
        )
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "services/api/Dockerfile"},
        )
        assert r.status_code == 200
        assert r.json()["format"] == "dockerfile"

        # Suffix precedence under the same override — the import list is
        # ignored for suffixed paths.
        _seed_doc(
            db,
            path="notes/README.dev",
            title="README.dev",
            content="dev note",
            chunks=0,
        )
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "notes/README.dev"},
        )
        assert r.status_code == 200
        assert r.json()["format"] == "dev"

        # Extensionless name NOT in the configured set → the ``text``
        # fallback (the existing README pin above covers the same case
        # under the default settings).
        _seed_doc(
            db,
            path="Containerfile",
            title="Containerfile",
            content="FROM alpine",
            chunks=0,
        )
        r = client.get(
            "/api/documents/content",
            params={"source": "Homelab", "path": "Containerfile"},
        )
        assert r.status_code == 200
        assert r.json()["format"] == "text"
    finally:
        get_settings.cache_clear()
        db.execute(text("TRUNCATE chunks, documents"))
        db.commit()