brain-of-reese/tests/integration/test_migration_0025.py

"""Integration: migration 0025 (documents.file_hash) schema contract
(phase 147, task 01).

Drives the **real Alembic engine** against the live dev database
(``podman compose up -d db``), mirroring the house pattern of
``test_migration_0024.py`` (information_schema assertions on the state
the migration must leave). The tests target the 0024 → 0025 step
explicitly so later migrations cannot break the pins:

* upgrade 0024 → 0025 → ``file_hash`` exists with the full contract —
  VARCHAR(64), NULLABLE, NO server default (NULL = "unknown" for
  pre-phase rows; a migration cannot backfill — it cannot see the
  source files on disk) — and is UNINDEXED (the fast path compares it
  in Python after the unique ``(source, path)`` lookup — internal
  bookkeeping, never a query surface) while the 0024 ``documents``
  schema survives — in particular ``content_hash`` stays NOT NULL and
  INDEXED (the retriever's dedupe/caching identity, untouched);
* rows inserted while the DB is at 0024 backfill ``file_hash`` to NULL
  — the pre-phase shape, no backfill, no default;
* the ORM contract agrees: a freshly inserted ``Document`` with no
  ``file_hash`` reads ``file_hash is None`` through a fresh session
  (the pre-phase shape), and an explicit digest round-trips;
* downgrade to 0024 → the column is GONE (A13) while the row + its
  content survive; upgrade back to 0025 → the column is back
  (round-trip).

The ``alembic`` fixture guarantees the DB ends at head even if a test
fails or the process is interrupted.
"""
from __future__ import annotations

import hashlib
import uuid
from collections.abc import Iterator
from typing import Any

import pytest
from alembic.config import Config
from sqlalchemy import text
from sqlalchemy.orm import Session

from alembic import command
from app.db import SessionLocal, db_available
from app.models import Document

SOURCE = "mig0025"
CONTENT = "content here"
CONTENT_HASH = hashlib.sha256(CONTENT.encode()).hexdigest()
FILE_HASH = hashlib.sha256(b"raw source bytes").hexdigest()


@pytest.fixture()
def alembic(db: Session) -> Iterator[Config]:
    """Real Alembic config bound to the dev DB (URL from app settings).

    Starts at head (repairs an interrupted earlier run); teardown
    upgrades to head no matter what happened, so the dev DB is never
    left below head.
    """
    if not db_available():
        pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
    cfg = Config()  # no alembic.ini file — env.py gets the URL from app config
    cfg.set_main_option("script_location", "alembic")
    command.upgrade(cfg, "head")
    try:
        yield cfg
    finally:
        # Release the test session's open transaction BEFORE the repair
        # DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
        # on ``documents``, which would deadlock the repair's
        # ``ALTER TABLE`` (0025) forever.
        db.rollback()
        command.upgrade(cfg, "head")


def _version(db: Session) -> str | None:
    return db.execute(text("SELECT version_num FROM alembic_version")).scalar()


def _column(db: Session, column: str) -> tuple[Any, ...] | None:
    """(data_type, is_nullable, column_default, char_max_length) for one
    documents column."""
    row = db.execute(
        text(
            "SELECT data_type, is_nullable, column_default, character_maximum_length"
            " FROM information_schema.columns"
            " WHERE table_name = 'documents' AND column_name = :c"
        ),
        {"c": column},
    ).fetchone()
    return tuple(row) if row is not None else None


def _index_covering(db: Session, column: str) -> str | None:
    """The name of any documents index whose definition mentions the
    column (None = UNINDEXED)."""
    return db.execute(
        text(
            "SELECT indexname FROM pg_indexes"
            " WHERE tablename = 'documents' AND indexdef ILIKE :like"
        ),
        {"like": f"%{column}%"},
    ).scalar()


def _insert_sql(db: Session, path: str) -> uuid.UUID:
    """Insert one documents row with the PRE-0025 column set (the 0024
    shape — the file_hash column, when present, is omitted so a NULL
    backfill is what the row reads)."""
    doc_id = uuid.uuid4()
    db.execute(
        text(
            "INSERT INTO documents (id, source, path, full_path, title, content,"
            " content_hash) VALUES (:id, :s, :p, :fp, :t, :c, :h)"
        ),
        {
            "id": doc_id,
            "s": SOURCE,
            "p": path,
            "fp": f"/tmp/{path}",
            "t": f"Doc {path}",
            "c": CONTENT,
            "h": CONTENT_HASH,
        },
    )
    db.commit()
    return doc_id


def _delete(db: Session, doc_id: uuid.UUID) -> None:
    db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id})
    db.commit()


def _assert_0024_schema_survives(db: Session) -> None:
    """The 0024 documents contract must survive the additive upgrade —
    in particular ``content_hash`` stays NOT NULL and indexed (the
    retriever's identity — untouched by phase 147)."""
    content = _column(db, "content_hash")
    assert content is not None and content[0] == "character varying" and content[1] == "NO", (
        "documents.content_hash (0001) must keep its 0024 contract"
    )
    assert content[3] == 64, "documents.content_hash must keep String(64)"
    assert (
        _index_covering(db, "content_hash") is not None
    ), "the ix_documents_content_hash index must survive the upgrade"

    indexed = _column(db, "indexed_at")
    assert indexed is not None and indexed[0] == "timestamp with time zone", (
        "documents.indexed_at (0001) must survive the upgrade"
    )
    created = _column(db, "created_at")
    assert created is not None and created[1] == "NO", (
        "documents.created_at (0020) must survive the upgrade"
    )
    manual = _column(db, "created_at_manual")
    assert manual is not None and manual[0] == "boolean" and manual[1] == "NO", (
        "documents.created_at_manual (0020) must survive the upgrade"
    )
    is_image = _column(db, "is_image")
    assert is_image is not None and is_image[0] == "boolean" and is_image[1] == "NO", (
        "documents.is_image (0022) must survive the upgrade"
    )
    image_path = _column(db, "image_path")
    assert image_path is not None and image_path[1] == "YES", (
        "documents.image_path (0022) must survive the upgrade"
    )
    image_ctx = _column(db, "image_context")
    assert image_ctx is not None and image_ctx[1] == "YES", (
        "documents.image_context (0023) must survive the upgrade"
    )
    # Match by definition, not name: the dev DB's ``documents`` indexes
    # carry a one-off repair prefix (``documents_heal_new_…_*`` — a
    # manual dev-DB heal, never a migration), so the name itself is not
    # a stable pin.
    uq = db.execute(
        text(
            "SELECT indexname FROM pg_indexes"
            " WHERE tablename = 'documents'"
            " AND indexdef LIKE 'CREATE UNIQUE INDEX %USING btree (source, path)'",
        )
    ).scalar()
    assert uq is not None, (
        "the unique (source, path) index must survive the upgrade"
    )


def test_upgrade_to_0025_adds_file_hash(db: Session, alembic: Config) -> None:
    """Upgrade 0024 → 0025: ``file_hash`` exists with the full contract
    (VARCHAR(64), NULLABLE, NO server default — NULL = "unknown" for
    pre-phase rows — and UNINDEXED: the fast path compares it in Python
    after the unique ``(source, path)`` lookup), is ABSENT at 0024, and
    every pre-0025 row backfills ``file_hash`` to NULL (no backfill
    possible — a migration cannot see the source files on disk) — while
    the 0024 table contract (incl. the INDEXED ``content_hash``)
    survives."""
    command.downgrade(alembic, "0024")  # start from the pre-0025 state
    assert _version(db) == "0024"
    assert _column(db, "file_hash") is None, "file_hash must be absent at 0024"

    pre_id = _insert_sql(db, "pre.md")  # the 0024 column set
    try:
        command.upgrade(alembic, "0025")
        assert _version(db) == "0025", "alembic_version must be at 0025"

        col = _column(db, "file_hash")
        assert col is not None, "documents.file_hash is missing"
        assert col[0] == "character varying", "file_hash must be VARCHAR"
        assert col[1] == "YES", "file_hash must be NULLABLE (NULL = unknown)"
        assert col[2] is None, (
            "file_hash must carry NO server default — NULL is the meaningful"
            " pre-phase state"
        )
        assert col[3] == 64, "file_hash must be VARCHAR(64) — a sha256 hex digest"
        assert _index_covering(db, "file_hash") is None, (
            "file_hash must be UNINDEXED — it is compared in Python after the"
            " unique (source, path) lookup, never a query surface"
        )

        # The pre-0025 row backfilled to NULL (the pre-phase shape — one
        # full pipeline pass stamps it on the next sync).
        assert db.execute(
            text("SELECT content_hash, file_hash FROM documents WHERE id = :id"),
            {"id": pre_id},
        ).fetchone() == (CONTENT_HASH, None), (
            "every pre-0025 row must backfill file_hash to NULL — no"
            " backfill, no default — while content_hash survives"
        )

        _assert_0024_schema_survives(db)
    finally:
        _delete(db, pre_id)


def test_orm_file_hash_round_trips(db: Session, alembic: Config) -> None:
    """The ORM contract agrees with the column contract: a freshly
    inserted ``Document`` with NO ``file_hash`` reads ``file_hash is
    None`` through a FRESH session (the pre-phase shape — NULL until the
    first post-upgrade sync stamps it), and an explicit digest
    round-trips the value."""
    command.upgrade(alembic, "head")
    row_stamped = Document(
        source=SOURCE,
        path="stamped.md",
        full_path="/tmp/stamped.md",
        title="stamped",
        content=CONTENT,
        content_hash=CONTENT_HASH,
        file_hash=FILE_HASH,
    )
    row_unstamped = Document(
        source=SOURCE,
        path="unstamped.md",
        full_path="/tmp/unstamped.md",
        title="unstamped",
        content=CONTENT,
        content_hash=CONTENT_HASH,
    )
    db.add(row_stamped)
    db.add(row_unstamped)
    db.commit()
    try:
        with SessionLocal() as fresh:
            reloaded_stamped = fresh.get(Document, row_stamped.id)
            assert reloaded_stamped is not None, "the stamped row must be readable"
            assert reloaded_stamped.file_hash == FILE_HASH, (
                "the explicit file_hash must round-trip through the DB"
            )
            reloaded_unstamped = fresh.get(Document, row_unstamped.id)
            assert reloaded_unstamped is not None, "the unstamped row must be readable"
            assert reloaded_unstamped.file_hash is None, (
                "a row created without file_hash must read NULL (the pre-phase"
                " shape)"
            )
            assert reloaded_unstamped.content_hash == CONTENT_HASH, (
                "content_hash must be untouched by the file_hash addition"
            )
    finally:
        _delete(db, row_stamped.id)
        _delete(db, row_unstamped.id)


def test_downgrade_to_0024_drops_the_column(db: Session, alembic: Config) -> None:
    """Downgrade 0025 → 0024: the file_hash column is gone (A13 — fully
    reversible, the hash is re-computable from the source files) while
    the row + its content (incl. the stamped file_hash value's sibling
    ``content_hash``) survive, and the rest of the 0024 table contract
    is intact."""
    command.upgrade(alembic, "head")
    row = Document(
        source=SOURCE,
        path="down.md",
        full_path="/tmp/down.md",
        title="down",
        content=CONTENT,
        content_hash=CONTENT_HASH,
        file_hash=FILE_HASH,
    )
    db.add(row)
    db.commit()
    try:
        command.downgrade(alembic, "0024")
        assert _version(db) == "0024"
        assert _column(db, "file_hash") is None, "file_hash must be dropped"

        surviving = db.execute(
            text(
                "SELECT source, path, content, content_hash, is_image, image_path,"
                " image_context FROM documents WHERE id = :id"
            ),
            {"id": row.id},
        ).fetchone()
        assert surviving is not None and surviving[0] == SOURCE, (
            "the row must survive the column drop"
        )
        assert surviving[1] == "down.md" and surviving[2] == CONTENT, (
            "path + content must survive the drop"
        )
        assert surviving[3] == CONTENT_HASH, (
            "content_hash (the retriever's identity) must survive the drop"
        )
        assert surviving[4] is False and surviving[5] is None and surviving[6] is None, (
            "is_image + image_path + image_context must survive the drop"
        )
        assert _index_covering(db, "content_hash") is not None, (
            "the content_hash index must survive the downgrade"
        )
    finally:
        _delete(db, row.id)
    # Repair: the fixture teardown re-upgrades to head.


def test_upgrade_round_trip_restores_the_column(db: Session, alembic: Config) -> None:
    """Downgrade to 0024, then upgrade back to 0025: ``file_hash`` is
    back with the full contract (VARCHAR(64), NULLABLE, NO server
    default, UNINDEXED), a row seeded at 0024 reads ``file_hash is
    None`` (the pre-phase shape on the round-trip), and the 0024 schema
    (incl. the indexed ``content_hash``) is intact."""
    command.downgrade(alembic, "0024")
    pre_id = _insert_sql(db, "round.md")
    try:
        command.upgrade(alembic, "0025")
        assert _version(db) == "0025", "round-trip upgrade must land at 0025"

        col = _column(db, "file_hash")
        assert col is not None, "documents.file_hash must be back"
        assert col[0] == "character varying", "file_hash must be VARCHAR after the round-trip"
        assert col[1] == "YES", "file_hash must be NULLABLE after the round-trip"
        assert col[2] is None, (
            "file_hash must still carry NO server default after the round-trip"
        )
        assert col[3] == 64, "file_hash must be VARCHAR(64) after the round-trip"
        assert _index_covering(db, "file_hash") is None, (
            "file_hash must stay UNINDEXED after the round-trip"
        )

        assert db.execute(
            text("SELECT file_hash FROM documents WHERE id = :id"),
            {"id": pre_id},
        ).scalar() is None, (
            "a row seeded at 0024 must read file_hash NULL after the round-trip"
        )

        _assert_0024_schema_survives(db)
    finally:
        _delete(db, pre_id)