brain-of-reese/tests/unit/test_agent.py

"""Unit: the grounded-turn agent loop (phase 37, ``app.rag.agent``).

A scripted fake LLM (canned stream sequences) + monkeypatched
``list_catalog`` / ``find_document`` — no database, no network. Covers
the loop mechanics: the list → read → answer happy path (event order,
holder state, the tools staying offered on every request — phase 45
removed the per-tool budgets, the assistant/tool message history), the
kill switch (``agent_max_rounds=0`` single-call path), the round cap
forcing a final no-tools answer (an always-calling stream and an
always-rejected stream), re-lists and multi-reads executing without
budgets, dedupe, unknown tool / missing args / unknown path, and the
``<tools>`` prompt section (HIGH only).
"""
from __future__ import annotations

import asyncio
import json
import uuid
from collections.abc import AsyncIterator
from copy import deepcopy
from typing import Any, cast

import pytest
from sqlalchemy.orm import Session

from app.config import Settings
from app.models import Document
from app.rag import agent
from app.rag.agent import (
    AGENT_TOOLS,
    AgentHolder,
    run_agent,
)
from app.rag.llm import LLMClient, StreamPiece, ToolCallPiece
from app.rag.prompts import TOOLS_SECTION, _base, build_deflect_prompt, build_high_prompt


def _settings(**kwargs: Any) -> Settings:
    kwargs.setdefault("_env_file", None)
    return Settings(**kwargs)  # pyright: ignore[reportCallIssue]


def _doc(source: str, path: str, title: str = "Title", content: str = "CONTENT") -> Document:
    return Document(
        id=uuid.uuid4(),
        source=source,
        path=path,
        full_path=f"/tmp/{path}",
        title=title,
        content=content,
        content_hash="0" * 64,
    )


class ScriptedLLM:
    """Canned stream sequences; records every ``chat_stream`` request so
    the tests can assert on the messages and the ``tools`` passthrough."""

    def __init__(self, *streams: list[StreamPiece | ToolCallPiece]) -> None:
        self.streams: list[list[StreamPiece | ToolCallPiece]] = list(streams)
        self.requests: list[tuple[list[dict[str, Any]], list[dict[str, Any]] | None]] = []

    async def chat_stream(
        self,
        messages: list[dict[str, str]],
        tools: list[dict[str, Any]] | None = None,
    ) -> AsyncIterator[StreamPiece | ToolCallPiece]:
        self.requests.append((deepcopy(messages), tools))
        if not self.streams:
            raise AssertionError("ScriptedLLM ran out of canned streams")
        for piece in self.streams.pop(0):
            yield piece


async def _run(
    llm: ScriptedLLM,
    holder: AgentHolder,
    settings: Settings,
    seed_docs: list[Document] | None = None,
) -> list[StreamPiece | ToolCallPiece]:
    out: list[StreamPiece | ToolCallPiece] = []
    async for piece in run_agent(
        cast("LLMClient", llm),
        cast("Session", None),
        system_prompt="SYSTEM_PROMPT",
        user_message="QUESTION",
        seed_docs=seed_docs or [],
        settings=settings,
        holder=holder,
    ):
        out.append(piece)
    return out


# ---------- AGENT_TOOLS shape ----------


def test_agent_tools_names_and_parameters() -> None:
    by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
    assert set(by_name) == {"list_documents", "read_document"}
    assert all(t["type"] == "function" for t in AGENT_TOOLS)
    list_params = by_name["list_documents"]["function"]["parameters"]
    assert list_params["type"] == "object"
    assert list_params["properties"] == {}  # no parameters
    read_params = by_name["read_document"]["function"]["parameters"]
    assert read_params["required"] == ["source", "path"]
    assert set(read_params["properties"]) == {"source", "path"}
    # Phase 45: the per-tool budgets are gone — "exactly one more"
    # dropped out of the read_document description.
    assert by_name["read_document"]["function"]["description"] == (
        "Add the full content of one more indexed document to your context"
    )
    # Phase 63 (A2): the parameter descriptions point the LLM at the
    # labeled `source:` / `path:` fields of the list_documents output.
    assert read_params["properties"]["source"]["description"] == (
        "The document's source, as shown after 'source: ' in the "
        "list_documents output (e.g. 'Homelab' from "
        "'source: Homelab | path: homelab/aws-route53.md')."
    )
    assert read_params["properties"]["path"]["description"] == (
        "The document's path, as shown after 'path: ' in the "
        "list_documents output (e.g. 'homelab/aws-route53.md' from "
        "'source: Homelab | path: homelab/aws-route53.md')."
    )


# ---------- happy path: list → read → answer ----------


def test_list_then_read_then_answer(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    catalog = [
        ("Deployments", "backups.md", "Backup Strategy"),
        ("Homelab", "aws-route53.md", "AWS Route53 Records"),
    ]
    monkeypatch.setattr(agent, "list_catalog", lambda db: catalog)
    target = _doc("Homelab", "aws-route53.md", "AWS Route53 Records", "R53-CONTENT")
    monkeypatch.setattr(agent, "find_document", lambda db, source, path: target)
    seed = [_doc("Homelab", "kubernetes.md", "Kubernetes", "K8S-CONTENT")]
    holder = AgentHolder()
    llm = ScriptedLLM(
        [ToolCallPiece(id="call_1", name="list_documents", arguments={})],
        [
            ToolCallPiece(
                id="call_2",
                name="read_document",
                arguments={"source": "Homelab", "path": "aws-route53.md"},
            )
        ],
        [StreamPiece("thinking", "hmm "), StreamPiece("content", "Done! ")],
    )

    pieces = asyncio.run(_run(llm, holder, _settings(), seed_docs=seed))

    # Event order: tool pieces before the answer content/thinking.
    assert [type(p) for p in pieces] == [
        ToolCallPiece,
        ToolCallPiece,
        StreamPiece,
        StreamPiece,
    ]
    assert pieces[0] == ToolCallPiece(id="call_1", name="list_documents", arguments={})
    assert isinstance(pieces[1], ToolCallPiece)
    assert pieces[1].name == "read_document"
    assert pieces[3] == StreamPiece("content", "Done! ")
    # The read document is recorded for done.sources / query_log (task 04).
    assert holder.read_docs == [target]
    assert holder.tool_calls == 2

    # Phase 45: no per-tool budgets — the tools stay offered on every
    # request (the round cap, not spent budgets, bounds the loop), so
    # the answer request still carries them (2 rounds < default cap 10).
    assert llm.requests[0][1] == AGENT_TOOLS
    assert llm.requests[1][1] == AGENT_TOOLS
    assert llm.requests[2][1] == AGENT_TOOLS
    assert len(llm.requests) == 3

    # The follow-up request carries the assistant tool-call + tool result.
    msgs = llm.requests[1][0]
    assert msgs[0] == {"role": "system", "content": "SYSTEM_PROMPT"}
    assert msgs[1] == {"role": "user", "content": "QUESTION"}
    assert msgs[2] == {
        "role": "assistant",
        "content": None,
        "tool_calls": [
            {
                "id": "call_1",
                "type": "function",
                "function": {"name": "list_documents", "arguments": "{}"},
            }
        ],
    }
    assert msgs[3] == {
        "role": "tool",
        "tool_call_id": "call_1",
        "content": (
            "2 documents:\n"
            "source: Deployments | path: backups.md | title: Backup Strategy\n"
            "source: Homelab | path: aws-route53.md | title: AWS Route53 Records"
        ),
    }
    # The second follow-up request carries the read call + the FULL text.
    msgs = llm.requests[2][0]
    assert msgs[4]["role"] == "assistant"
    assert msgs[4]["tool_calls"][0]["id"] == "call_2"
    assert json.loads(msgs[4]["tool_calls"][0]["function"]["arguments"]) == {
        "source": "Homelab",
        "path": "aws-route53.md",
    }
    assert msgs[5] == {
        "role": "tool",
        "tool_call_id": "call_2",
        "content": "Document Homelab/aws-route53.md:\nR53-CONTENT",  # full text, no cap
    }


def test_empty_catalog_listing_says_zero_documents(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(agent, "list_catalog", lambda db: [])
    holder = AgentHolder()
    llm = ScriptedLLM(
        [ToolCallPiece(id="call_1", name="list_documents", arguments={})],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings()))
    assert llm.requests[1][0][3]["content"] == "0 documents:\n"
    assert holder.tool_calls == 1


def test_content_and_tool_call_in_one_stream_keeps_both(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Rare stream with content AND a tool call: the content stays (it was
    already emitted) and the tool still runs."""
    monkeypatch.setattr(agent, "list_catalog", lambda db: [])
    holder = AgentHolder()
    llm = ScriptedLLM(
        [
            StreamPiece("content", "Let me check "),
            ToolCallPiece(id="call_1", name="list_documents", arguments={}),
        ],
        [StreamPiece("content", "the answer")],
    )
    pieces = asyncio.run(_run(llm, holder, _settings()))
    assert [type(p) for p in pieces] == [StreamPiece, ToolCallPiece, StreamPiece]
    assert holder.tool_calls == 1  # the tool ran despite the content
    assert llm.requests[1][0][3]["content"] == "0 documents:\n"


# ---------- round cap (phase 45: replaces the per-tool budgets) ----------


def test_always_list_bounded_by_round_cap(monkeypatch: pytest.MonkeyPatch) -> None:
    """A model that keeps calling ``list_documents`` gets exactly
    ``agent_max_rounds`` tool rounds, then one forced ``tools=None``
    request streams the answer — the cap is the only forced exit."""
    monkeypatch.setattr(agent, "list_catalog", lambda db: [("S", "a.md", "A")])
    listing = "1 documents:\nsource: S | path: a.md | title: A"
    holder = AgentHolder()
    llm = ScriptedLLM(
        [ToolCallPiece(id="call_1", name="list_documents", arguments={})],
        [ToolCallPiece(id="call_2", name="list_documents", arguments={})],
        [ToolCallPiece(id="call_3", name="list_documents", arguments={})],
        [StreamPiece("content", "forced answer")],
    )
    pieces = asyncio.run(_run(llm, holder, _settings(agent_max_rounds=3)))
    assert [type(p) for p in pieces] == [
        ToolCallPiece,
        ToolCallPiece,
        ToolCallPiece,
        StreamPiece,
    ]
    assert len(llm.requests) == 4  # 3 tool rounds + the forced answer
    # The three tool rounds were offered the tools…
    assert llm.requests[0][1] == AGENT_TOOLS
    assert llm.requests[1][1] == AGENT_TOOLS
    assert llm.requests[2][1] == AGENT_TOOLS
    # …and the forced final request carries no tools, whatever is left.
    assert llm.requests[3][1] is None
    # Every re-list executed and counted.
    assert holder.tool_calls == 3
    # The final request carries all three executed listings as history.
    final_msgs = llm.requests[3][0]
    assert len(final_msgs) == 8  # 2 + 3 rounds × (assistant + tool)
    assert final_msgs[3]["content"] == listing
    assert final_msgs[5]["content"] == listing
    assert final_msgs[7]["content"] == listing


def test_zero_max_rounds_is_one_request_without_tools() -> None:
    """``agent_max_rounds=0`` — the kill switch: exactly one request,
    ``tools=None``, no tool lines, no history growth (byte-identical to
    the pre-phase-37 path)."""
    holder = AgentHolder()
    llm = ScriptedLLM([StreamPiece("thinking", "t "), StreamPiece("content", "direct answer")])
    pieces = asyncio.run(_run(llm, holder, _settings(agent_max_rounds=0)))
    assert [type(p) for p in pieces] == [StreamPiece, StreamPiece]
    assert len(llm.requests) == 1
    assert llm.requests[0][1] is None
    assert llm.requests[0][0] == [
        {"role": "system", "content": "SYSTEM_PROMPT"},
        {"role": "user", "content": "QUESTION"},
    ]
    assert holder.read_docs == [] and holder.tool_calls == 0


def test_rejected_read_spam_runs_to_round_cap(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Every call rejected (unknown path — "No document at …"): rejections
    no longer end the loop early via budgets — the round cap bounds them
    and forces the final no-tools answer."""
    monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
    holder = AgentHolder()
    llm = ScriptedLLM(
        [
            ToolCallPiece(
                id="call_1",
                name="read_document",
                arguments={"source": "S", "path": "ghost.md"},
            )
        ],
        [
            ToolCallPiece(
                id="call_2",
                name="read_document",
                arguments={"source": "S", "path": "ghost.md"},
            )
        ],
        [
            ToolCallPiece(
                id="call_3",
                name="read_document",
                arguments={"source": "S", "path": "ghost.md"},
            )
        ],
        [StreamPiece("content", "forced answer")],
    )
    asyncio.run(_run(llm, holder, _settings(agent_max_rounds=3)))
    assert len(llm.requests) == 4  # 3 rejected rounds + the forced answer
    assert llm.requests[0][1] == AGENT_TOOLS
    assert llm.requests[1][1] == AGENT_TOOLS
    assert llm.requests[2][1] == AGENT_TOOLS
    assert llm.requests[3][1] is None  # the forced final request: no tools
    assert holder.read_docs == [] and holder.tool_calls == 0  # nothing executed
    refusal = "No document at S/ghost.md — check the list_documents output."
    assert llm.requests[1][0][3]["content"] == refusal
    assert llm.requests[2][0][5]["content"] == refusal
    assert llm.requests[3][0][7]["content"] == refusal


# ---------- unlimited calls: re-lists and multi-reads (phase 45) ----------


def test_relist_executes_and_counts(monkeypatch: pytest.MonkeyPatch) -> None:
    """Re-lists execute — a second ``list_documents`` in one turn returns
    the catalog again and counts in ``tool_calls`` (no budget to
    exhaust)."""
    catalog = [
        ("Deployments", "backups.md", "Backup Strategy"),
        ("Homelab", "aws-route53.md", "AWS Route53 Records"),
    ]
    monkeypatch.setattr(agent, "list_catalog", lambda db: catalog)
    holder = AgentHolder()
    llm = ScriptedLLM(
        [ToolCallPiece(id="call_1", name="list_documents", arguments={})],
        [ToolCallPiece(id="call_2", name="list_documents", arguments={})],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings()))
    assert holder.tool_calls == 2  # both re-lists executed and counted
    listing = (
        "2 documents:\n"
        "source: Deployments | path: backups.md | title: Backup Strategy\n"
        "source: Homelab | path: aws-route53.md | title: AWS Route53 Records"
    )
    # The answer request carries the catalog a second time as a tool result.
    assert llm.requests[2][0][3]["content"] == listing  # first listing
    assert llm.requests[2][0][5]["content"] == listing  # the re-list
    assert llm.requests[2][1] == AGENT_TOOLS  # still offered (no budgets)


def test_multi_read_executes_without_budgets(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Reads are no longer budgeted either — two different documents can
    be read in one turn (re-reading the same one is still deduped via
    ALREADY_IN_CONTEXT — see the rejection tests)."""
    a = _doc("S", "a.md", "A", "A-CONTENT")
    b = _doc("S", "b.md", "B", "B-CONTENT")
    monkeypatch.setattr(
        agent, "find_document", lambda db, source, path: {"a.md": a, "b.md": b}[path]
    )
    holder = AgentHolder()
    llm = ScriptedLLM(
        [
            ToolCallPiece(
                id="call_1", name="read_document", arguments={"source": "S", "path": "a.md"}
            )
        ],
        [
            ToolCallPiece(
                id="call_2", name="read_document", arguments={"source": "S", "path": "b.md"}
            )
        ],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings()))
    assert holder.read_docs == [a, b]  # both reads appended, in order
    assert holder.tool_calls == 2
    assert llm.requests[1][0][3]["content"] == "Document S/a.md:\nA-CONTENT"
    assert llm.requests[2][0][5]["content"] == "Document S/b.md:\nB-CONTENT"
    assert llm.requests[2][1] == AGENT_TOOLS  # the second read was still offered


# ---------- rejections (non-budget; the round cap bounds their repetition) ----------


def test_reading_a_seed_doc_is_already_in_context(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    seed = [_doc("Homelab", "kubernetes.md", "Kubernetes", "K8S-CONTENT")]

    def _boom(*_a: Any, **_k: Any) -> None:
        raise AssertionError("find_document must not be called for a seeded doc")

    monkeypatch.setattr(agent, "list_catalog", lambda db: [])
    monkeypatch.setattr(agent, "find_document", _boom)
    holder = AgentHolder()
    llm = ScriptedLLM(
        [
            ToolCallPiece(
                id="call_1",
                name="read_document",
                arguments={"source": "Homelab", "path": "kubernetes.md"},
            )
        ],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings(), seed_docs=seed))
    assert holder.read_docs == [] and holder.tool_calls == 0
    assert llm.requests[1][0][3]["content"] == agent.ALREADY_IN_CONTEXT
    # Rejected → the tools are still offered on the next request (the
    # round cap is the only bound).
    assert llm.requests[1][1] == AGENT_TOOLS


def test_reading_an_already_read_doc_is_deduped(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    doc = _doc("S", "a.md", "A", "A-CONTENT")
    monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
    holder = AgentHolder()
    llm = ScriptedLLM(
        [
            ToolCallPiece(
                id="call_1", name="read_document", arguments={"source": "S", "path": "a.md"}
            )
        ],
        [
            ToolCallPiece(
                id="call_2", name="read_document", arguments={"source": "S", "path": "a.md"}
            )
        ],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings()))
    assert holder.read_docs == [doc]  # appended exactly once
    assert holder.tool_calls == 1
    assert llm.requests[2][0][5]["content"] == agent.ALREADY_IN_CONTEXT
    # Rejected → the tools are still offered on the next request…
    assert llm.requests[2][1] == AGENT_TOOLS


def test_unknown_path_refused(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
    holder = AgentHolder()
    llm = ScriptedLLM(
        [
            ToolCallPiece(
                id="call_1",
                name="read_document",
                arguments={"source": "S", "path": "ghost.md"},
            )
        ],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings()))
    assert holder.read_docs == [] and holder.tool_calls == 0
    assert (
        llm.requests[1][0][3]["content"]
        == "No document at S/ghost.md — check the list_documents output."
    )
    assert llm.requests[1][1] == AGENT_TOOLS  # tools stay offered (cap bounds)


def test_unknown_tool_name_refused(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(agent, "list_catalog", lambda db: [])
    holder = AgentHolder()
    llm = ScriptedLLM(
        [ToolCallPiece(id="call_1", name="delete_universe", arguments={"x": 1})],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings()))
    assert holder.read_docs == [] and holder.tool_calls == 0
    assert llm.requests[1][0][3]["content"] == agent.UNKNOWN_TOOL
    assert llm.requests[1][1] == AGENT_TOOLS  # rejected → tools stay offered


@pytest.mark.parametrize(
    ("arguments", "label"),
    [
        ({}, "no arguments"),
        ({"source": "S"}, "path missing"),
        ({"path": "p.md"}, "source missing"),
        ({"source": "", "path": "p.md"}, "empty source"),
        ({"source": "S", "path": "  "}, "blank path"),
        ({"source": 7, "path": "p.md"}, "non-string source"),
    ],
)
def test_read_document_missing_arguments_refused(
    monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str
) -> None:
    def _boom(*_a: Any, **_k: Any) -> None:
        raise AssertionError(f"find_document must not be called ({label})")

    monkeypatch.setattr(agent, "list_catalog", lambda db: [])
    monkeypatch.setattr(agent, "find_document", _boom)
    holder = AgentHolder()
    llm = ScriptedLLM(
        [ToolCallPiece(id="call_1", name="read_document", arguments=arguments)],
        [StreamPiece("content", "ans")],
    )
    asyncio.run(_run(llm, holder, _settings()))
    assert holder.read_docs == [] and holder.tool_calls == 0
    assert llm.requests[1][0][3]["content"] == agent.MISSING_READ_ARGS
    assert llm.requests[1][1] == AGENT_TOOLS


# ---------- prompts: <tools> section (HIGH only) ----------


def test_high_prompt_carries_tools_section_after_documents() -> None:
    prompt = build_high_prompt([_doc("S", "a.md", "A", "A-CONTENT")])
    assert TOOLS_SECTION in prompt
    assert "call `list_documents`" in prompt
    assert "then `read_document` to pull in exactly one more document" in prompt
    assert "do not read more than one extra document" in prompt
    # After the mode body: <tools> follows </documents>.
    assert prompt.index("</documents>") < prompt.index("<tools>")
    assert prompt.rstrip().endswith("</tools>")


def test_high_prompt_tools_section_with_notes_and_kb() -> None:
    prompt = build_high_prompt(
        [_doc("S", "a.md", "A", "A-CONTENT")], notes=["be concise"], kb_overview="- KB"
    )
    assert prompt.index("<knowledge_base>") < prompt.index("<tuning>")
    assert prompt.index("<tuning>") < prompt.index("<documents>")
    assert prompt.index("<documents>") < prompt.index("<tools>")


def test_low_prompt_is_byte_identical_and_tool_free() -> None:
    expected = (
        _base("LOW")
        + "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
        "your notes come to the question. They are titles only; do not pretend "
        "they answer it. Use them to propose 2-3 alternative questions.\n"
        + "- T1\n- T2"
    )
    assert build_deflect_prompt(["T1", "T2"]) == expected
    for prompt in (
        build_deflect_prompt(["T1"]),
        build_deflect_prompt(["T1"], notes=["be concise"]),
        build_deflect_prompt(["T1"], kb_overview="- KB"),
        build_deflect_prompt(["T1"], notes=["be concise"], kb_overview="- KB"),
    ):
        assert "<tools>" not in prompt
        assert TOOLS_SECTION not in prompt