From fb17d15fbe5158c61782c6cbf85eaad979d27065 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Mon, 19 Oct 2026 12:19:51 -0400 Subject: [PATCH] phase: 192_verbatim_inline_midturn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 192 — final verification pass complete. - **Verified (all 4 tasks already complete):** additive `verbatim_block` SSE frame (`VerbatimBlockPiece` → `ChatVerbatimBlockEvent` → chat dispatch, after the round's `tool`/`tool_result`, before the next round); `hydrateVerbatimNow` matched-only mid-turn pass + app.js branch; `dropPlacedTwin` per-locator done/restore dedupe; the dedicated E2E suite. - **Root cause of the rejected attempt:** my own concurrent background `pytest` run deadlocked the harness validation run on the shared Postgres (`psycopg.errors.DeadlockDetected`). No code defect — re-ran every gate **exclusively**. - `uv run pytest -q --cov=app --cov-report=term-missing` → **exit 0**, `TOTAL 7181 16 99%` (>90%) - `uv run pytest tests/e2e/test_verbatim_inline_midturn.py --no-cov` → **6 passed** (isolated) - `uv run pytest tests/e2e/test_verbatim_show_text.py tests/e2e/test_chat_history.py --no-cov` → **18 passed** (`test_chat_history.py` byte-unedited; the phase-182 suite's "no eighth type" pin was tightened, not loosened, to seven + `verbatim_block` with frame-order asserts) - `uv run ruff check .` → All checks passed (exit 0) · `uv run pyright` → **0 errors** - **Completion criteria:** single frame per success + zero per refusal ✅ · `BOR_ANSWER_TEXT=0` wire byte-identical (integration spelled baseline + E2E App B) ✅ · fence hydrates in place / unfenced lands at the call moment, before `done` ✅ · append+fence → 1 panel, two ranges → 2 panels ✅ · restore 1 panel/locator, `done.answer_text` + record key + `verbatim_blocks=N` unchanged ✅ · suite green, coverage 99%, lint/types clean ✅ · no-show_text turn byte-identical ✅ · commit → left to the harness (no git index/history commands run). - **Docs:** README reviewed line-by-line (2399 lines) + AGENTS.md in full; mechanically verified all 86 `BOR_*` settings, every `.env.example` var, table defaults, internal anchors (0 broken), ports 8123/8901, tree depth 1–8, pagination 25/1–100, 17 palette colors, `0029`/`0030` migrations, and every cited test file. Fixed 1 inconsistency: the verbatim "Parity" bullet claimed the live turn/reload/shared page share "the same three globals" — now names the settle-path trio and states that `hydrateVerbatimNow` is the ONE chat-only addition (a shared page has no stream). - **Notable:** `run_agent`'s four repeated piece unions were collapsed into the ONE exported `AgentPiece` alias, so a future piece type is spelled in one place; `test_frontend_edit_message.py`'s `runTurn` byte-pin was regenerated, not weakened. - **Next pending phase:** `193_sync_remaining_count`. --- .../192_verbatim_inline_midturn/00_phase.md | 0 .../01_verbatim_block_sse.md | 0 .../02_midturn_hydration_and_append.md | 0 .../03_done_and_restore_dedupe.md | 0 .../192_verbatim_inline_midturn/04_e2e.md | 0 ...2_verbatim_inline_midturn__00_phase.a1.err | 0 ...92_verbatim_inline_midturn__00_phase.a1.md | 1 + ...atim_inline_midturn__00_phase.a1.md.status | 1 + ...batim_inline_midturn__00_phase.a1.validate | 13026 ++++++++++++++++ ...2_verbatim_inline_midturn__00_phase.a2.err | 0 ...92_verbatim_inline_midturn__00_phase.a2.md | 12 + ...atim_inline_midturn__00_phase.a2.md.status | 1 + ...batim_inline_midturn__00_phase.a2.validate | 150 + ...line_midturn__01_verbatim_block_sse.a1.err | 0 ...nline_midturn__01_verbatim_block_sse.a1.md | 19 + ...idturn__01_verbatim_block_sse.a1.md.status | 1 + ...midturn__01_verbatim_block_sse.a1.validate | 150 + ...rn__02_midturn_hydration_and_append.a1.err | 0 ...urn__02_midturn_hydration_and_append.a1.md | 19 + ..._midturn_hydration_and_append.a1.md.status | 1 + ...2_midturn_hydration_and_append.a1.validate | 150 + ...midturn__03_done_and_restore_dedupe.a1.err | 0 ..._midturn__03_done_and_restore_dedupe.a1.md | 18 + ...n__03_done_and_restore_dedupe.a1.md.status | 1 + ...rn__03_done_and_restore_dedupe.a1.validate | 150 + ...192_verbatim_inline_midturn__04_e2e.a1.err | 0 .../192_verbatim_inline_midturn__04_e2e.a1.md | 28 + ...rbatim_inline_midturn__04_e2e.a1.md.status | 1 + ...erbatim_inline_midturn__04_e2e.a1.validate | 150 + AGENTS.md | 33 +- README.md | 50 +- app/api/chat.py | 68 +- app/rag/agent.py | 91 +- app/schemas.py | 40 + frontend/assets/app.js | 24 + frontend/assets/verbatim.js | 121 +- tests/e2e/mock_llm.py | 195 + tests/e2e/test_verbatim_inline_midturn.py | 1296 ++ tests/e2e/test_verbatim_show_text.py | 34 +- tests/integration/test_agent_tools.py | 18 +- tests/integration/test_agent_tools_dates.py | 8 +- tests/integration/test_chat_api.py | 255 +- .../test_read_line_ranges_handbook.py | 15 +- .../test_read_range_dedupe_two_ranges.py | 15 +- tests/unit/test_agent.py | 263 +- tests/unit/test_agent_short_lived_sessions.py | 14 +- tests/unit/test_frontend_edit_message.py | 36 +- tests/unit/test_frontend_verbatim_blocks.py | 846 + tests/unit/test_schemas.py | 114 + 49 files changed, 17313 insertions(+), 102 deletions(-) rename .agents/phases/{todo => complete}/192_verbatim_inline_midturn/00_phase.md (100%) rename .agents/phases/{todo => complete}/192_verbatim_inline_midturn/01_verbatim_block_sse.md (100%) rename .agents/phases/{todo => complete}/192_verbatim_inline_midturn/02_midturn_hydration_and_append.md (100%) rename .agents/phases/{todo => complete}/192_verbatim_inline_midturn/03_done_and_restore_dedupe.md (100%) rename .agents/phases/{todo => complete}/192_verbatim_inline_midturn/04_e2e.md (100%) create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.err create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md.status create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.validate create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.err create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md.status create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.validate create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.err create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md.status create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.validate create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.err create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md.status create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.validate create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.err create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md.status create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.validate create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.err create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md.status create mode 100644 .agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.validate create mode 100644 tests/e2e/test_verbatim_inline_midturn.py diff --git a/.agents/phases/todo/192_verbatim_inline_midturn/00_phase.md b/.agents/phases/complete/192_verbatim_inline_midturn/00_phase.md similarity index 100% rename from .agents/phases/todo/192_verbatim_inline_midturn/00_phase.md rename to .agents/phases/complete/192_verbatim_inline_midturn/00_phase.md diff --git a/.agents/phases/todo/192_verbatim_inline_midturn/01_verbatim_block_sse.md b/.agents/phases/complete/192_verbatim_inline_midturn/01_verbatim_block_sse.md similarity index 100% rename from .agents/phases/todo/192_verbatim_inline_midturn/01_verbatim_block_sse.md rename to .agents/phases/complete/192_verbatim_inline_midturn/01_verbatim_block_sse.md diff --git a/.agents/phases/todo/192_verbatim_inline_midturn/02_midturn_hydration_and_append.md b/.agents/phases/complete/192_verbatim_inline_midturn/02_midturn_hydration_and_append.md similarity index 100% rename from .agents/phases/todo/192_verbatim_inline_midturn/02_midturn_hydration_and_append.md rename to .agents/phases/complete/192_verbatim_inline_midturn/02_midturn_hydration_and_append.md diff --git a/.agents/phases/todo/192_verbatim_inline_midturn/03_done_and_restore_dedupe.md b/.agents/phases/complete/192_verbatim_inline_midturn/03_done_and_restore_dedupe.md similarity index 100% rename from .agents/phases/todo/192_verbatim_inline_midturn/03_done_and_restore_dedupe.md rename to .agents/phases/complete/192_verbatim_inline_midturn/03_done_and_restore_dedupe.md diff --git a/.agents/phases/todo/192_verbatim_inline_midturn/04_e2e.md b/.agents/phases/complete/192_verbatim_inline_midturn/04_e2e.md similarity index 100% rename from .agents/phases/todo/192_verbatim_inline_midturn/04_e2e.md rename to .agents/phases/complete/192_verbatim_inline_midturn/04_e2e.md diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.err b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md new file mode 100644 index 0000000..aab6454 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md @@ -0,0 +1 @@ +All 4 tasks are complete. This is the final verification pass. Let me review the reports and completions. diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md.status b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.validate b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.validate new file mode 100644 index 0000000..ee6e813 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a1.validate @@ -0,0 +1,13026 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 19%] +........................................................................ [ 21%] +...........................F............................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +..........................................F..........F............F..... [ 39%] +........................................................F..FF........... [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 46%] +........................................................F..FE...FE.F...F [ 48%] +EFEFE........EEEEEE.E.................FFFFFF.F.FFF......F.F............. [ 49%] +........................................................................ [ 50%] +.......................................F......................F......... [ 52%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 97%] +........................................................................ [ 99%] +............................................... [100%] +==================================== ERRORS ==================================== +_ ERROR at teardown of test_document_list_reaching_the_model_leads_with_newest_docs _ + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('6adc3bd3-d044-478c-bc43-553053031f0c'), 'source': 's', 'path': '01_old_a.md', 'full_path': '/tmp/01_old_a.md', 'title': 'Phase 01', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +__ ERROR at teardown of test_regenerate_llm_error_leaves_previous_row_intact ___ + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('0dea1149-0bbd-4834-9fc8-c98e02aa435a'), 'source': 'Homelab', 'path': 'a.md', 'full_path': '/tmp/a.md', 'title': 'A', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': None, 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +_ ERROR at teardown of test_regenerate_model_input_never_gains_the_inventory_tail _ + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('008284d1-01ef-48ad-97db-c865cc9c7da3'), 'source': 'Deployments', 'path': 'b.yaml', 'full_path': '/tmp/b.yaml', 'title': 'B', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': None, 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +_ ERROR at teardown of test_regenerate_zero_docs_keeps_tailless_pre_phase_row __ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: DELETE FROM kb_overview] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +_ ERROR at teardown of test_regenerate_llm_error_preserves_an_existing_inventory_tail _ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: DELETE FROM kb_overview] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +____ ERROR at setup of test_regenerate_model_input_carries_the_display_name ____ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM source_labels', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedTable: relation "source_labels" does not exist +E LINE 1: DELETE FROM source_labels +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedTable + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_source_labels(db: Session): + """Empty ``source_labels`` before and after (migration-0030 state).""" +> db.execute(text("DELETE FROM source_labels")) + +tests/unit/test_overview.py:679: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM source_labels', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedTable) relation "source_labels" does not exist +E LINE 1: DELETE FROM source_labels +E ^ +E [SQL: DELETE FROM source_labels] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError +__ ERROR at teardown of test_regenerate_model_input_carries_the_display_name ___ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: DELETE FROM kb_overview] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +___ ERROR at setup of test_regenerate_stored_content_keeps_the_derived_names ___ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM source_labels', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedTable: relation "source_labels" does not exist +E LINE 1: DELETE FROM source_labels +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedTable + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_source_labels(db: Session): + """Empty ``source_labels`` before and after (migration-0030 state).""" +> db.execute(text("DELETE FROM source_labels")) + +tests/unit/test_overview.py:679: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM source_labels', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedTable) relation "source_labels" does not exist +E LINE 1: DELETE FROM source_labels +E ^ +E [SQL: DELETE FROM source_labels] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError +_ ERROR at teardown of test_regenerate_stored_content_keeps_the_derived_names __ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: DELETE FROM kb_overview] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +_ ERROR at setup of test_regenerate_without_label_rows_is_byte_identical_to_pre_phase _ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM source_labels', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedTable: relation "source_labels" does not exist +E LINE 1: DELETE FROM source_labels +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedTable + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_source_labels(db: Session): + """Empty ``source_labels`` before and after (migration-0030 state).""" +> db.execute(text("DELETE FROM source_labels")) + +tests/unit/test_overview.py:679: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM source_labels', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedTable) relation "source_labels" does not exist +E LINE 1: DELETE FROM source_labels +E ^ +E [SQL: DELETE FROM source_labels] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError +_ ERROR at teardown of test_regenerate_without_label_rows_is_byte_identical_to_pre_phase _ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = + + @pytest.fixture() + def clean_overview(db: Session): + db.execute(text("DELETE FROM kb_overview")) + db.commit() + yield +> db.execute(text("DELETE FROM kb_overview")) + +tests/unit/test_overview.py:303: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'DELETE FROM kb_overview', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: DELETE FROM kb_overview] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +________ ERROR at teardown of test_regenerate_reads_the_label_rows_once ________ + + Exception Group Traceback (most recent call last): + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 361, in from_call + | result: TResult | None = func() + | ~~~~^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 250, in + | lambda: runtest_hook(item=item, **kwds), + | ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pluggy/_hooks.py", line 512, in __call__ + | return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) + | ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pluggy/_manager.py", line 120, in _hookexec + | return self._inner_hookexec(hook_name, methods, kwargs, firstresult) + | ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall + | raise exception + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 139, in _multicall + | teardown.throw(exception) + | ~~~~~~~~~~~~~~^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/logging.py", line 873, in pytest_runtest_teardown + | yield + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 139, in _multicall + | teardown.throw(exception) + | ~~~~~~~~~~~~~~^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/capture.py", line 905, in pytest_runtest_teardown + | return (yield) + | ^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 121, in _multicall + | res = hook_impl.function(*args) + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 199, in pytest_runtest_teardown + | item.session._setupstate.teardown_exact(nextitem) + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 579, in teardown_exact + | raise exceptions[0] + | ExceptionGroup: errors while tearing down (2 sub-exceptions) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1969, in _exec_single_context + | self.dialect.do_execute( + | ~~~~~~~~~~~~~~~~~~~~~~~^ + | cursor, str_statement, effective_parameters, context + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute + | cursor.execute(statement, parameters) + | ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/psycopg/cursor.py", line 117, in execute + | raise ex.with_traceback(None) + | psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + | + | The above exception was the direct cause of the following exception: + | + | Traceback (most recent call last): + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 568, in teardown_exact + | fin() + | ~~~^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/fixtures.py", line 1166, in finish + | raise exceptions[0] + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/fixtures.py", line 1156, in finish + | fin() + | ~~~^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/fixtures.py", line 1014, in _teardown_yield_fixture + | next(it) + | ~~~~^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/tests/unit/test_overview.py", line 303, in clean_overview + | db.execute(text("DELETE FROM kb_overview")) + | ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py", line 2373, in execute + | return self._execute_internal( + | ~~~~~~~~~~~~~~~~~~~~~~^ + | statement, + | ^^^^^^^^^^ + | ...<4 lines>... + | _add_event=_add_event, + | ^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py", line 2280, in _execute_internal + | result = conn.execute( + | statement, params or {}, execution_options=execution_options + | ) + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1421, in execute + | return meth( + | self, + | distilled_parameters, + | execution_options or NO_OPTIONS, + | ) + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection + | return connection._execute_clauseelement( + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + | self, distilled_params, execution_options + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1643, in _execute_clauseelement + | ret = self._execute_context( + | dialect, + | ...<8 lines>... + | cache_hit=cache_hit, + | ) + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1848, in _execute_context + | return self._exec_single_context( + | ~~~~~~~~~~~~~~~~~~~~~~~~~^ + | dialect, context, statement, parameters + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1988, in _exec_single_context + | self._handle_dbapi_exception( + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + | e, str_statement, effective_parameters, cursor, context + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 2365, in _handle_dbapi_exception + | raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1969, in _exec_single_context + | self.dialect.do_execute( + | ~~~~~~~~~~~~~~~~~~~~~~~^ + | cursor, str_statement, effective_parameters, context + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute + | cursor.execute(statement, parameters) + | ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/psycopg/cursor.py", line 117, in execute + | raise ex.with_traceback(None) + | sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block + | [SQL: DELETE FROM kb_overview] + | (Background on this error at: https://sqlalche.me/e/20/2j85) + +---------------- 2 ---------------- + | Traceback (most recent call last): + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1969, in _exec_single_context + | self.dialect.do_execute( + | ~~~~~~~~~~~~~~~~~~~~~~~^ + | cursor, str_statement, effective_parameters, context + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute + | cursor.execute(statement, parameters) + | ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/psycopg/cursor.py", line 117, in execute + | raise ex.with_traceback(None) + | psycopg.errors.UndefinedTable: relation "source_labels" does not exist + | LINE 1: DELETE FROM source_labels + | ^ + | + | The above exception was the direct cause of the following exception: + | + | Traceback (most recent call last): + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 568, in teardown_exact + | fin() + | ~~~^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/fixtures.py", line 1166, in finish + | raise exceptions[0] + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/fixtures.py", line 1156, in finish + | fin() + | ~~~^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/_pytest/fixtures.py", line 1014, in _teardown_yield_fixture + | next(it) + | ~~~~^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/tests/unit/test_overview.py", line 682, in clean_source_labels + | db.execute(text("DELETE FROM source_labels")) + | ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py", line 2373, in execute + | return self._execute_internal( + | ~~~~~~~~~~~~~~~~~~~~~~^ + | statement, + | ^^^^^^^^^^ + | ...<4 lines>... + | _add_event=_add_event, + | ^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py", line 2280, in _execute_internal + | result = conn.execute( + | statement, params or {}, execution_options=execution_options + | ) + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1421, in execute + | return meth( + | self, + | distilled_parameters, + | execution_options or NO_OPTIONS, + | ) + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection + | return connection._execute_clauseelement( + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + | self, distilled_params, execution_options + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1643, in _execute_clauseelement + | ret = self._execute_context( + | dialect, + | ...<8 lines>... + | cache_hit=cache_hit, + | ) + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1848, in _execute_context + | return self._exec_single_context( + | ~~~~~~~~~~~~~~~~~~~~~~~~~^ + | dialect, context, statement, parameters + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1988, in _exec_single_context + | self._handle_dbapi_exception( + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + | e, str_statement, effective_parameters, cursor, context + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 2365, in _handle_dbapi_exception + | raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py", line 1969, in _exec_single_context + | self.dialect.do_execute( + | ~~~~~~~~~~~~~~~~~~~~~~~^ + | cursor, str_statement, effective_parameters, context + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ) + | ^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute + | cursor.execute(statement, parameters) + | ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ + | File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/psycopg/cursor.py", line 117, in execute + | raise ex.with_traceback(None) + | sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedTable) relation "source_labels" does not exist + | LINE 1: DELETE FROM source_labels + | ^ + | [SQL: DELETE FROM source_labels] + | (Background on this error at: https://sqlalche.me/e/20/f405) + +------------------------------------ +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:54:00 INFO app.rag.overview :: overview: regenerated docs=2 chars=91 +------------------------------ Captured log call ------------------------------- +INFO app.rag.overview:overview.py:315 overview: regenerated docs=2 chars=91 +=================================== FAILURES =================================== +________ test_on_progress_fires_once_per_candidate_in_sorted_key_order _________ + +db = +clean_tables = None + + def test_on_progress_fires_once_per_candidate_in_sorted_key_order( + db: Session, clean_tables + ) -> None: + """Phase 98 (task 01): the hook fires once per candidate, in the + same sorted ``(source, folder_path)`` order the folders are + attempted — done climbs 1..total, total = the candidate count + (the single-doc FSU-solo root IS a candidate at the ≥ 1 rule).""" + _seed_catalogue(db) + llm = _FakeLLM() + events, record = _record_progress() + stats = asyncio.run(generate_folder_summaries(db, llm, on_progress=record)) +> assert events == [ + (1, 4, "FSU", ""), + (2, 4, "FSU", "a"), + (3, 4, "FSU", "a/b"), + (4, 4, "FSU-solo", ""), + ] +E AssertionError: assert [(1, 5, 'FSU'...'casekb', '')] == [(1, 4, 'FSU'...SU-solo', '')] +E +E At index 0 diff: (1, 5, 'FSU', '') != (1, 4, 'FSU', '') +E Left contains one more item: (5, 5, 'casekb', '') +E Use -v to get more diff + +tests/unit/test_folder_summaries.py:1094: AssertionError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:27 INFO app.rag.folder_summaries :: folder_summaries: generated=5 failed=0 pruned=0 kept_manual=0 +------------------------------ Captured log call ------------------------------- +INFO app.rag.folder_summaries:folder_summaries.py:610 folder_summaries: generated=5 failed=0 pruned=0 kept_manual=0 +___________ test_on_chunk_progress_reports_per_file_batch_sequences ____________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_on_chunk_progress_reports0') + + def test_on_chunk_progress_reports_per_file_batch_sequences( + db, tmp_path: Path + ) -> None: + """A multi-batch file's embed position is fully observable: the hook + fires before each embed batch with ``(first_chunk_1based, total)`` — + ``total`` is the file's content-chunk count, the first indices climb + by the previous batch's size, and a SECOND file in the same run gets + its OWN sequence (the two files' totals never mix).""" + root = tmp_path / "chunkprog" + root.mkdir() + content_a = _markdown_with_sections("Alpha", 8) # preamble + 8 = 9 chunks + content_b = _markdown_with_sections("Beta", 4) # preamble + 4 = 5 chunks + (root / "alpha.md").write_text(content_a) + (root / "beta.md").write_text(content_b) + n_a, n_b = _chunk_count(content_a, "alpha.md"), _chunk_count(content_b, "beta.md") + assert (n_a, n_b) == (9, 5) + + batch_size = 3 + + def expected(n: int) -> list[tuple[int, int]]: + return [(start + 1, n) for start in range(0, n, batch_size)] + + # The task's multi-batch case: file A spans >= 2 batches. + assert len(expected(n_a)) >= 2 + + llm = _BatchingEmbedder(batch_size=batch_size) + chunk_events: list[tuple[int, int]] = [] + file_events: list[tuple[str, str, int, int]] = [] + try: +> summary = asyncio.run( + import_sources( + [root], llm, session=db, + progress=lambda s, r, d, t: file_events.append((s, r, d, t)), + on_chunk_progress=lambda f, t: chunk_events.append((f, t)), + ) + ) + +tests/unit/test_importer.py:1591: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1704: in _index_file + await _store_summary( +app/rag/importer.py:1772: in _store_summary + session.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:86: in save_obj + _emit_update_statements( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +base_mapper = +uowtransaction = +mapper = +table = Table('documents', MetaData(), Column('id', UUID(), table=, primary_key=True, nullable=False, default=Calla...ge_count', Integer(), table=), Column('page_strip_mode', String(length=16), table=), schema=None) +update = +bookkeeping = True, use_orm_update_stmt = None, enable_check_rowcount = True + + def _emit_update_statements( + base_mapper, + uowtransaction, + mapper, + table, + update, + *, + bookkeeping=True, + use_orm_update_stmt=None, + enable_check_rowcount=True, + ): + """Emit UPDATE statements corresponding to value lists collected + by _collect_update_commands().""" + + needs_version_id = ( + mapper.version_id_col is not None + and mapper.version_id_col in mapper._cols_by_table[table] + ) + + execution_options = {"compiled_cache": base_mapper._compiled_cache} + + def update_stmt(existing_stmt=None): + clauses = BooleanClauseList._construct_raw(operators.and_) + + for col in mapper._pks_by_table[table]: + clauses._append_inplace( + col == sql.bindparam(col._label, type_=col.type) + ) + + if needs_version_id: + clauses._append_inplace( + mapper.version_id_col + == sql.bindparam( + mapper.version_id_col._label, + type_=mapper.version_id_col.type, + ) + ) + + if existing_stmt is not None: + stmt = existing_stmt.where(clauses) + else: + stmt = table.update().where(clauses) + return stmt + + if use_orm_update_stmt is not None: + cached_stmt = update_stmt(use_orm_update_stmt) + + else: + cached_stmt = base_mapper._memo(("update", table), update_stmt) + + for ( + (connection, paramkeys, hasvalue, has_all_defaults, has_all_pks), + records, + ) in groupby( + update, + lambda rec: ( + rec[4], # connection + set(rec[2]), # set of parameter keys + bool(rec[5]), # whether or not we have "value" parameters + rec[6], # has_all_defaults + rec[7], # has all pks + ), + ): + rows = 0 + records = list(records) + + statement = cached_stmt + + if use_orm_update_stmt is not None: + statement = statement._annotate( + { + "_emit_update_table": table, + "_emit_update_mapper": mapper, + } + ) + + return_defaults = False + + if not has_all_pks: + statement = statement.return_defaults(*mapper._pks_by_table[table]) + return_defaults = True + + if ( + bookkeeping + and not has_all_defaults + and mapper.base_mapper.eager_defaults is True + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + and table.implicit_returning + and connection.dialect.update_returning + ): + statement = statement.return_defaults( + *mapper._server_onupdate_default_cols[table] + ) + return_defaults = True + + if mapper._version_id_has_server_side_value: + statement = statement.return_defaults(mapper.version_id_col) + return_defaults = True + + assert_singlerow = connection.dialect.supports_sane_rowcount + + assert_multirow = ( + assert_singlerow + and connection.dialect.supports_sane_multi_rowcount + ) + + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + allow_executemany = not return_defaults and not needs_version_id + + if hasvalue: + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement.values(value_params), + params, + execution_options=execution_options, + ) + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + check_rowcount = enable_check_rowcount and assert_singlerow + else: + if not allow_executemany: + check_rowcount = enable_check_rowcount and assert_singlerow + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement, params, execution_options=execution_options + ) + + # TODO: why with bookkeeping=False? + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + else: + multiparams = [rec[2] for rec in records] + + check_rowcount = enable_check_rowcount and ( + assert_multirow + or (assert_singlerow and len(multiparams) == 1) + ) + + c = connection.execute( + statement, multiparams, execution_options=execution_options + ) + + rows += c.rowcount + + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + ( + c.returned_defaults + if not c.context.executemany + else None + ), + ) + + if check_rowcount: + if rows != len(records): +> raise orm_exc.StaleDataError( + "UPDATE statement on table '%s' expected to " + "update %d row(s); %d were matched." + % (table.description, len(records), rows) + ) +E sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:949: StaleDataError + +During handling of the above exception, another exception occurred: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_on_chunk_progress_reports0') + + def test_on_chunk_progress_reports_per_file_batch_sequences( + db, tmp_path: Path + ) -> None: + """A multi-batch file's embed position is fully observable: the hook + fires before each embed batch with ``(first_chunk_1based, total)`` — + ``total`` is the file's content-chunk count, the first indices climb + by the previous batch's size, and a SECOND file in the same run gets + its OWN sequence (the two files' totals never mix).""" + root = tmp_path / "chunkprog" + root.mkdir() + content_a = _markdown_with_sections("Alpha", 8) # preamble + 8 = 9 chunks + content_b = _markdown_with_sections("Beta", 4) # preamble + 4 = 5 chunks + (root / "alpha.md").write_text(content_a) + (root / "beta.md").write_text(content_b) + n_a, n_b = _chunk_count(content_a, "alpha.md"), _chunk_count(content_b, "beta.md") + assert (n_a, n_b) == (9, 5) + + batch_size = 3 + + def expected(n: int) -> list[tuple[int, int]]: + return [(start + 1, n) for start in range(0, n, batch_size)] + + # The task's multi-batch case: file A spans >= 2 batches. + assert len(expected(n_a)) >= 2 + + llm = _BatchingEmbedder(batch_size=batch_size) + chunk_events: list[tuple[int, int]] = [] + file_events: list[tuple[str, str, int, int]] = [] + try: + summary = asyncio.run( + import_sources( + [root], llm, session=db, + progress=lambda s, r, d, t: file_events.append((s, r, d, t)), + on_chunk_progress=lambda f, t: chunk_events.append((f, t)), + ) + ) + assert (summary.added, summary.chunks) == (2, n_a + n_b) + finally: +> _cleanup_source(db, root.name) + +tests/unit/test_importer.py:1600: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_importer.py:115: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:36 INFO app.importer :: import: added source=chunkprog path=alpha.md chunks=9 +2026-10-19 11:53:36 INFO app.importer :: import: summary source=chunkprog path=alpha.md chars=39 +2026-10-19 11:53:36 INFO app.importer :: import: added source=chunkprog path=beta.md chunks=5 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:1697 import: added source=chunkprog path=alpha.md chunks=9 +INFO app.importer:importer.py:1784 import: summary source=chunkprog path=alpha.md chars=39 +INFO app.importer:importer.py:1697 import: added source=chunkprog path=beta.md chunks=5 +__________ test_on_image_progress_embedded_pass_climbs_one_per_image ___________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_on_image_progress_embedde0') + + def test_on_image_progress_embedded_pass_climbs_one_per_image( + db, tmp_path: Path + ) -> None: + """The phase-152 core pin: a PDF with N DISTINCT embedded images + (one placement each — post-dedupe, post-blank-gate: the work + actually about to happen) fires the hook ONCE per image, 1-based, + with the span's total, at the START of each image's pipeline step: + ``(1, N) … (N, N)``. A repeated placement of the first image (the + parser's dedupe, phase 151) adds NO position (the total is the + post-dedupe count), a clean usable text layer burns no fallback or + pictorial span (the embedded pass's totals stand alone), and the + phase-147 fast path on a re-sync (raw bytes unchanged) burns no + vision call and fires the hook never.""" + root = tmp_path / "imgprog" + root.mkdir() + values = (90, 130, 170, 210) # 4 distinct images, all under the blank floor + _build_image_pdf(root / "report.pdf", values, repeat_first=True) + n = len(values) + llm = _PageVisionLLM() + llm.settings = _pdf_settings(tmp_path, images=True) + events: list[tuple[int, int]] = [] + try: + summary = asyncio.run( + import_sources( + [root], llm, session=db, + on_image_progress=lambda pos, total: events.append((pos, total)), + ) + ) + # The PDF + its N image docs; the repeat placement deduped. + assert (summary.files, summary.added) == (1, 1 + n) + assert summary.pdf_images_deduped == 1 + assert (summary.images_failed, summary.pdfs_screenshotted, + summary.pdfs_pictorial) == (0, 0, 0) + # The embedded pass is the ONLY vision span: N slow calls. + assert llm.vision_calls == n + # The core pin: once per image, 1-based, the span's total. + assert events == [(i, n) for i in range(1, n + 1)] + # The phase-147 fast path: a re-sync (bytes unchanged) burns no + # vision call and fires the hook never — the events list grows + # not at all. + summary2 = asyncio.run( + import_sources( + [root], llm, session=db, + on_image_progress=lambda pos, total: events.append((pos, total)), + ) + ) +> assert summary2.unchanged == 1 +E AssertionError: assert 0 == 1 +E + where 0 = ImportSummary(files=1, added=3, updated=0, unchanged=0, pruned=0, errors=0, chunks=3, embed_batches=16, summaries=3, s...ort.pdf'), ('imgprog', 'report.pdf#page=1#img=0.png'), ('imgprog', 'report.pdf#page=1#img=1.png')], formats={'pdf': 1}).unchanged + +tests/unit/test_importer.py:2311: AssertionError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf chars=50 +2026-10-19 11:53:39 INFO app.rag.pdf :: pdf: dropped 1 duplicate image occurrence(s) from /tmp/pytest-of-ducoterra/pytest-2115/test_on_image_progress_embedde0/imgprog/report.pdf (4 unique) +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf#page=1#img=0.png chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf#page=1#img=0.png chars=308 +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf#page=1#img=1.png chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf#page=1#img=1.png chars=308 +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf#page=1#img=2.png chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf#page=1#img=2.png chars=308 +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf#page=1#img=3.png chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf#page=1#img=3.png chars=308 +2026-10-19 11:53:39 INFO app.importer :: import: summary files=1 added=5 updated=0 unchanged=0 pruned=0 errors=0 chunks=5 embed_batches=10 summaries=5 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=1 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf chars=50 +2026-10-19 11:53:39 INFO app.rag.pdf :: pdf: dropped 1 duplicate image occurrence(s) from /tmp/pytest-of-ducoterra/pytest-2115/test_on_image_progress_embedde0/imgprog/report.pdf (4 unique) +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf#page=1#img=0.png chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf#page=1#img=0.png chars=308 +2026-10-19 11:53:39 INFO app.importer :: import: added source=imgprog path=report.pdf#page=1#img=1.png chunks=1 +2026-10-19 11:53:39 INFO app.importer :: import: summary source=imgprog path=report.pdf#page=1#img=1.png chars=308 +2026-10-19 11:53:39 INFO app.importer :: import: summary files=1 added=3 updated=0 unchanged=0 pruned=0 errors=0 chunks=3 embed_batches=16 summaries=3 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=1 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:2128 import: added source=imgprog path=report.pdf chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf chars=50 +INFO app.rag.pdf:pdf.py:784 pdf: dropped 1 duplicate image occurrence(s) from /tmp/pytest-of-ducoterra/pytest-2115/test_on_image_progress_embedde0/imgprog/report.pdf (4 unique) +INFO app.importer:importer.py:2925 import: added source=imgprog path=report.pdf#page=1#img=0.png chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf#page=1#img=0.png chars=308 +INFO app.importer:importer.py:2925 import: added source=imgprog path=report.pdf#page=1#img=1.png chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf#page=1#img=1.png chars=308 +INFO app.importer:importer.py:2925 import: added source=imgprog path=report.pdf#page=1#img=2.png chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf#page=1#img=2.png chars=308 +INFO app.importer:importer.py:2925 import: added source=imgprog path=report.pdf#page=1#img=3.png chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf#page=1#img=3.png chars=308 +INFO app.importer:importer.py:530 import: summary files=1 added=5 updated=0 unchanged=0 pruned=0 errors=0 chunks=5 embed_batches=10 summaries=5 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=1 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +INFO app.importer:importer.py:2128 import: added source=imgprog path=report.pdf chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf chars=50 +INFO app.rag.pdf:pdf.py:784 pdf: dropped 1 duplicate image occurrence(s) from /tmp/pytest-of-ducoterra/pytest-2115/test_on_image_progress_embedde0/imgprog/report.pdf (4 unique) +INFO app.importer:importer.py:2925 import: added source=imgprog path=report.pdf#page=1#img=0.png chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf#page=1#img=0.png chars=308 +INFO app.importer:importer.py:2925 import: added source=imgprog path=report.pdf#page=1#img=1.png chunks=1 +INFO app.importer:importer.py:1784 import: summary source=imgprog path=report.pdf#page=1#img=1.png chars=308 +INFO app.importer:importer.py:530 import: summary files=1 added=3 updated=0 unchanged=0 pruned=0 errors=0 chunks=3 embed_batches=16 summaries=3 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=1 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +_______________ test_manual_row_date_survives_unchanged_reimport _______________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_manual_row_date_survives_0') + + def test_manual_row_date_survives_unchanged_reimport(db, tmp_path: Path) -> None: + """D1: a row carrying the owner's correction (``created_at_manual``) + is left ENTIRELY alone on the unchanged path — the moved mtime does + not refresh it and ``dates_updated`` stays 0 (the sibling of the + phase-97 ``manually_edited`` precedent).""" + root, file = _make_root(tmp_path, "DateManual") + llm = FakeEmbedder() + correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC) + try: +> first = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_importer_dates.py:194: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1704: in _index_file + await _store_summary( +app/rag/importer.py:1772: in _store_summary + session.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:86: in save_obj + _emit_update_statements( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +base_mapper = +uowtransaction = +mapper = +table = Table('documents', MetaData(), Column('id', UUID(), table=, primary_key=True, nullable=False, default=Calla...ge_count', Integer(), table=), Column('page_strip_mode', String(length=16), table=), schema=None) +update = +bookkeeping = True, use_orm_update_stmt = None, enable_check_rowcount = True + + def _emit_update_statements( + base_mapper, + uowtransaction, + mapper, + table, + update, + *, + bookkeeping=True, + use_orm_update_stmt=None, + enable_check_rowcount=True, + ): + """Emit UPDATE statements corresponding to value lists collected + by _collect_update_commands().""" + + needs_version_id = ( + mapper.version_id_col is not None + and mapper.version_id_col in mapper._cols_by_table[table] + ) + + execution_options = {"compiled_cache": base_mapper._compiled_cache} + + def update_stmt(existing_stmt=None): + clauses = BooleanClauseList._construct_raw(operators.and_) + + for col in mapper._pks_by_table[table]: + clauses._append_inplace( + col == sql.bindparam(col._label, type_=col.type) + ) + + if needs_version_id: + clauses._append_inplace( + mapper.version_id_col + == sql.bindparam( + mapper.version_id_col._label, + type_=mapper.version_id_col.type, + ) + ) + + if existing_stmt is not None: + stmt = existing_stmt.where(clauses) + else: + stmt = table.update().where(clauses) + return stmt + + if use_orm_update_stmt is not None: + cached_stmt = update_stmt(use_orm_update_stmt) + + else: + cached_stmt = base_mapper._memo(("update", table), update_stmt) + + for ( + (connection, paramkeys, hasvalue, has_all_defaults, has_all_pks), + records, + ) in groupby( + update, + lambda rec: ( + rec[4], # connection + set(rec[2]), # set of parameter keys + bool(rec[5]), # whether or not we have "value" parameters + rec[6], # has_all_defaults + rec[7], # has all pks + ), + ): + rows = 0 + records = list(records) + + statement = cached_stmt + + if use_orm_update_stmt is not None: + statement = statement._annotate( + { + "_emit_update_table": table, + "_emit_update_mapper": mapper, + } + ) + + return_defaults = False + + if not has_all_pks: + statement = statement.return_defaults(*mapper._pks_by_table[table]) + return_defaults = True + + if ( + bookkeeping + and not has_all_defaults + and mapper.base_mapper.eager_defaults is True + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + and table.implicit_returning + and connection.dialect.update_returning + ): + statement = statement.return_defaults( + *mapper._server_onupdate_default_cols[table] + ) + return_defaults = True + + if mapper._version_id_has_server_side_value: + statement = statement.return_defaults(mapper.version_id_col) + return_defaults = True + + assert_singlerow = connection.dialect.supports_sane_rowcount + + assert_multirow = ( + assert_singlerow + and connection.dialect.supports_sane_multi_rowcount + ) + + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + allow_executemany = not return_defaults and not needs_version_id + + if hasvalue: + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement.values(value_params), + params, + execution_options=execution_options, + ) + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + check_rowcount = enable_check_rowcount and assert_singlerow + else: + if not allow_executemany: + check_rowcount = enable_check_rowcount and assert_singlerow + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement, params, execution_options=execution_options + ) + + # TODO: why with bookkeeping=False? + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + else: + multiparams = [rec[2] for rec in records] + + check_rowcount = enable_check_rowcount and ( + assert_multirow + or (assert_singlerow and len(multiparams) == 1) + ) + + c = connection.execute( + statement, multiparams, execution_options=execution_options + ) + + rows += c.rowcount + + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + ( + c.returned_defaults + if not c.context.executemany + else None + ), + ) + + if check_rowcount: + if rows != len(records): +> raise orm_exc.StaleDataError( + "UPDATE statement on table '%s' expected to " + "update %d row(s); %d were matched." + % (table.description, len(records), rows) + ) +E sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:949: StaleDataError + +During handling of the above exception, another exception occurred: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_manual_row_date_survives_0') + + def test_manual_row_date_survives_unchanged_reimport(db, tmp_path: Path) -> None: + """D1: a row carrying the owner's correction (``created_at_manual``) + is left ENTIRELY alone on the unchanged path — the moved mtime does + not refresh it and ``dates_updated`` stays 0 (the sibling of the + phase-97 ``manually_edited`` precedent).""" + root, file = _make_root(tmp_path, "DateManual") + llm = FakeEmbedder() + correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC) + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert first.added == 1 + doc = _doc(db, root.name, "note.md") + doc.created_at = correction # the owner's correction (task 05's API) + doc.created_at_manual = True + db.commit() + _utime(file, OLD_2021) # the source moved to a DIFFERENT date + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.pruned, second.unchanged) == (0, 0, 0, 1) + assert second.dates_updated == 0 + db.expire_all() + doc = _doc(db, root.name, "note.md") + assert doc.created_at == correction # the correction survived + assert doc.created_at_manual is True + finally: +> _cleanup_source(db, root.name) + +tests/unit/test_importer_dates.py:209: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_importer_dates.py:48: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:42 INFO app.importer :: import: added source=DateManual path=note.md chunks=1 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:1697 import: added source=DateManual path=note.md chunks=1 +_______________ test_regate_fires_on_the_content_digest_hit_path _______________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_regate_fires_on_the_conte0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f531f3dfa80> + + def test_regate_fires_on_the_content_digest_hit_path( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """The SECOND unchanged path (the PDF branch's content-digest hit — + a pre-phase row with ``file_hash`` NULL): the branch re-runs its own + gate (the byte-change's honest cost) and lands unchanged, and the + shared helper's tail still heals the strip deleted out from under + the row — the re-gate is not fast-path-only.""" + root = tmp_path / "rgbranch" + root.mkdir() + _build_ikea_like_pdf(root / "ikea.pdf") + llm = _PictorialLLM() + llm.settings = _regate_settings(tmp_path) + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = _doc(db, "rgbranch", "ikea.pdf") + doc.file_hash = None # the pre-phase row: the fast path cannot fire + db.commit() + page_dir = _pages_dir(tmp_path) + for path in _strip_copies(doc, page_dir, 10): + path.unlink() + + with caplog.at_level(logging.INFO, logger="app.importer"): + second = asyncio.run(import_sources([root], llm, session=db)) + + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + db.expire_all() +> doc2 = _doc(db, "rgbranch", "ikea.pdf") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_importer_unchanged_fast_path.py:943: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +db = +source = 'rgbranch', rel = 'ikea.pdf' + + def _doc(db, source: str, rel: str) -> Document: + doc = db.scalar(select(Document).where(Document.source == source, Document.path == rel)) +> assert doc is not None, f"no documents row for ({source!r}, {rel!r})" +E AssertionError: no documents row for ('rgbranch', 'ikea.pdf') +E assert None is not None + +tests/unit/test_importer_unchanged_fast_path.py:95: AssertionError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:49 INFO app.importer :: pdf: pictorial pass source=rgbranch path=ikea.pdf — 6 candidate pages, 6 described (min_chars=100) +2026-10-19 11:53:50 INFO app.importer :: import: added source=rgbranch path=ikea.pdf chunks=10 +2026-10-19 11:53:50 INFO app.importer :: import: summary source=rgbranch path=ikea.pdf chars=49 +2026-10-19 11:53:50 INFO app.importer :: import: summary files=1 added=1 updated=0 unchanged=0 pruned=0 errors=0 chunks=10 embed_batches=2 summaries=1 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=1 pdf_pictorial_pages=6 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +2026-10-19 11:53:50 INFO app.importer :: pdf: pictorial pass source=rgbranch path=ikea.pdf — 6 candidate pages, 6 described (min_chars=100) +2026-10-19 11:53:50 INFO app.importer :: import: unchanged source=rgbranch path=ikea.pdf +2026-10-19 11:53:50 INFO app.importer :: import: PDF strip re-gate source=rgbranch path=ikea.pdf — healed 10 pages +2026-10-19 11:53:50 INFO app.importer :: import: summary files=1 added=0 updated=0 unchanged=1 pruned=0 errors=0 chunks=0 embed_batches=2 summaries=0 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=1 pdf_pictorial_pages=6 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:2380 pdf: pictorial pass source=rgbranch path=ikea.pdf — 6 candidate pages, 6 described (min_chars=100) +INFO app.importer:importer.py:2128 import: added source=rgbranch path=ikea.pdf chunks=10 +INFO app.importer:importer.py:1784 import: summary source=rgbranch path=ikea.pdf chars=49 +INFO app.importer:importer.py:530 import: summary files=1 added=1 updated=0 unchanged=0 pruned=0 errors=0 chunks=10 embed_batches=2 summaries=1 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=1 pdf_pictorial_pages=6 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +INFO app.importer:importer.py:2380 pdf: pictorial pass source=rgbranch path=ikea.pdf — 6 candidate pages, 6 described (min_chars=100) +INFO app.importer:importer.py:1240 import: unchanged source=rgbranch path=ikea.pdf +INFO app.importer:importer.py:1450 import: PDF strip re-gate source=rgbranch path=ikea.pdf — healed 10 pages +INFO app.importer:importer.py:530 import: summary files=1 added=0 updated=0 unchanged=1 pruned=0 errors=0 chunks=0 embed_batches=2 summaries=0 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=1 pdf_pictorial_pages=6 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=pdf:1 rss_mb=306 +___________ test_manual_date_null_hash_row_stamps_without_date_write ___________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_manual_date_null_hash_row0') + + def test_manual_date_null_hash_row_stamps_without_date_write( + db, tmp_path: Path + ) -> None: + """D1 on the BRANCH path (a pre-phase NULL-hash row that also + carries the manual correction): the shared helper's stamp is the + only write — it COMMITS (so the next sync is fast even for a + manual-date row) — while the DATE stays the manual value + (``dates_updated`` 0, the flag intact).""" + root = tmp_path / "fastmanualnull" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\ncontent for the KB\n") + llm = FakeEmbedder() + correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC) + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = _doc(db, root.name, "note.md") + doc.created_at = correction + doc.created_at_manual = True + doc.file_hash = None # pre-phase row + db.commit() + _utime(path, OLD_2021) # a refresh is DUE — the lock must withhold it + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.dates_updated == 0 + # No summary work of any kind on this run (run 1's summary + # count is the baseline). + assert len(llm.chat_calls) == chats1 + assert len(llm.calls) == embeds1 + + db.expire_all() +> doc2 = _doc(db, root.name, "note.md") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_importer_unchanged_fast_path.py:1055: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +db = +source = 'fastmanualnull', rel = 'note.md' + + def _doc(db, source: str, rel: str) -> Document: + doc = db.scalar(select(Document).where(Document.source == source, Document.path == rel)) +> assert doc is not None, f"no documents row for ({source!r}, {rel!r})" +E AssertionError: no documents row for ('fastmanualnull', 'note.md') +E assert None is not None + +tests/unit/test_importer_unchanged_fast_path.py:95: AssertionError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:50 INFO app.importer :: import: added source=fastmanualnull path=note.md chunks=1 +2026-10-19 11:53:50 INFO app.importer :: import: summary source=fastmanualnull path=note.md chars=43 +2026-10-19 11:53:50 INFO app.importer :: import: summary files=1 added=1 updated=0 unchanged=0 pruned=0 errors=0 chunks=1 embed_batches=2 summaries=1 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=md:1 rss_mb=306 +2026-10-19 11:53:50 INFO app.importer :: import: unchanged source=fastmanualnull path=note.md +2026-10-19 11:53:50 INFO app.importer :: import: summary files=1 added=0 updated=0 unchanged=1 pruned=0 errors=0 chunks=0 embed_batches=2 summaries=0 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=md:1 rss_mb=306 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:1697 import: added source=fastmanualnull path=note.md chunks=1 +INFO app.importer:importer.py:1784 import: summary source=fastmanualnull path=note.md chars=43 +INFO app.importer:importer.py:530 import: summary files=1 added=1 updated=0 unchanged=0 pruned=0 errors=0 chunks=1 embed_batches=2 summaries=1 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=md:1 rss_mb=306 +INFO app.importer:importer.py:1240 import: unchanged source=fastmanualnull path=note.md +INFO app.importer:importer.py:530 import: summary files=1 added=0 updated=0 unchanged=1 pruned=0 errors=0 chunks=0 embed_batches=2 summaries=0 summary_errors=0 summary_backfilled=0 dates_updated=0 images_failed=0 svg_indexed=0 image_context_backfilled=0 pdfs_screenshotted=0 pdfs_pictorial=0 pdf_pictorial_pages=0 pdf_images_deduped=0 pdf_images_blank_skipped=0 formats=md:1 rss_mb=306 +______________ test_unchanged_fast_path_date_refresh_may_go_older ______________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_unchanged_fast_path_date_0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e32be660> + + def test_unchanged_fast_path_date_refresh_may_go_older( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """D4 on the fast path: an unchanged file whose mtime-derived date + moved (OLDER — no monotonic guard) refreshes the stored date on + the fast path: ``dates_updated == 1``, the date lands, and zero + llm calls (the refresh is bookkeeping, not a re-index). The + PRE-CHANGE log wordings are pinned verbatim (``import: unchanged`` + + ``import: date-refreshed``).""" + root = tmp_path / "fastdate" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\ncontent for the KB\n") + llm = FakeEmbedder() + try: +> asyncio.run(import_sources([root], llm, session=db)) + +tests/unit/test_importer_unchanged_fast_path.py:1084: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1704: in _index_file + await _store_summary( +app/rag/importer.py:1772: in _store_summary + session.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:86: in save_obj + _emit_update_statements( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +base_mapper = +uowtransaction = +mapper = +table = Table('documents', MetaData(), Column('id', UUID(), table=, primary_key=True, nullable=False, default=Calla...ge_count', Integer(), table=), Column('page_strip_mode', String(length=16), table=), schema=None) +update = +bookkeeping = True, use_orm_update_stmt = None, enable_check_rowcount = True + + def _emit_update_statements( + base_mapper, + uowtransaction, + mapper, + table, + update, + *, + bookkeeping=True, + use_orm_update_stmt=None, + enable_check_rowcount=True, + ): + """Emit UPDATE statements corresponding to value lists collected + by _collect_update_commands().""" + + needs_version_id = ( + mapper.version_id_col is not None + and mapper.version_id_col in mapper._cols_by_table[table] + ) + + execution_options = {"compiled_cache": base_mapper._compiled_cache} + + def update_stmt(existing_stmt=None): + clauses = BooleanClauseList._construct_raw(operators.and_) + + for col in mapper._pks_by_table[table]: + clauses._append_inplace( + col == sql.bindparam(col._label, type_=col.type) + ) + + if needs_version_id: + clauses._append_inplace( + mapper.version_id_col + == sql.bindparam( + mapper.version_id_col._label, + type_=mapper.version_id_col.type, + ) + ) + + if existing_stmt is not None: + stmt = existing_stmt.where(clauses) + else: + stmt = table.update().where(clauses) + return stmt + + if use_orm_update_stmt is not None: + cached_stmt = update_stmt(use_orm_update_stmt) + + else: + cached_stmt = base_mapper._memo(("update", table), update_stmt) + + for ( + (connection, paramkeys, hasvalue, has_all_defaults, has_all_pks), + records, + ) in groupby( + update, + lambda rec: ( + rec[4], # connection + set(rec[2]), # set of parameter keys + bool(rec[5]), # whether or not we have "value" parameters + rec[6], # has_all_defaults + rec[7], # has all pks + ), + ): + rows = 0 + records = list(records) + + statement = cached_stmt + + if use_orm_update_stmt is not None: + statement = statement._annotate( + { + "_emit_update_table": table, + "_emit_update_mapper": mapper, + } + ) + + return_defaults = False + + if not has_all_pks: + statement = statement.return_defaults(*mapper._pks_by_table[table]) + return_defaults = True + + if ( + bookkeeping + and not has_all_defaults + and mapper.base_mapper.eager_defaults is True + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + and table.implicit_returning + and connection.dialect.update_returning + ): + statement = statement.return_defaults( + *mapper._server_onupdate_default_cols[table] + ) + return_defaults = True + + if mapper._version_id_has_server_side_value: + statement = statement.return_defaults(mapper.version_id_col) + return_defaults = True + + assert_singlerow = connection.dialect.supports_sane_rowcount + + assert_multirow = ( + assert_singlerow + and connection.dialect.supports_sane_multi_rowcount + ) + + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + allow_executemany = not return_defaults and not needs_version_id + + if hasvalue: + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement.values(value_params), + params, + execution_options=execution_options, + ) + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + check_rowcount = enable_check_rowcount and assert_singlerow + else: + if not allow_executemany: + check_rowcount = enable_check_rowcount and assert_singlerow + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement, params, execution_options=execution_options + ) + + # TODO: why with bookkeeping=False? + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + else: + multiparams = [rec[2] for rec in records] + + check_rowcount = enable_check_rowcount and ( + assert_multirow + or (assert_singlerow and len(multiparams) == 1) + ) + + c = connection.execute( + statement, multiparams, execution_options=execution_options + ) + + rows += c.rowcount + + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + ( + c.returned_defaults + if not c.context.executemany + else None + ), + ) + + if check_rowcount: + if rows != len(records): +> raise orm_exc.StaleDataError( + "UPDATE statement on table '%s' expected to " + "update %d row(s); %d were matched." + % (table.description, len(records), rows) + ) +E sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:949: StaleDataError + +During handling of the above exception, another exception occurred: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_unchanged_fast_path_date_0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e32be660> + + def test_unchanged_fast_path_date_refresh_may_go_older( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """D4 on the fast path: an unchanged file whose mtime-derived date + moved (OLDER — no monotonic guard) refreshes the stored date on + the fast path: ``dates_updated == 1``, the date lands, and zero + llm calls (the refresh is bookkeeping, not a re-index). The + PRE-CHANGE log wordings are pinned verbatim (``import: unchanged`` + + ``import: date-refreshed``).""" + root = tmp_path / "fastdate" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\ncontent for the KB\n") + llm = FakeEmbedder() + try: + asyncio.run(import_sources([root], llm, session=db)) + _utime(path, OLD_2021) # the source date moved; content identical + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + + with caplog.at_level(logging.INFO, logger="app.importer"): + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.dates_updated == 1 # the refresh ran (fast path) + assert len(llm.calls) == embeds1 and len(llm.chat_calls) == chats1 + # The preserved wordings (the shared helper's log lines, verbatim + # from the per-branch blocks). + messages = [r.getMessage() for r in caplog.records] + assert f"import: unchanged source={root.name} path=note.md" in messages + assert any( + m.startswith(f"import: date-refreshed source={root.name} path=note.md date=") + for m in messages + ) + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert abs(doc2.created_at - OLD_2021) <= _TOL + finally: +> _cleanup_source(db, root.name) + +tests/unit/test_importer_unchanged_fast_path.py:1106: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_importer_unchanged_fast_path.py:100: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:50 INFO app.importer :: import: added source=fastdate path=note.md chunks=1 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:1697 import: added source=fastdate path=note.md chunks=1 +______________ test_regenerate_happy_path_creates_single_row_id_1 ______________ + +db = +clean_overview = None +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f531f021fd0> + + def test_regenerate_happy_path_creates_single_row_id_1( + db: Session, clean_overview, caplog: pytest.LogCaptureFixture + ) -> None: + _truncate_documents(db) + try: + _add_doc(db, "Homelab", "b.md", "B") + _add_doc(db, "Homelab", "a.md", "A", "A summary.\nSource: Homelab/a.md") + llm = _FakeLLM() + with caplog.at_level(logging.INFO, logger="app.rag.overview"): + ok = asyncio.run(regenerate_overview(llm, db)) + assert ok is True + assert llm.calls == 1 + row = db.get(KbOverview, 1) +> assert row is not None, "the single row must be upserted" +E AssertionError: the single row must be upserted +E assert None is not None + +tests/unit/test_overview.py:320: AssertionError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:53:54 INFO app.rag.overview :: overview: regenerated docs=2 chars=87 +------------------------------ Captured log call ------------------------------- +INFO app.rag.overview:overview.py:315 overview: regenerated docs=2 chars=87 +_________ test_document_list_reaching_the_model_leads_with_newest_docs _________ + +self = +dialect = +context = +statement = +parameters = [{'id': UUID('6adc3bd3-d044-478c-bc43-553053031f0c'), 'source': 's', 'path': '01_old_a.md', 'full_path': '/tmp/01_old_a.md', ...}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summa...::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at' +params = {'id': UUID('6adc3bd3-d044-478c-bc43-553053031f0c'), 'source': 's', 'path': '01_old_a.md', 'full_path': '/tmp/01_old_a.md', ...} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +clean_overview = None + + def test_document_list_reaching_the_model_leads_with_newest_docs( + db: Session, clean_overview + ) -> None: + """Phase 142, task 01 repro, flipped green by task 02 (the strict + xfail marker is removed — a pass is now a pass). + + The document list reaching the lite model must LEAD with the NEWEST + rows (``created_at`` desc — the document's date, phase 106) so that, + when the cap bites, the newest documents — not the lexicographically + first — are the ones the outline covers. Pre-fix the query was plain + ``order_by(source, path)``: with the phase-dir rows below (recency + strictly OPPOSITE to the path order) that put ``01_old_a.md`` — the + OLDEST — first and ``20_new_t.md`` — the NEWEST — last, the traced + "summaries only up to phase 113" failure at unit scale. The + assertion is the POST-FIX order task 02's ``created_at``-desc + ordering delivers. + """ + _truncate_documents(db) + try: + base = datetime(2026, 1, 1, tzinfo=UTC) + for i, (source, path, title, summary) in enumerate(_phase_dir_rows()): +> _add_doc(db, source, path, title, summary) + +tests/unit/test_overview.py:415: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_overview.py:285: in _add_doc + db.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:94: in save_obj + _emit_insert_statements( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:1234: in _emit_insert_statements + result = connection.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summa...::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at' +params = {'id': UUID('6adc3bd3-d044-478c-bc43-553053031f0c'), 'source': 's', 'path': '01_old_a.md', 'full_path': '/tmp/01_old_a.md', ...} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('6adc3bd3-d044-478c-bc43-553053031f0c'), 'source': 's', 'path': '01_old_a.md', 'full_path': '/tmp/01_old_a.md', 'title': 'Phase 01', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +db = +clean_overview = None + + def test_document_list_reaching_the_model_leads_with_newest_docs( + db: Session, clean_overview + ) -> None: + """Phase 142, task 01 repro, flipped green by task 02 (the strict + xfail marker is removed — a pass is now a pass). + + The document list reaching the lite model must LEAD with the NEWEST + rows (``created_at`` desc — the document's date, phase 106) so that, + when the cap bites, the newest documents — not the lexicographically + first — are the ones the outline covers. Pre-fix the query was plain + ``order_by(source, path)``: with the phase-dir rows below (recency + strictly OPPOSITE to the path order) that put ``01_old_a.md`` — the + OLDEST — first and ``20_new_t.md`` — the NEWEST — last, the traced + "summaries only up to phase 113" failure at unit scale. The + assertion is the POST-FIX order task 02's ``created_at``-desc + ordering delivers. + """ + _truncate_documents(db) + try: + base = datetime(2026, 1, 1, tzinfo=UTC) + for i, (source, path, title, summary) in enumerate(_phase_dir_rows()): + _add_doc(db, source, path, title, summary) + # Row i (0-based) is i days after the base: index 0 + # (``01_old_a.md``) is the OLDEST, index 19 (``20_new_t.md``) + # the NEWEST — recency strictly opposite to the path order. + db.execute( + text("UPDATE documents SET created_at = :d WHERE path = :p"), + {"d": base + timedelta(days=i), "p": path}, + ) + db.commit() + llm = _FakeLLM() + assert asyncio.run(regenerate_overview(llm, db)) is True + finally: +> _truncate_documents(db) + +tests/unit/test_overview.py:427: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_overview.py:290: in _truncate_documents + db.execute(text("TRUNCATE chunks, documents")) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('6adc3bd3-d044-478c-bc43-553053031f0c'), 'source': 's', 'path': '01_old_a.md', 'full_path': '/tmp/01_old_a.md', 'title': 'Phase 01', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +_____________ test_regenerate_llm_error_leaves_previous_row_intact _____________ + +self = +dialect = +context = +statement = +parameters = [{'id': UUID('0dea1149-0bbd-4834-9fc8-c98e02aa435a'), 'source': 'Homelab', 'path': 'a.md', 'full_path': '/tmp/a.md', ...}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summa...::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at' +params = {'id': UUID('0dea1149-0bbd-4834-9fc8-c98e02aa435a'), 'source': 'Homelab', 'path': 'a.md', 'full_path': '/tmp/a.md', ...} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +clean_overview = None +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e32ca820> + + def test_regenerate_llm_error_leaves_previous_row_intact( + db: Session, clean_overview, caplog: pytest.LogCaptureFixture + ) -> None: + db.execute(text("DELETE FROM kb_overview")) + db.add(KbOverview(id=1, content="old outline")) + db.commit() + _truncate_documents(db) + try: +> _add_doc(db, "Homelab", "a.md", "A") + +tests/unit/test_overview.py:491: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_overview.py:285: in _add_doc + db.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:94: in save_obj + _emit_insert_statements( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:1234: in _emit_insert_statements + result = connection.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summa...::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at' +params = {'id': UUID('0dea1149-0bbd-4834-9fc8-c98e02aa435a'), 'source': 'Homelab', 'path': 'a.md', 'full_path': '/tmp/a.md', ...} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('0dea1149-0bbd-4834-9fc8-c98e02aa435a'), 'source': 'Homelab', 'path': 'a.md', 'full_path': '/tmp/a.md', 'title': 'A', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': None, 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +db = +clean_overview = None +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e32ca820> + + def test_regenerate_llm_error_leaves_previous_row_intact( + db: Session, clean_overview, caplog: pytest.LogCaptureFixture + ) -> None: + db.execute(text("DELETE FROM kb_overview")) + db.add(KbOverview(id=1, content="old outline")) + db.commit() + _truncate_documents(db) + try: + _add_doc(db, "Homelab", "a.md", "A") + llm = _FakeLLM(fail=LLMError("simulated lite-model failure")) + with caplog.at_level(logging.ERROR, logger="app.rag.overview"): + ok = asyncio.run(regenerate_overview(llm, db)) + assert ok is False, "fail-soft: the import must not be dragged down" + row = db.get(KbOverview, 1) + assert row is not None and row.content == "old outline", ( + "the previous outline stays — an old outline is better than none" + ) + assert "overview: regeneration failed —" in caplog.text + assert "simulated lite-model failure" in caplog.text + finally: +> _truncate_documents(db) + +tests/unit/test_overview.py:503: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_overview.py:290: in _truncate_documents + db.execute(text("TRUNCATE chunks, documents")) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('0dea1149-0bbd-4834-9fc8-c98e02aa435a'), 'source': 'Homelab', 'path': 'a.md', 'full_path': '/tmp/a.md', 'title': 'A', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': None, 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +_________________ test_regenerate_opens_own_session_when_none __________________ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT source_labels.source, source_labels.display_name, source_labels.label, source_labels.tags, source_labels.updated_at \nFROM source_labels ORDER BY source_labels.source' +params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedTable: relation "source_labels" does not exist +E LINE 2: FROM source_labels ORDER BY source_labels.source +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedTable + +The above exception was the direct cause of the following exception: + +db = +clean_overview = None + + def test_regenerate_opens_own_session_when_none(db: Session, clean_overview) -> None: + """``session=None`` → a private ``SessionLocal`` session is opened, + committed, and closed (the import-script call path).""" + _truncate_documents(db) + try: + _add_doc(db, "Homelab", "a.md", "A") +> ok = asyncio.run(regenerate_overview(_FakeLLM())) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_overview.py:527: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/overview.py:287: in regenerate_overview + labels = all_labels(session) + ^^^^^^^^^^^^^^^^^^^ +app/rag/source_labels.py:113: in all_labels + rows = db.scalars(select(SourceLabel).order_by(SourceLabel.source)).all() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT source_labels.source, source_labels.display_name, source_labels.label, source_labels.tags, source_labels.updated_at \nFROM source_labels ORDER BY source_labels.source' +params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedTable) relation "source_labels" does not exist +E LINE 2: FROM source_labels ORDER BY source_labels.source +E ^ +E [SQL: SELECT source_labels.source, source_labels.display_name, source_labels.label, source_labels.tags, source_labels.updated_at +E FROM source_labels ORDER BY source_labels.source] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError +__________ test_regenerate_model_input_never_gains_the_inventory_tail __________ + +self = +dialect = +context = +statement = +parameters = [{'id': UUID('008284d1-01ef-48ad-97db-c865cc9c7da3'), 'source': 'Deployments', 'path': 'b.yaml', 'full_path': '/tmp/b.yaml', ...}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summa...::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at' +params = {'id': UUID('008284d1-01ef-48ad-97db-c865cc9c7da3'), 'source': 'Deployments', 'path': 'b.yaml', 'full_path': '/tmp/b.yaml', ...} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +clean_overview = None + + def test_regenerate_model_input_never_gains_the_inventory_tail( + db: Session, clean_overview + ) -> None: + """The model input is byte-identical to the pre-phase prompt: the + user message the fake received equals ``build_overview_prompt`` for + the same catalogue rows — the tail is never sent to ``lite``.""" + _truncate_documents(db) + try: + _add_doc(db, "Homelab", "a.md", "A", "A summary.\nSource: Homelab/a.md") +> _add_doc(db, "Deployments", "b.yaml", "B") + +tests/unit/test_overview.py:610: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_overview.py:285: in _add_doc + db.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:94: in save_obj + _emit_insert_statements( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:1234: in _emit_insert_statements + result = connection.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summa...::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at' +params = {'id': UUID('008284d1-01ef-48ad-97db-c865cc9c7da3'), 'source': 'Deployments', 'path': 'b.yaml', 'full_path': '/tmp/b.yaml', ...} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('008284d1-01ef-48ad-97db-c865cc9c7da3'), 'source': 'Deployments', 'path': 'b.yaml', 'full_path': '/tmp/b.yaml', 'title': 'B', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': None, 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +db = +clean_overview = None + + def test_regenerate_model_input_never_gains_the_inventory_tail( + db: Session, clean_overview + ) -> None: + """The model input is byte-identical to the pre-phase prompt: the + user message the fake received equals ``build_overview_prompt`` for + the same catalogue rows — the tail is never sent to ``lite``.""" + _truncate_documents(db) + try: + _add_doc(db, "Homelab", "a.md", "A", "A summary.\nSource: Homelab/a.md") + _add_doc(db, "Deployments", "b.yaml", "B") + llm = _FakeLLM() + assert asyncio.run(regenerate_overview(llm, db)) is True + rows = db.execute( + select(Document.source, Document.path, Document.title, Document.summary) + .order_by(Document.created_at.desc(), Document.source, Document.path) + ).all() + finally: +> _truncate_documents(db) + +tests/unit/test_overview.py:618: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_overview.py:290: in _truncate_documents + db.execute(text("TRUNCATE chunks, documents")) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg.errors.UndefinedColumn) column "file_hash" of relation "documents" does not exist +E LINE 1: ...e, path, full_path, title, content, content_hash, file_hash,... +E ^ +E [SQL: INSERT INTO documents (id, source, path, full_path, title, content, content_hash, file_hash, created_at_manual, summary, is_image, image_path, image_context, page_count, page_strip_mode) VALUES (%(id)s::UUID, %(source)s::VARCHAR, %(path)s::VARCHAR, %(full_path)s::VARCHAR, %(title)s::VARCHAR, %(content)s::VARCHAR, %(content_hash)s::VARCHAR, %(file_hash)s::VARCHAR, %(created_at_manual)s, %(summary)s::VARCHAR, %(is_image)s, %(image_path)s::VARCHAR, %(image_context)s::VARCHAR, %(page_count)s::INTEGER, %(page_strip_mode)s::VARCHAR) RETURNING documents.indexed_at, documents.created_at] +E [parameters: {'id': UUID('008284d1-01ef-48ad-97db-c865cc9c7da3'), 'source': 'Deployments', 'path': 'b.yaml', 'full_path': '/tmp/b.yaml', 'title': 'B', 'content': 'body', 'content_hash': '0000000000000000000000000000000000000000000000000000000000000000', 'file_hash': None, 'created_at_manual': False, 'summary': None, 'is_image': False, 'image_path': None, 'image_context': None, 'page_count': None, 'page_strip_mode': None}] +E (Background on this error at: https://sqlalche.me/e/20/f405) (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +____________ test_regenerate_zero_docs_keeps_tailless_pre_phase_row ____________ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.source, documents.path, documents.title, documents.summary \nFROM documents ORDER BY documents.created_at DESC, documents.source, documents.path' +params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.created_at does not exist +E LINE 2: FROM documents ORDER BY documents.created_at DESC, documents... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +clean_overview = None + + def test_regenerate_zero_docs_keeps_tailless_pre_phase_row( + db: Session, clean_overview + ) -> None: + """Zero-doc early return: byte-identical pre-phase behavior — the + pre-phase (tail-less) row survives untouched, no tail is invented.""" + db.execute(text("DELETE FROM kb_overview")) + pre_phase_row = "- Old outline\n- from before phase 162" + db.add(KbOverview(id=1, content=pre_phase_row)) + db.commit() + _truncate_documents(db) + llm = _FakeLLM() +> ok = asyncio.run(regenerate_overview(llm, db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_overview.py:637: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/overview.py:266: in regenerate_overview + result = session.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.source, documents.path, documents.title, documents.summary \nFROM documents ORDER BY documents.created_at DESC, documents.source, documents.path' +params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.created_at does not exist +E LINE 2: FROM documents ORDER BY documents.created_at DESC, documents... +E ^ +E [SQL: SELECT documents.source, documents.path, documents.title, documents.summary +E FROM documents ORDER BY documents.created_at DESC, documents.source, documents.path] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError +________ test_regenerate_llm_error_preserves_an_existing_inventory_tail ________ + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT source_labels.source, source_labels.display_name, source_labels.label, source_labels.tags, source_labels.updated_at \nFROM source_labels ORDER BY source_labels.source' +params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedTable: relation "source_labels" does not exist +E LINE 2: FROM source_labels ORDER BY source_labels.source +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedTable + +The above exception was the direct cause of the following exception: + +db = +clean_overview = None + + def test_regenerate_llm_error_preserves_an_existing_inventory_tail( + db: Session, clean_overview + ) -> None: + """The tail is CONTENT: a previous row that already carried a + ``Sources: …`` tail survives an ``LLMError`` byte-identically — + including the tail (an old outline is better than none).""" + db.execute(text("DELETE FROM kb_overview")) + previous = "- Old outline\nSources: Homelab (3 docs)" + db.add(KbOverview(id=1, content=previous)) + db.commit() + _truncate_documents(db) + try: + _add_doc(db, "Homelab", "a.md", "A") + llm = _FakeLLM(fail=LLMError("simulated lite-model failure")) +> ok = asyncio.run(regenerate_overview(llm, db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_overview.py:659: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/overview.py:287: in regenerate_overview + labels = all_labels(session) + ^^^^^^^^^^^^^^^^^^^ +app/rag/source_labels.py:113: in all_labels + rows = db.scalars(select(SourceLabel).order_by(SourceLabel.source)).all() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT source_labels.source, source_labels.display_name, source_labels.label, source_labels.tags, source_labels.updated_at \nFROM source_labels ORDER BY source_labels.source' +params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedTable) relation "source_labels" does not exist +E LINE 2: FROM source_labels ORDER BY source_labels.source +E ^ +E [SQL: SELECT source_labels.source, source_labels.display_name, source_labels.label, source_labels.tags, source_labels.updated_at +E FROM source_labels ORDER BY source_labels.source] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'TRUNCATE chunks, documents', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +clean_overview = None + + def test_regenerate_llm_error_preserves_an_existing_inventory_tail( + db: Session, clean_overview + ) -> None: + """The tail is CONTENT: a previous row that already carried a + ``Sources: …`` tail survives an ``LLMError`` byte-identically — + including the tail (an old outline is better than none).""" + db.execute(text("DELETE FROM kb_overview")) + previous = "- Old outline\nSources: Homelab (3 docs)" + db.add(KbOverview(id=1, content=previous)) + db.commit() + _truncate_documents(db) + try: + _add_doc(db, "Homelab", "a.md", "A") + llm = _FakeLLM(fail=LLMError("simulated lite-model failure")) + ok = asyncio.run(regenerate_overview(llm, db)) + assert ok is False + row = db.get(KbOverview, 1) + assert row is not None and row.content == previous + finally: +> _truncate_documents(db) + +tests/unit/test_overview.py:664: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_overview.py:290: in _truncate_documents + db.execute(text("TRUNCATE chunks, documents")) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2373: in execute + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2280: in _execute_internal + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'TRUNCATE chunks, documents', params = {}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: TRUNCATE chunks, documents] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +___________ test_pdf_becomes_a_document_with_text_chunks_and_summary ___________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfsrc', 'path_1': 'report.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfsrc', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_becomes_a_document_wi0') + + def test_pdf_becomes_a_document_with_text_chunks_and_summary( + db, tmp_path: Path + ) -> None: + """The primary path: a PDF in a source → sync → the document's + ``content`` is the extracted text (the ``---PAGE-N---`` markers, + A2), the page-aware chunks carry the ``Page N:`` anchor, every + chunk (content + summary) is embedded, the title is the metadata + title, and the phase-30/118 summary is stored.""" + root = tmp_path / "pdfsrc" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("First page text about networking.", []), + ("Second page text about storage.", [])], + title="Quarterly Report", + ) + llm = _pdf_llm(tmp_path) + try: +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:279: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfsrc', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfsrc', 'path_1': 'report.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfsrc'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfsrc'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_becomes_a_document_wi0') + + def test_pdf_becomes_a_document_with_text_chunks_and_summary( + db, tmp_path: Path + ) -> None: + """The primary path: a PDF in a source → sync → the document's + ``content`` is the extracted text (the ``---PAGE-N---`` markers, + A2), the page-aware chunks carry the ``Page N:`` anchor, every + chunk (content + summary) is embedded, the title is the metadata + title, and the phase-30/118 summary is stored.""" + root = tmp_path / "pdfsrc" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("First page text about networking.", []), + ("Second page text about storage.", [])], + title="Quarterly Report", + ) + llm = _pdf_llm(tmp_path) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added) == (1, 1) + assert summary.formats == {"pdf": 1} + + doc = _doc(db, "pdfsrc", "report.pdf") + assert doc.title == "Quarterly Report" + assert doc.is_image is False and doc.image_path is None + assert doc.content == ( + f"{PAGE_SEPARATOR.format(page=1)}\nFirst page text about networking.\n\n" + f"{PAGE_SEPARATOR.format(page=2)}\nSecond page text about storage." + ) + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + content_chunks = [c for c in chunks if not c.is_summary] + assert [c.content for c in content_chunks] == [ + "Page 1:\nFirst page text about networking.", + "Page 2:\nSecond page text about storage.", + ] + for c in chunks: + assert c.embedding is not None and len(c.embedding) == 768 + summary_chunks = [c for c in chunks if c.is_summary] + assert len(summary_chunks) == 1 and summary_chunks[0].position == -1 + assert doc.summary is not None + assert doc.summary.endswith("Source: pdfsrc/report.pdf") + finally: +> _cleanup_source(db, "pdfsrc") + +tests/unit/test_pdf.py:305: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfsrc'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfsrc'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +_______________ test_pdf_embedded_images_become_image_documents ________________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfimg', 'path_1': 'report.pdf#page=2#img=0.png'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfimg', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_images_becom0') + + def test_pdf_embedded_images_become_image_documents(db, tmp_path: Path) -> None: + """A3 with the toggle ON: each embedded image joins the phase-122 + pipeline — the single seam describes it (the CHAT model), the copy + lands in ``image_dir`` named by the doc id, and the doc row carries + the synthetic ``#page=#img=`` path, ``is_image`` + + ``image_path``, and the enriched content (phase 137, task 02: the + document-context block FIRST, then the description — the summary + chunk mirrors it, the phase-122 convention).""" + root = tmp_path / "pdfimg" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [png])]) + llm = _pdf_llm(tmp_path) + try: +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:322: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1609: in _index_file + return await _index_pdf_file( +app/rag/importer.py:2143: in _index_pdf_file + await _index_pdf_embedded_images( +app/rag/importer.py:2742: in _index_pdf_embedded_images + await _index_one_embedded_image( +app/rag/importer.py:2786: in _index_one_embedded_image + image_doc = session.scalar(_doc_row_stmt(source, image_rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfimg', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfimg', 'path_1': 'report.pdf#page=2#img=0.png'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfimg'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfimg'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_images_becom0') + + def test_pdf_embedded_images_become_image_documents(db, tmp_path: Path) -> None: + """A3 with the toggle ON: each embedded image joins the phase-122 + pipeline — the single seam describes it (the CHAT model), the copy + lands in ``image_dir`` named by the doc id, and the doc row carries + the synthetic ``#page=#img=`` path, ``is_image`` + + ``image_path``, and the enriched content (phase 137, task 02: the + document-context block FIRST, then the description — the summary + chunk mirrors it, the phase-122 convention).""" + root = tmp_path / "pdfimg" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [png])]) + llm = _pdf_llm(tmp_path) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + # The PDF + its one embedded image: both count added. + assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0) + + # Phase 137 (task 02): the document context — the origin line + + # page 2's text — is the stored ``image_context`` and the + # ``content``/``summary`` enrichment (context first, then the + # description — the text the embedder indexes carries it). + expected_context = ( + f"Extracted from {root.name}/report.pdf (page 2):\nText page two." + ) + expected_content = f"{expected_context}\n\n{DESCRIPTION}" + + image_rel = "report.pdf#page=2#img=0.png" + img = _doc(db, "pdfimg", image_rel) + assert img.is_image is True and img.image_path is not None + assert img.title == "Embedded image from report page 2" + assert img.image_context == expected_context + assert img.content == expected_content and img.summary == expected_content + copy = Path(img.image_path) + assert copy.parent == Path(llm.settings.image_dir).expanduser() + assert copy.name == f"{img.id}.png" # row and copy agree on the id + assert copy.read_bytes() == png + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == img.id).order_by(Chunk.position) + ).all() + assert all(c.embedding is not None and len(c.embedding) == 768 for c in chunks) + summary_chunks = [c for c in chunks if c.is_summary] + assert len(summary_chunks) == 1 and summary_chunks[0].content == expected_content + + # The vision describe call went to the CHAT model (the PDF's own + # lite summary is the first chat call — the phase-30 path). + assert llm.chat_models == [ + llm.settings.llm_summary_model, llm.settings.llm_chat_model, + ] + finally: +> _cleanup_source(db, "pdfimg") + +tests/unit/test_pdf.py:358: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfimg'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfimg'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:54:01 INFO app.importer :: import: added source=pdfimg path=report.pdf chunks=2 +2026-10-19 11:54:01 INFO app.importer :: import: summary source=pdfimg path=report.pdf chars=49 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:2128 import: added source=pdfimg path=report.pdf chunks=2 +INFO app.importer:importer.py:1784 import: summary source=pdfimg path=report.pdf chars=49 +_______________ test_pdf_embedded_images_skipped_when_images_off _______________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfoff', 'path_1': 'report.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfoff', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_images_skipp0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e4120fa0> + + def test_pdf_embedded_images_skipped_when_images_off( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """The toggle OFF: the PDF text indexes normally, the embedded + images are skipped (one debug line, NO warning, NO + ``images_failed`` — a skip is not a failure), no image doc, no + copy — and not even the image dir.""" + root = tmp_path / "pdfoff" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", [_png_bytes(40, 30)]), ("Text page two.", [])], + ) + llm = _pdf_llm(tmp_path, images=False) + try: + with caplog.at_level(logging.DEBUG, logger="app.importer"): +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:377: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfoff', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfoff', 'path_1': 'report.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfoff'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfoff'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_images_skipp0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e4120fa0> + + def test_pdf_embedded_images_skipped_when_images_off( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """The toggle OFF: the PDF text indexes normally, the embedded + images are skipped (one debug line, NO warning, NO + ``images_failed`` — a skip is not a failure), no image doc, no + copy — and not even the image dir.""" + root = tmp_path / "pdfoff" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", [_png_bytes(40, 30)]), ("Text page two.", [])], + ) + llm = _pdf_llm(tmp_path, images=False) + try: + with caplog.at_level(logging.DEBUG, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0) + doc = _doc(db, "pdfoff", "report.pdf") + assert "Text page one." in doc.content and doc.summary is not None + assert ( + db.scalar( + select(Document).where( + Document.source == "pdfoff", Document.is_image.is_(True) + ) + ) + is None + ), "toggle off: no image documents" + assert not Path(llm.settings.image_dir).expanduser().exists(), ( + "no copy — and not even the image dir — may be left behind" + ) + # The skip is debug-logged (not a warning, not a failure). + debugs = [ + r for r in caplog.records if "images toggle off" in r.getMessage() + ] + assert len(debugs) == 1 and debugs[0].levelno == logging.DEBUG + assert not [ + r for r in caplog.records if "description failed" in r.getMessage() + ] + # No vision call at all — only the PDF's lite summary. + assert llm.chat_models == [llm.settings.llm_summary_model] + finally: +> _cleanup_source(db, "pdfoff") + +tests/unit/test_pdf.py:403: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfoff'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfoff'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +__________________ test_pdf_unchanged_rerun_skips_everything ___________________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfonce', 'path_1': 'report.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfonce', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_unchanged_rerun_skips0') + + def test_pdf_unchanged_rerun_skips_everything(db, tmp_path: Path) -> None: + """The unchanged path (same text digest) is the text path's rule: + no re-extract, no re-embed, NO image pass — the second sync costs + no vision and no embed calls at all.""" + root = tmp_path / "pdfonce" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", [_png_bytes(40, 30)]), ("Text page two.", [])], + ) + llm = _pdf_llm(tmp_path) + try: +> s1 = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:418: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfonce', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfonce', 'path_1': 'report.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfonce'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfonce'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_unchanged_rerun_skips0') + + def test_pdf_unchanged_rerun_skips_everything(db, tmp_path: Path) -> None: + """The unchanged path (same text digest) is the text path's rule: + no re-extract, no re-embed, NO image pass — the second sync costs + no vision and no embed calls at all.""" + root = tmp_path / "pdfonce" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", [_png_bytes(40, 30)]), ("Text page two.", [])], + ) + llm = _pdf_llm(tmp_path) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert (s1.added, s1.unchanged) == (2, 0) + calls1, chat1 = len(llm.calls), len(llm.chat_models) + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.files, s2.added, s2.updated, s2.unchanged) == (1, 0, 0, 1) + assert len(llm.calls) == calls1, "unchanged: no re-embed" + assert len(llm.chat_models) == chat1, "unchanged: no vision/summary calls" + finally: +> _cleanup_source(db, "pdfonce") + +tests/unit/test_pdf.py:426: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfonce'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfonce'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +___________ test_pdf_unchanged_path_backfills_and_honors_manual_date ___________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfdate', 'path_1': 'report.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfdate', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_unchanged_path_backfi0') + + def test_pdf_unchanged_path_backfills_and_honors_manual_date( + db, tmp_path: Path + ) -> None: + """The unchanged path's phase-118/D1/D4 dance, PDF flavour: a + NULL-summary unchanged doc gets the summary backfill (counted + ``summary_backfilled``, never ``summaries``); a manual-date row + (D1) is not touched at all (the backfill runs BEFORE the + early-return — the flag protects the DATE only); and with the flag + off the stored date REFRESHES from the file's mtime (D4 — counted + ``dates_updated``, still ``unchanged``).""" + root = tmp_path / "pdfdate" + root.mkdir() + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [])]) + llm = _pdf_llm(tmp_path, images=False) + try: +> s1 = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:444: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfdate', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfdate', 'path_1': 'report.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfdate'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfdate'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_unchanged_path_backfi0') + + def test_pdf_unchanged_path_backfills_and_honors_manual_date( + db, tmp_path: Path + ) -> None: + """The unchanged path's phase-118/D1/D4 dance, PDF flavour: a + NULL-summary unchanged doc gets the summary backfill (counted + ``summary_backfilled``, never ``summaries``); a manual-date row + (D1) is not touched at all (the backfill runs BEFORE the + early-return — the flag protects the DATE only); and with the flag + off the stored date REFRESHES from the file's mtime (D4 — counted + ``dates_updated``, still ``unchanged``).""" + root = tmp_path / "pdfdate" + root.mkdir() + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [])]) + llm = _pdf_llm(tmp_path, images=False) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert s1.added == 1 + doc = _doc(db, "pdfdate", "report.pdf") + + # Run 2 — simulate a pre-phase-30 row: NULL summary + a manual + # date correction. + doc.summary = None + doc.created_at_manual = True + doc.created_at = doc.created_at - timedelta(days=30) + db.commit() + manual_date = doc.created_at + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.unchanged, s2.summary_backfilled, s2.summaries) == (1, 1, 0) + assert s2.dates_updated == 0, "the manual date must survive the sync" + doc2 = _doc(db, "pdfdate", "report.pdf") + assert doc2.created_at_manual is True and doc2.created_at == manual_date + assert doc2.summary is not None, "the backfill restored the summary" + + # Run 3 — the manual flag off: the date REFRESHES from the + # file's mtime (D4) — and a date-only refresh is still + # unchanged (no KB-change count). + doc2.created_at_manual = False + doc2.created_at = doc2.created_at - timedelta(days=60) + db.commit() + s3 = asyncio.run(import_sources([root], llm, session=db)) + assert (s3.unchanged, s3.dates_updated, s3.added, s3.updated) == (1, 1, 0, 0) + doc3 = _doc(db, "pdfdate", "report.pdf") + assert doc3.created_at != manual_date + finally: +> _cleanup_source(db, "pdfdate") + +tests/unit/test_pdf.py:473: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfdate'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfdate'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +_________ test_pdf_text_change_updates_doc_and_keeps_unchanged_images __________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfchg', 'path_1': 'report.pdf#page=2#img=0.png'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfchg', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_text_change_updates_d0') + + def test_pdf_text_change_updates_doc_and_keeps_unchanged_images( + db, tmp_path: Path + ) -> None: + """A text change re-indexes the PDF (updated) and re-runs the image + pass — but an embedded image whose BYTES are unchanged (same digest) + is skipped: no re-describe (no new vision call for it), the doc id + and the copy path are stable.""" + root = tmp_path / "pdfchg" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf( + root / "report.pdf", + [("Original first page.", []), ("Second page.", [png])], + ) + llm = _pdf_llm(tmp_path) + try: +> s1 = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:492: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1609: in _index_file + return await _index_pdf_file( +app/rag/importer.py:2143: in _index_pdf_file + await _index_pdf_embedded_images( +app/rag/importer.py:2742: in _index_pdf_embedded_images + await _index_one_embedded_image( +app/rag/importer.py:2786: in _index_one_embedded_image + image_doc = session.scalar(_doc_row_stmt(source, image_rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfchg', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfchg', 'path_1': 'report.pdf#page=2#img=0.png'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfchg'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfchg'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_text_change_updates_d0') + + def test_pdf_text_change_updates_doc_and_keeps_unchanged_images( + db, tmp_path: Path + ) -> None: + """A text change re-indexes the PDF (updated) and re-runs the image + pass — but an embedded image whose BYTES are unchanged (same digest) + is skipped: no re-describe (no new vision call for it), the doc id + and the copy path are stable.""" + root = tmp_path / "pdfchg" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf( + root / "report.pdf", + [("Original first page.", []), ("Second page.", [png])], + ) + llm = _pdf_llm(tmp_path) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert s1.added == 2 + image_rel = "report.pdf#page=2#img=0.png" + img1 = _doc(db, "pdfchg", image_rel) + + # Same image bytes, changed page-1 text. + (root / "report.pdf").unlink() + _build_pdf( + root / "report.pdf", + [("Revised first page.", []), ("Second page.", [png])], + ) + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.added, s2.updated, s2.unchanged) == (0, 1, 0) + + img2 = _doc(db, "pdfchg", image_rel) + assert img2.id == img1.id and img2.image_path == img1.image_path + assert img2.image_path is not None + assert Path(img2.image_path).read_bytes() == png + # The lite summary of the changed PDF fired again — but NO new + # describe call (the image bytes are unchanged). + assert llm.chat_models == [ + llm.settings.llm_summary_model, # run 1: PDF summary + llm.settings.llm_chat_model, # run 1: the image description + llm.settings.llm_summary_model, # run 2: the revised PDF summary + ] + finally: +> _cleanup_source(db, "pdfchg") + +tests/unit/test_pdf.py:518: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfchg'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfchg'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:54:04 INFO app.importer :: import: added source=pdfchg path=report.pdf chunks=2 +2026-10-19 11:54:04 INFO app.importer :: import: summary source=pdfchg path=report.pdf chars=49 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:2128 import: added source=pdfchg path=report.pdf chunks=2 +INFO app.importer:importer.py:1784 import: summary source=pdfchg path=report.pdf chars=49 +_______________ test_corrupt_pdf_file_becomes_an_empty_document ________________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfbad', 'path_1': 'broken.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfbad', 'path_1': 'broken.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_corrupt_pdf_file_becomes_0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e4116820> + + def test_corrupt_pdf_file_becomes_an_empty_document( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """A .pdf file that is not a PDF at all: the parser's fail-soft + contract (no text, no images) AND the phase-133 fallback's + render-less branch (``render_pdf_pages`` → ``[]`` → the helper's + ``""``) — the document lands with empty content (the empty byte + string's digest) and no chunks, the fallback is still counted + (the gate fired, the render produced nothing), and the phase-126 + "no extractable text" warning fires — the doc really is empty.""" + root = tmp_path / "pdfbad" + root.mkdir() + (root / "broken.pdf").write_bytes(b"this is not a pdf at all") + llm = _pdf_llm(tmp_path) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:584: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfbad', 'path_1': 'broken.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfbad', 'path_1': 'broken.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfbad'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfbad'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_corrupt_pdf_file_becomes_0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e4116820> + + def test_corrupt_pdf_file_becomes_an_empty_document( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """A .pdf file that is not a PDF at all: the parser's fail-soft + contract (no text, no images) AND the phase-133 fallback's + render-less branch (``render_pdf_pages`` → ``[]`` → the helper's + ``""``) — the document lands with empty content (the empty byte + string's digest) and no chunks, the fallback is still counted + (the gate fired, the render produced nothing), and the phase-126 + "no extractable text" warning fires — the doc really is empty.""" + root = tmp_path / "pdfbad" + root.mkdir() + (root / "broken.pdf").write_bytes(b"this is not a pdf at all") + llm = _pdf_llm(tmp_path) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + assert summary.pdfs_screenshotted == 1 # the gate fired + doc = _doc(db, "pdfbad", "broken.pdf") + assert doc.content == "" and doc.title == "broken" + assert doc.content_hash == hashlib.sha256(b"").hexdigest() + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + assert [c for c in chunks if not c.is_summary] == [] + warnings = [ + r for r in caplog.records if "no extractable text" in r.getMessage() + ] + assert len(warnings) == 1 and warnings[0].levelno == logging.WARNING + finally: +> _cleanup_source(db, "pdfbad") + +tests/unit/test_pdf.py:599: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfbad'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfbad'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +_______ test_pdf_embedded_image_failed_description_skips_image_keeps_pdf _______ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdffail', 'path_1': 'report.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdffail', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_failed0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e31c5a90> + + def test_pdf_embedded_image_failed_description_skips_image_keeps_pdf( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """LOCKED A3 fail-soft: a failed description skips the IMAGE + entirely (no row, no copy — and not even the image dir), counts + ``images_failed`` + a warning, and the PDF's text index is + unaffected (already committed).""" + root = tmp_path / "pdffail" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", []), ("Text page two.", [_png_bytes(40, 30)])], + ) + llm = _pdf_llm(tmp_path, _VisionFailsLLM) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:669: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdffail', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdffail', 'path_1': 'report.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdffail'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdffail'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_failed0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e31c5a90> + + def test_pdf_embedded_image_failed_description_skips_image_keeps_pdf( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """LOCKED A3 fail-soft: a failed description skips the IMAGE + entirely (no row, no copy — and not even the image dir), counts + ``images_failed`` + a warning, and the PDF's text index is + unaffected (already committed).""" + root = tmp_path / "pdffail" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", []), ("Text page two.", [_png_bytes(40, 30)])], + ) + llm = _pdf_llm(tmp_path, _VisionFailsLLM) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 1, 1) + doc = _doc(db, "pdffail", "report.pdf") + assert "Text page one." in doc.content and doc.summary is not None + assert ( + db.scalar( + select(Document).where( + Document.source == "pdffail", Document.is_image.is_(True) + ) + ) + is None + ), "a failed description must leave NO image row" + assert not Path(llm.settings.image_dir).expanduser().exists(), ( + "no copy — and not even the image dir — may be left behind" + ) + warnings = [ + r + for r in caplog.records + if "embedded image description failed" in r.getMessage() + ] + assert len(warnings) == 1 and warnings[0].levelno == logging.WARNING + finally: +> _cleanup_source(db, "pdffail") + +tests/unit/test_pdf.py:691: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdffail'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdffail'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +____________ test_pdf_embedded_image_embed_failure_keeps_the_others ____________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfemb', 'path_1': 'report.pdf#page=2#img=0.png'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfemb', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_embed_0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e40d4f30> + + def test_pdf_embedded_image_embed_failure_keeps_the_others( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """A per-image embed failure rolls back THAT image only (no row, + no orphan copy), counts ``errors`` + the error line, and the pass + moves on — the second image is still indexed, and the PDF's text + index (committed earlier) is untouched.""" + root = tmp_path / "pdfemb" + root.mkdir() + _build_pdf( + root / "report.pdf", + [ + ("Text page one.", []), + ("Text page two.", [_png_bytes(40, 30)]), + ("Text page three.", [_png_bytes(30, 20)]), + ], + ) + llm = _pdf_llm(tmp_path, _FirstImageEmbedFailsLLM) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:714: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1609: in _index_file + return await _index_pdf_file( +app/rag/importer.py:2143: in _index_pdf_file + await _index_pdf_embedded_images( +app/rag/importer.py:2742: in _index_pdf_embedded_images + await _index_one_embedded_image( +app/rag/importer.py:2786: in _index_one_embedded_image + image_doc = session.scalar(_doc_row_stmt(source, image_rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfemb', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfemb', 'path_1': 'report.pdf#page=2#img=0.png'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfemb'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfemb'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_embed_0') +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f52e40d4f30> + + def test_pdf_embedded_image_embed_failure_keeps_the_others( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """A per-image embed failure rolls back THAT image only (no row, + no orphan copy), counts ``errors`` + the error line, and the pass + moves on — the second image is still indexed, and the PDF's text + index (committed earlier) is untouched.""" + root = tmp_path / "pdfemb" + root.mkdir() + _build_pdf( + root / "report.pdf", + [ + ("Text page one.", []), + ("Text page two.", [_png_bytes(40, 30)]), + ("Text page three.", [_png_bytes(30, 20)]), + ], + ) + llm = _pdf_llm(tmp_path, _FirstImageEmbedFailsLLM) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + # PDF + the surviving second image; the first image's embed failed. + assert (summary.added, summary.errors, summary.images_failed) == (2, 1, 0) + errors = [ + r for r in caplog.records if "embed failed" in r.getMessage() + ] + assert len(errors) == 1 and errors[0].levelno == logging.ERROR + + # Only the second image (page 3) landed. + img = _doc(db, "pdfemb", "report.pdf#page=3#img=0.png") + assert img.is_image is True + assert ( + db.scalar( + select(Document).where( + Document.source == "pdfemb", + Document.path == "report.pdf#page=2#img=0.png", + ) + ) + is None + ) + # No orphan copy of the failed image: exactly one file in the dir. + copies = list(Path(llm.settings.image_dir).expanduser().iterdir()) + assert len(copies) == 1 and copies[0].name == f"{img.id}.png" + finally: +> _cleanup_source(db, "pdfemb") + +tests/unit/test_pdf.py:738: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfemb'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfemb'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:54:06 INFO app.importer :: import: added source=pdfemb path=report.pdf chunks=3 +2026-10-19 11:54:07 INFO app.importer :: import: summary source=pdfemb path=report.pdf chars=49 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:2128 import: added source=pdfemb path=report.pdf chunks=3 +INFO app.importer:importer.py:1784 import: summary source=pdfemb path=report.pdf chars=49 +______________ test_pdf_changed_embedded_image_replaces_the_copy _______________ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfrep', 'path_1': 'report.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfrep', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_changed_embedded_imag0') + + def test_pdf_changed_embedded_image_replaces_the_copy(db, tmp_path: Path) -> None: + """A PDF whose text AND an embedded image's bytes changed: the PDF + is ``updated`` and the image doc is ``updated`` in place — same id, + the stale copy is deleted before the new one is written (one file, + the new bytes), and the image is re-described (a second vision + call).""" + root = tmp_path / "pdfrep" + root.mkdir() + png_a = _png_bytes(40, 30) + png_b = _png_bytes(25, 18) # different size → different bytes + _build_pdf( + root / "report.pdf", + [("Original first page.", []), ("Second page.", [png_a])], + ) + llm = _pdf_llm(tmp_path) + try: +> s1 = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:757: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfrep', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfrep', 'path_1': 'report.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfrep'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfrep'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_changed_embedded_imag0') + + def test_pdf_changed_embedded_image_replaces_the_copy(db, tmp_path: Path) -> None: + """A PDF whose text AND an embedded image's bytes changed: the PDF + is ``updated`` and the image doc is ``updated`` in place — same id, + the stale copy is deleted before the new one is written (one file, + the new bytes), and the image is re-described (a second vision + call).""" + root = tmp_path / "pdfrep" + root.mkdir() + png_a = _png_bytes(40, 30) + png_b = _png_bytes(25, 18) # different size → different bytes + _build_pdf( + root / "report.pdf", + [("Original first page.", []), ("Second page.", [png_a])], + ) + llm = _pdf_llm(tmp_path) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert s1.added == 2 + image_rel = "report.pdf#page=2#img=0.png" + img1 = _doc(db, "pdfrep", image_rel) + assert img1.image_path is not None + copy1 = Path(img1.image_path) + + # Changed page-1 text AND changed image bytes. + (root / "report.pdf").unlink() + _build_pdf( + root / "report.pdf", + [("Revised first page.", []), ("Second page.", [png_b])], + ) + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.added, s2.updated, s2.unchanged) == (0, 2, 0) + + img2 = _doc(db, "pdfrep", image_rel) + assert img2.id == img1.id, "the changed image keeps its doc id" + assert img2.image_path == img1.image_path, "the copy path is stable" + assert img2.image_path is not None + assert Path(img2.image_path).read_bytes() == png_b + # Exactly one copy in the dir — the stale one was deleted. + copies = list(Path(llm.settings.image_dir).expanduser().iterdir()) + assert len(copies) == 1 and copies[0] == copy1 + # The re-describe fired (a second vision call after run 2's lite + # summary of the revised PDF). + assert llm.chat_models == [ + llm.settings.llm_summary_model, # run 1: PDF summary + llm.settings.llm_chat_model, # run 1: the image description + llm.settings.llm_summary_model, # run 2: the revised PDF summary + llm.settings.llm_chat_model, # run 2: the re-description + ] + finally: +> _cleanup_source(db, "pdfrep") + +tests/unit/test_pdf.py:790: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfrep'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfrep'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +_______ test_pdf_embedded_image_vision_call_carries_the_document_context _______ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfctx', 'path_1': 'report.pdf'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfctx', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_vision0') + + def test_pdf_embedded_image_vision_call_carries_the_document_context( + db, tmp_path: Path + ) -> None: + """THE forward path (the task's core proof): a PDF with page text + + one embedded image → the vision message's text part is + ``DESCRIBE_PROMPT`` + the (now-neutral) prefix + the exact + document-context block (the prefix bytes asserted explicitly), the + stored ``image_context`` is the SAME block, and + ``content``/``summary`` are ``\\n\\n`` — the + text the embedder indexes carries the context. A second (unchanged) + sync re-describes NOTHING (the phase-126 digest skip is + untouched).""" + root = tmp_path / "pdfctx" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [png])]) + llm = _pdf_llm(tmp_path, _CapturingVisionLLM) + try: +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:928: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1566: in _index_file + doc = session.scalar(_doc_row_stmt(source, rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdfctx', 'path_1': 'report.pdf'}, prepare = None +binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.file_hash does not exist +E LINE 1: ...ll_path, documents.title, documents.content_hash, documents.... +E ^ +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfctx', 'path_1': 'report.pdf'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdfctx'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfctx'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_vision0') + + def test_pdf_embedded_image_vision_call_carries_the_document_context( + db, tmp_path: Path + ) -> None: + """THE forward path (the task's core proof): a PDF with page text + + one embedded image → the vision message's text part is + ``DESCRIBE_PROMPT`` + the (now-neutral) prefix + the exact + document-context block (the prefix bytes asserted explicitly), the + stored ``image_context`` is the SAME block, and + ``content``/``summary`` are ``\\n\\n`` — the + text the embedder indexes carries the context. A second (unchanged) + sync re-describes NOTHING (the phase-126 digest skip is + untouched).""" + root = tmp_path / "pdfctx" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [png])]) + llm = _pdf_llm(tmp_path, _CapturingVisionLLM) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0) + + expected_block = f"Extracted from {root.name}/report.pdf (page 2):\nText page two." + # (a) the vision model received the context — the locked prompt + # + the exact generalized prefix + the block. + assert llm.text_parts == [DESCRIBE_PROMPT + NEUTRAL_CONTEXT_PREFIX + expected_block] + assert llm.text_parts[0].startswith(DESCRIBE_PROMPT), "the locked prompt is untouched" + # (b) the stored image_context IS the same combined value. + img = _doc(db, root.name, "report.pdf#page=2#img=0.png") + assert img.image_context == expected_block + # (c) content/summary carry it too — context first, then the + # description (the embedder indexes the same text). + expected_content = f"{expected_block}\n\n{DESCRIPTION}" + assert img.content == expected_content + assert img.summary == expected_content + chunks = db.scalars(select(Chunk).where(Chunk.document_id == img.id)).all() + packed = "\n".join(expected_content.split("\n\n")) + assert [c.content for c in chunks if not c.is_summary] == [packed] + assert [c.content for c in chunks if c.is_summary] == [expected_content] + + # The unchanged-image digest skip is untouched: a re-sync of the + # same PDF re-describes nothing (no new chat call at all — the + # unchanged path skips the image pass entirely). + calls1 = len(llm.chat_calls) + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1) + assert len(llm.chat_calls) == calls1, "unchanged: no re-describe" + finally: +> _cleanup_source(db, root.name) + +tests/unit/test_pdf.py:957: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdfctx'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdfctx'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +_____ test_pdf_embedded_image_textless_page_still_carries_the_origin_line ______ + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdftextless', 'path_1': 'report.pdf#page=2#img=0.png'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdftextless', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.UndefinedColumn: column documents.page_count does not exist at character 298 + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: UndefinedColumn + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_textle0') + + def test_pdf_embedded_image_textless_page_still_carries_the_origin_line( + db, tmp_path: Path + ) -> None: + """The owner's core complaint, pinned at the real seam: an image + on a TEXT-LESS page (page 1 carries text, so the layer is usable — + this is the text-less-PAGE case, not the garbage-LAYER case) → the + block is the origin line ONLY, and it is STILL stored + enriched + (the provenance never depends on the page having text).""" + root = tmp_path / "pdftextless" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf(root / "report.pdf", [("Text page one.", []), ("", [png])]) + llm = _pdf_llm(tmp_path, _CapturingVisionLLM) + try: +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pdf.py:1020: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1609: in _index_file + return await _index_pdf_file( +app/rag/importer.py:2143: in _index_pdf_file + await _index_pdf_embedded_images( +app/rag/importer.py:2742: in _index_pdf_embedded_images + await _index_one_embedded_image( +app/rag/importer.py:2786: in _index_one_embedded_image + image_doc = session.scalar(_doc_row_stmt(source, image_rel)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2421: in scalar + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, ..._strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR' +params = {'source_1': 'pdftextless', 'path_1': 'report.pdf#page=2#img=0.png'} +prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.ProgrammingError: (psycopg.errors.UndefinedColumn) column documents.page_count does not exist at character 298 +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR AND documents.path = %(path_1)s::VARCHAR] +E [parameters: {'source_1': 'pdftextless', 'path_1': 'report.pdf#page=2#img=0.png'}] +E (Background on this error at: https://sqlalche.me/e/20/f405) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: ProgrammingError + +During handling of the above exception, another exception occurred: + +self = +dialect = +context = +statement = +parameters = [{'source_1': 'pdftextless'}] + + def _exec_single_context( + self, + dialect: Dialect, + context: ExecutionContext, + statement: Union[str, Compiled], + parameters: Optional[_AnyMultiExecuteParams], + ) -> CursorResult[Any]: + """continue the _execute_context() method for a single DBAPI + cursor.execute() or cursor.executemany() call. + + """ + if dialect.bind_typing is BindTyping.SETINPUTSIZES: + generic_setinputsizes = context._prepare_set_input_sizes() + + if generic_setinputsizes: + try: + dialect.do_set_input_sizes( + context.cursor, generic_setinputsizes, context + ) + except BaseException as e: + self._handle_dbapi_exception( + e, str(statement), parameters, None, context + ) + + cursor, str_statement, parameters = ( + context.cursor, + context.statement, + context.parameters, + ) + + effective_parameters: Optional[_AnyExecuteParams] + + if not context.executemany: + effective_parameters = parameters[0] + else: + effective_parameters = parameters + + if self._has_events or self.engine._has_events: + for fn in self.dispatch.before_cursor_execute: + str_statement, effective_parameters = fn( + self, + cursor, + str_statement, + effective_parameters, + context, + context.executemany, + ) + + if self._echo: + self._log_info(str_statement) + + stats = context._get_cache_stats() + + if not self.engine.hide_parameters: + self._log_info( + "[%s] %r", + stats, + sql_util._repr_params( + effective_parameters, + batches=10, + ismulti=context.executemany, + ), + ) + else: + self._log_info( + "[%s] [SQL parameters hidden due to hide_parameters=True]", + stats, + ) + + evt_handled: bool = False + try: + if context.execute_style is ExecuteStyle.EXECUTEMANY: + effective_parameters = cast( + "_CoreMultiExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_executemany: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: + self.dialect.do_executemany( + cursor, + str_statement, + effective_parameters, + context, + ) + elif not effective_parameters and context.no_parameters: + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute_no_params: + if fn(cursor, str_statement, context): + evt_handled = True + break + if not evt_handled: + self.dialect.do_execute_no_params( + cursor, str_statement, context + ) + else: + effective_parameters = cast( + "_CoreSingleExecuteParams", effective_parameters + ) + if self.dialect._has_events: + for fn in self.dialect.dispatch.do_execute: + if fn( + cursor, + str_statement, + effective_parameters, + context, + ): + evt_handled = True + break + if not evt_handled: +> self.dialect.do_execute( + cursor, str_statement, effective_parameters, context + ) + +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdftextless'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E psycopg.errors.InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InFailedSqlTransaction + +The above exception was the direct cause of the following exception: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_pdf_embedded_image_textle0') + + def test_pdf_embedded_image_textless_page_still_carries_the_origin_line( + db, tmp_path: Path + ) -> None: + """The owner's core complaint, pinned at the real seam: an image + on a TEXT-LESS page (page 1 carries text, so the layer is usable — + this is the text-less-PAGE case, not the garbage-LAYER case) → the + block is the origin line ONLY, and it is STILL stored + enriched + (the provenance never depends on the page having text).""" + root = tmp_path / "pdftextless" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf(root / "report.pdf", [("Text page one.", []), ("", [png])]) + llm = _pdf_llm(tmp_path, _CapturingVisionLLM) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0) + expected_block = f"Extracted from {root.name}/report.pdf (page 2):" + assert llm.text_parts == [DESCRIBE_PROMPT + NEUTRAL_CONTEXT_PREFIX + expected_block] + img = _doc(db, root.name, "report.pdf#page=2#img=0.png") + assert img.image_context == expected_block + assert img.content == f"{expected_block}\n\n{DESCRIPTION}" + assert img.summary == img.content + finally: +> _cleanup_source(db, root.name) + +tests/unit/test_pdf.py:1029: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pdf.py:220: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2271: in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/context.py:306: in orm_execute_statement + result = conn.execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1421: in execute + return meth( +.venv/lib/python3.13/site-packages/sqlalchemy/sql/elements.py:526: in _execute_on_connection + return connection._execute_clauseelement( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1643: in _execute_clauseelement + ret = self._execute_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1848: in _execute_context + return self._exec_single_context( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1988: in _exec_single_context + self._handle_dbapi_exception( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:2365: in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e +.venv/lib/python3.13/site-packages/sqlalchemy/engine/base.py:1969: in _exec_single_context + self.dialect.do_execute( +.venv/lib/python3.13/site-packages/sqlalchemy/engine/default.py:952: in do_execute + cursor.execute(statement, parameters) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +query = 'SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, docum...ext, documents.page_count, documents.page_strip_mode \nFROM documents \nWHERE documents.source = %(source_1)s::VARCHAR' +params = {'source_1': 'pdftextless'}, prepare = None, binary = None + + def execute( + self, + query: Query, + params: Params | None = None, + *, + prepare: bool | None = None, + binary: bool | None = None, + ) -> Self: + """ + Execute a query or command to the database. + """ + try: + with self._conn.lock: + self._conn.wait( + self._execute_gen(query, params, prepare=prepare, binary=binary) + ) + except e._NO_TRACEBACK as ex: +> raise ex.with_traceback(None) +E sqlalchemy.exc.InternalError: (psycopg.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block +E [SQL: SELECT documents.id, documents.source, documents.path, documents.full_path, documents.title, documents.content, documents.content_hash, documents.file_hash, documents.indexed_at, documents.created_at, documents.created_at_manual, documents.summary, documents.is_image, documents.image_path, documents.image_context, documents.page_count, documents.page_strip_mode +E FROM documents +E WHERE documents.source = %(source_1)s::VARCHAR] +E [parameters: {'source_1': 'pdftextless'}] +E (Background on this error at: https://sqlalche.me/e/20/2j85) + +.venv/lib/python3.13/site-packages/psycopg/cursor.py:117: InternalError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:54:09 INFO app.importer :: import: added source=pdftextless path=report.pdf chunks=1 +2026-10-19 11:54:09 INFO app.importer :: import: summary source=pdftextless path=report.pdf chars=54 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:2128 import: added source=pdftextless path=report.pdf chunks=1 +INFO app.importer:importer.py:1784 import: summary source=pdftextless path=report.pdf chars=54 +___________ test_page_copies_are_written_atomically_tmp_then_replace ___________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_page_copies_are_written_a0') +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7f531f0f7850> + + def test_page_copies_are_written_atomically_tmp_then_replace( + db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """THE atomic pin (LOCKED A4): the writer NEVER writes a final + ``-p{N}.png`` path directly — every page's bytes land on + ``.png.tmp`` and ``os.replace`` moves them onto the final + name in order (a reader opening the final path never sees truncated + bytes), and no ``.tmp`` survives the successful call.""" + root = tmp_path / "pgatom" + root.mkdir() + _build_ikea_like_pdf(root / "ikea.pdf") + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) + written: list[Path] = [] + original_write = Path.write_bytes + + def write_spy(self: Path, data: bytes) -> int: + written.append(self) + return original_write(self, data) + + replaced: list[tuple[str, str]] = [] + real_replace = os.replace + + def replace_spy(src, dst, *args, **kwargs): # pyright: ignore[reportUnknownParameterType] + replaced.append((os.fspath(src), os.fspath(dst))) + return real_replace(src, dst, *args, **kwargs) + + monkeypatch.setattr(Path, "write_bytes", write_spy) # pyright: ignore[reportArgumentType] + monkeypatch.setattr(importer.os, "replace", replace_spy) + try: +> asyncio.run(import_sources([root], llm, session=db)) + +tests/unit/test_pdf_page_copies.py:768: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1609: in _index_file + return await _index_pdf_file( +app/rag/importer.py:2133: in _index_pdf_file + await _store_summary( +app/rag/importer.py:1772: in _store_summary + session.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:86: in save_obj + _emit_update_statements( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +base_mapper = +uowtransaction = +mapper = +table = Table('documents', MetaData(), Column('id', UUID(), table=, primary_key=True, nullable=False, default=Calla...ge_count', Integer(), table=), Column('page_strip_mode', String(length=16), table=), schema=None) +update = +bookkeeping = True, use_orm_update_stmt = None, enable_check_rowcount = True + + def _emit_update_statements( + base_mapper, + uowtransaction, + mapper, + table, + update, + *, + bookkeeping=True, + use_orm_update_stmt=None, + enable_check_rowcount=True, + ): + """Emit UPDATE statements corresponding to value lists collected + by _collect_update_commands().""" + + needs_version_id = ( + mapper.version_id_col is not None + and mapper.version_id_col in mapper._cols_by_table[table] + ) + + execution_options = {"compiled_cache": base_mapper._compiled_cache} + + def update_stmt(existing_stmt=None): + clauses = BooleanClauseList._construct_raw(operators.and_) + + for col in mapper._pks_by_table[table]: + clauses._append_inplace( + col == sql.bindparam(col._label, type_=col.type) + ) + + if needs_version_id: + clauses._append_inplace( + mapper.version_id_col + == sql.bindparam( + mapper.version_id_col._label, + type_=mapper.version_id_col.type, + ) + ) + + if existing_stmt is not None: + stmt = existing_stmt.where(clauses) + else: + stmt = table.update().where(clauses) + return stmt + + if use_orm_update_stmt is not None: + cached_stmt = update_stmt(use_orm_update_stmt) + + else: + cached_stmt = base_mapper._memo(("update", table), update_stmt) + + for ( + (connection, paramkeys, hasvalue, has_all_defaults, has_all_pks), + records, + ) in groupby( + update, + lambda rec: ( + rec[4], # connection + set(rec[2]), # set of parameter keys + bool(rec[5]), # whether or not we have "value" parameters + rec[6], # has_all_defaults + rec[7], # has all pks + ), + ): + rows = 0 + records = list(records) + + statement = cached_stmt + + if use_orm_update_stmt is not None: + statement = statement._annotate( + { + "_emit_update_table": table, + "_emit_update_mapper": mapper, + } + ) + + return_defaults = False + + if not has_all_pks: + statement = statement.return_defaults(*mapper._pks_by_table[table]) + return_defaults = True + + if ( + bookkeeping + and not has_all_defaults + and mapper.base_mapper.eager_defaults is True + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + and table.implicit_returning + and connection.dialect.update_returning + ): + statement = statement.return_defaults( + *mapper._server_onupdate_default_cols[table] + ) + return_defaults = True + + if mapper._version_id_has_server_side_value: + statement = statement.return_defaults(mapper.version_id_col) + return_defaults = True + + assert_singlerow = connection.dialect.supports_sane_rowcount + + assert_multirow = ( + assert_singlerow + and connection.dialect.supports_sane_multi_rowcount + ) + + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + allow_executemany = not return_defaults and not needs_version_id + + if hasvalue: + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement.values(value_params), + params, + execution_options=execution_options, + ) + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + check_rowcount = enable_check_rowcount and assert_singlerow + else: + if not allow_executemany: + check_rowcount = enable_check_rowcount and assert_singlerow + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement, params, execution_options=execution_options + ) + + # TODO: why with bookkeeping=False? + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + else: + multiparams = [rec[2] for rec in records] + + check_rowcount = enable_check_rowcount and ( + assert_multirow + or (assert_singlerow and len(multiparams) == 1) + ) + + c = connection.execute( + statement, multiparams, execution_options=execution_options + ) + + rows += c.rowcount + + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + ( + c.returned_defaults + if not c.context.executemany + else None + ), + ) + + if check_rowcount: + if rows != len(records): +> raise orm_exc.StaleDataError( + "UPDATE statement on table '%s' expected to " + "update %d row(s); %d were matched." + % (table.description, len(records), rows) + ) +E sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:949: StaleDataError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:54:48 INFO app.importer :: pdf: pictorial pass source=pgatom path=ikea.pdf — 6 candidate pages, 6 described (min_chars=100) +2026-10-19 11:54:48 INFO app.importer :: import: added source=pgatom path=ikea.pdf chunks=10 +------------------------------ Captured log call ------------------------------- +INFO app.importer:importer.py:2380 pdf: pictorial pass source=pgatom path=ikea.pdf — 6 candidate pages, 6 described (min_chars=100) +INFO app.importer:importer.py:2128 import: added source=pgatom path=ikea.pdf chunks=10 +____________ test_unreadable_page_placeholder_skips_the_vision_call ____________ + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_unreadable_page_placehold0') +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7f531ef4ca60> +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f531ef4db70> + + def test_unreadable_page_placeholder_skips_the_vision_call( + db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + """A candidate page that could not be RENDERED (the zero-byte + ``render_pdf_pages`` placeholder — the phase-133 handling) is + skipped with a warning: it burns NO vision call, keeps its text + only, and the OTHER candidates keep their positions + + descriptions (the result's order never shifts).""" + root = tmp_path / "halfblind" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + + real_render = importer.render_pdf_pages + + def _render(path: Path, dpi: int = 150, pages: Any = None) -> list[bytes]: + assert pages is not None # only the pass calls it (with a selector) + pngs = real_render(path, dpi=dpi, pages=pages) + return [ + b"" if n == 6 else png + for n, png in zip(pages, pngs, strict=True) + ] + + monkeypatch.setattr(importer, "render_pdf_pages", _render) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) + try: + with caplog.at_level(logging.WARNING, logger="app.importer"): +> summary = asyncio.run(import_sources([root], llm, session=db)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +tests/unit/test_pictorial_import.py:547: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ +app/rag/importer.py:1041: in import_sources + await _index_file( +app/rag/importer.py:1609: in _index_file + return await _index_pdf_file( +app/rag/importer.py:2133: in _index_pdf_file + await _store_summary( +app/rag/importer.py:1772: in _store_summary + session.commit() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2035: in commit + trans.commit(_to_root=True) +:2: in commit + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1316: in commit + self._prepare_impl() +:2: in _prepare_impl + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:137: in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:1290: in _prepare_impl + self.session.flush() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4353: in flush + self._flush(objects) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4488: in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py:122: in __exit__ + raise exc_value.with_traceback(exc_tb) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:4449: in _flush + flush_context.execute() +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:465: in execute + rec.execute(self) +.venv/lib/python3.13/site-packages/sqlalchemy/orm/unitofwork.py:641: in execute + util.preloaded.orm_persistence.save_obj( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:86: in save_obj + _emit_update_statements( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +base_mapper = +uowtransaction = +mapper = +table = Table('documents', MetaData(), Column('id', UUID(), table=, primary_key=True, nullable=False, default=Calla...ge_count', Integer(), table=), Column('page_strip_mode', String(length=16), table=), schema=None) +update = +bookkeeping = True, use_orm_update_stmt = None, enable_check_rowcount = True + + def _emit_update_statements( + base_mapper, + uowtransaction, + mapper, + table, + update, + *, + bookkeeping=True, + use_orm_update_stmt=None, + enable_check_rowcount=True, + ): + """Emit UPDATE statements corresponding to value lists collected + by _collect_update_commands().""" + + needs_version_id = ( + mapper.version_id_col is not None + and mapper.version_id_col in mapper._cols_by_table[table] + ) + + execution_options = {"compiled_cache": base_mapper._compiled_cache} + + def update_stmt(existing_stmt=None): + clauses = BooleanClauseList._construct_raw(operators.and_) + + for col in mapper._pks_by_table[table]: + clauses._append_inplace( + col == sql.bindparam(col._label, type_=col.type) + ) + + if needs_version_id: + clauses._append_inplace( + mapper.version_id_col + == sql.bindparam( + mapper.version_id_col._label, + type_=mapper.version_id_col.type, + ) + ) + + if existing_stmt is not None: + stmt = existing_stmt.where(clauses) + else: + stmt = table.update().where(clauses) + return stmt + + if use_orm_update_stmt is not None: + cached_stmt = update_stmt(use_orm_update_stmt) + + else: + cached_stmt = base_mapper._memo(("update", table), update_stmt) + + for ( + (connection, paramkeys, hasvalue, has_all_defaults, has_all_pks), + records, + ) in groupby( + update, + lambda rec: ( + rec[4], # connection + set(rec[2]), # set of parameter keys + bool(rec[5]), # whether or not we have "value" parameters + rec[6], # has_all_defaults + rec[7], # has all pks + ), + ): + rows = 0 + records = list(records) + + statement = cached_stmt + + if use_orm_update_stmt is not None: + statement = statement._annotate( + { + "_emit_update_table": table, + "_emit_update_mapper": mapper, + } + ) + + return_defaults = False + + if not has_all_pks: + statement = statement.return_defaults(*mapper._pks_by_table[table]) + return_defaults = True + + if ( + bookkeeping + and not has_all_defaults + and mapper.base_mapper.eager_defaults is True + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + and table.implicit_returning + and connection.dialect.update_returning + ): + statement = statement.return_defaults( + *mapper._server_onupdate_default_cols[table] + ) + return_defaults = True + + if mapper._version_id_has_server_side_value: + statement = statement.return_defaults(mapper.version_id_col) + return_defaults = True + + assert_singlerow = connection.dialect.supports_sane_rowcount + + assert_multirow = ( + assert_singlerow + and connection.dialect.supports_sane_multi_rowcount + ) + + # change as of #8889 - if RETURNING is not going to be used anyway, + # (applies to MySQL, MariaDB which lack UPDATE RETURNING) ensure + # we can do an executemany UPDATE which is more efficient + allow_executemany = not return_defaults and not needs_version_id + + if hasvalue: + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement.values(value_params), + params, + execution_options=execution_options, + ) + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + check_rowcount = enable_check_rowcount and assert_singlerow + else: + if not allow_executemany: + check_rowcount = enable_check_rowcount and assert_singlerow + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + c = connection.execute( + statement, params, execution_options=execution_options + ) + + # TODO: why with bookkeeping=False? + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + c.returned_defaults, + ) + rows += c.rowcount + else: + multiparams = [rec[2] for rec in records] + + check_rowcount = enable_check_rowcount and ( + assert_multirow + or (assert_singlerow and len(multiparams) == 1) + ) + + c = connection.execute( + statement, multiparams, execution_options=execution_options + ) + + rows += c.rowcount + + for ( + state, + state_dict, + params, + mapper, + connection, + value_params, + has_all_defaults, + has_all_pks, + ) in records: + if bookkeeping: + _postfetch( + mapper, + uowtransaction, + table, + state, + state_dict, + c, + c.context.compiled_parameters[0], + value_params, + True, + ( + c.returned_defaults + if not c.context.executemany + else None + ), + ) + + if check_rowcount: + if rows != len(records): +> raise orm_exc.StaleDataError( + "UPDATE statement on table '%s' expected to " + "update %d row(s); %d were matched." + % (table.description, len(records), rows) + ) +E sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/persistence.py:949: StaleDataError + +During handling of the above exception, another exception occurred: + +db = +tmp_path = PosixPath('/tmp/pytest-of-ducoterra/pytest-2115/test_unreadable_page_placehold0') +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7f531ef4ca60> +caplog = <_pytest.logging.LogCaptureFixture object at 0x7f531ef4db70> + + def test_unreadable_page_placeholder_skips_the_vision_call( + db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + """A candidate page that could not be RENDERED (the zero-byte + ``render_pdf_pages`` placeholder — the phase-133 handling) is + skipped with a warning: it burns NO vision call, keeps its text + only, and the OTHER candidates keep their positions + + descriptions (the result's order never shifts).""" + root = tmp_path / "halfblind" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + + real_render = importer.render_pdf_pages + + def _render(path: Path, dpi: int = 150, pages: Any = None) -> list[bytes]: + assert pages is not None # only the pass calls it (with a selector) + pngs = real_render(path, dpi=dpi, pages=pages) + return [ + b"" if n == 6 else png + for n, png in zip(pages, pngs, strict=True) + ] + + monkeypatch.setattr(importer, "render_pdf_pages", _render) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) + try: + with caplog.at_level(logging.WARNING, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + assert llm.pictorial_calls == 5 # the blank page burned no call + + content, candidates = _expected_content(pdf, fail_pages=frozenset({6})) + doc = _pdf_doc(db, "halfblind", "ikea.pdf") + assert doc.content == content + assert "[Page 6 illustration" not in doc.content + assert "---PAGE-6---\nIllustration 6" in doc.content + assert summary.pdfs_pictorial == 1 + assert summary.pdf_pictorial_pages == 5 + warnings = [ + r + for r in caplog.records + if r.levelno == logging.WARNING + and "skipping pictorial page 6" in r.getMessage() + ] + assert len(warnings) == 1 + finally: +> _cleanup_source(db, "halfblind") + +tests/unit/test_pictorial_import.py:566: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +tests/unit/test_pictorial_import.py:244: in _cleanup_source + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2481: in scalars + return self._execute_internal( +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2261: in _execute_internal + conn = self._connection_for_bind(bind) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:2113: in _connection_for_bind + return trans._connection_for_bind(engine, execution_options) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:2: in _connection_for_bind + ??? +.venv/lib/python3.13/site-packages/sqlalchemy/orm/state_changes.py:101: in _go + self._raise_for_prerequisite_state(fn.__name__, current_state) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +operation_name = '_connection_for_bind' +state = + + def _raise_for_prerequisite_state( + self, operation_name: str, state: _StateChangeState + ) -> NoReturn: + if state is SessionTransactionState.DEACTIVE: + if self._rollback_exception: +> raise sa_exc.PendingRollbackError( + "This Session's transaction has been rolled back " + "due to a previous exception during flush." + " To begin a new transaction with this Session, " + "first issue Session.rollback()." + f" Original exception was: {self._rollback_exception}", + code="7s2a", + ) +E sqlalchemy.exc.PendingRollbackError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: UPDATE statement on table 'documents' expected to update 1 row(s); 0 were matched. (Background on this error at: https://sqlalche.me/e/20/7s2a) + +.venv/lib/python3.13/site-packages/sqlalchemy/orm/session.py:975: PendingRollbackError +----------------------------- Captured stdout call ----------------------------- +2026-10-19 11:54:55 WARNING app.importer :: pdf: skipping pictorial page 6 of halfblind/ikea.pdf — the render is empty (unreadable page) +2026-10-19 11:54:55 WARNING app.importer :: import: PDF page render failed source=halfblind path=ikea.pdf — assert None is not None +------------------------------ Captured log call ------------------------------- +WARNING app.importer:importer.py:2349 pdf: skipping pictorial page 6 of halfblind/ikea.pdf — the render is empty (unreadable page) +WARNING app.importer:importer.py:2106 import: PDF page render failed source=halfblind path=ikea.pdf — assert None is not None +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 332 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7181 16 99% +=========================== short test summary info ============================ +FAILED tests/unit/test_folder_summaries.py::test_on_progress_fires_once_per_candidate_in_sorted_key_order +FAILED tests/unit/test_importer.py::test_on_chunk_progress_reports_per_file_batch_sequences +FAILED tests/unit/test_importer.py::test_on_image_progress_embedded_pass_climbs_one_per_image +FAILED tests/unit/test_importer_dates.py::test_manual_row_date_survives_unchanged_reimport +FAILED tests/unit/test_importer_unchanged_fast_path.py::test_regate_fires_on_the_content_digest_hit_path +FAILED tests/unit/test_importer_unchanged_fast_path.py::test_manual_date_null_hash_row_stamps_without_date_write +FAILED tests/unit/test_importer_unchanged_fast_path.py::test_unchanged_fast_path_date_refresh_may_go_older +FAILED tests/unit/test_overview.py::test_regenerate_happy_path_creates_single_row_id_1 +FAILED tests/unit/test_overview.py::test_document_list_reaching_the_model_leads_with_newest_docs +FAILED tests/unit/test_overview.py::test_regenerate_llm_error_leaves_previous_row_intact +FAILED tests/unit/test_overview.py::test_regenerate_opens_own_session_when_none +FAILED tests/unit/test_overview.py::test_regenerate_model_input_never_gains_the_inventory_tail +FAILED tests/unit/test_overview.py::test_regenerate_zero_docs_keeps_tailless_pre_phase_row +FAILED tests/unit/test_overview.py::test_regenerate_llm_error_preserves_an_existing_inventory_tail +FAILED tests/unit/test_pdf.py::test_pdf_becomes_a_document_with_text_chunks_and_summary +FAILED tests/unit/test_pdf.py::test_pdf_embedded_images_become_image_documents +FAILED tests/unit/test_pdf.py::test_pdf_embedded_images_skipped_when_images_off +FAILED tests/unit/test_pdf.py::test_pdf_unchanged_rerun_skips_everything - sq... +FAILED tests/unit/test_pdf.py::test_pdf_unchanged_path_backfills_and_honors_manual_date +FAILED tests/unit/test_pdf.py::test_pdf_text_change_updates_doc_and_keeps_unchanged_images +FAILED tests/unit/test_pdf.py::test_corrupt_pdf_file_becomes_an_empty_document +FAILED tests/unit/test_pdf.py::test_pdf_embedded_image_failed_description_skips_image_keeps_pdf +FAILED tests/unit/test_pdf.py::test_pdf_embedded_image_embed_failure_keeps_the_others +FAILED tests/unit/test_pdf.py::test_pdf_changed_embedded_image_replaces_the_copy +FAILED tests/unit/test_pdf.py::test_pdf_embedded_image_vision_call_carries_the_document_context +FAILED tests/unit/test_pdf.py::test_pdf_embedded_image_textless_page_still_carries_the_origin_line +FAILED tests/unit/test_pdf_page_copies.py::test_page_copies_are_written_atomically_tmp_then_replace +FAILED tests/unit/test_pictorial_import.py::test_unreadable_page_placeholder_skips_the_vision_call +ERROR tests/unit/test_overview.py::test_document_list_reaching_the_model_leads_with_newest_docs +ERROR tests/unit/test_overview.py::test_regenerate_llm_error_leaves_previous_row_intact +ERROR tests/unit/test_overview.py::test_regenerate_model_input_never_gains_the_inventory_tail +ERROR tests/unit/test_overview.py::test_regenerate_zero_docs_keeps_tailless_pre_phase_row +ERROR tests/unit/test_overview.py::test_regenerate_llm_error_preserves_an_existing_inventory_tail +ERROR tests/unit/test_overview.py::test_regenerate_model_input_carries_the_display_name +ERROR tests/unit/test_overview.py::test_regenerate_model_input_carries_the_display_name +ERROR tests/unit/test_overview.py::test_regenerate_stored_content_keeps_the_derived_names +ERROR tests/unit/test_overview.py::test_regenerate_stored_content_keeps_the_derived_names +ERROR tests/unit/test_overview.py::test_regenerate_without_label_rows_is_byte_identical_to_pre_phase +ERROR tests/unit/test_overview.py::test_regenerate_without_label_rows_is_byte_identical_to_pre_phase +ERROR tests/unit/test_overview.py::test_regenerate_reads_the_label_rows_once +tests FAILED: unit + integration suite did not pass +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation FAILED (see output above) diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.err b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md new file mode 100644 index 0000000..ab5e1b9 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md @@ -0,0 +1,12 @@ +Phase 192 — final verification pass complete. + +- **Verified (all 4 tasks already complete):** additive `verbatim_block` SSE frame (`VerbatimBlockPiece` → `ChatVerbatimBlockEvent` → chat dispatch, after the round's `tool`/`tool_result`, before the next round); `hydrateVerbatimNow` matched-only mid-turn pass + app.js branch; `dropPlacedTwin` per-locator done/restore dedupe; the dedicated E2E suite. +- **Root cause of the rejected attempt:** my own concurrent background `pytest` run deadlocked the harness validation run on the shared Postgres (`psycopg.errors.DeadlockDetected`). No code defect — re-ran every gate **exclusively**. +- `uv run pytest -q --cov=app --cov-report=term-missing` → **exit 0**, `TOTAL 7181 16 99%` (>90%) +- `uv run pytest tests/e2e/test_verbatim_inline_midturn.py --no-cov` → **6 passed** (isolated) +- `uv run pytest tests/e2e/test_verbatim_show_text.py tests/e2e/test_chat_history.py --no-cov` → **18 passed** (`test_chat_history.py` byte-unedited; the phase-182 suite's "no eighth type" pin was tightened, not loosened, to seven + `verbatim_block` with frame-order asserts) +- `uv run ruff check .` → All checks passed (exit 0) · `uv run pyright` → **0 errors** +- **Completion criteria:** single frame per success + zero per refusal ✅ · `BOR_ANSWER_TEXT=0` wire byte-identical (integration spelled baseline + E2E App B) ✅ · fence hydrates in place / unfenced lands at the call moment, before `done` ✅ · append+fence → 1 panel, two ranges → 2 panels ✅ · restore 1 panel/locator, `done.answer_text` + record key + `verbatim_blocks=N` unchanged ✅ · suite green, coverage 99%, lint/types clean ✅ · no-show_text turn byte-identical ✅ · commit → left to the harness (no git index/history commands run). +- **Docs:** README reviewed line-by-line (2399 lines) + AGENTS.md in full; mechanically verified all 86 `BOR_*` settings, every `.env.example` var, table defaults, internal anchors (0 broken), ports 8123/8901, tree depth 1–8, pagination 25/1–100, 17 palette colors, `0029`/`0030` migrations, and every cited test file. Fixed 1 inconsistency: the verbatim "Parity" bullet claimed the live turn/reload/shared page share "the same three globals" — now names the settle-path trio and states that `hydrateVerbatimNow` is the ONE chat-only addition (a shared page has no stream). +- **Notable:** `run_agent`'s four repeated piece unions were collapsed into the ONE exported `AgentPiece` alias, so a future piece type is spelled in one place; `test_frontend_edit_message.py`'s `runTurn` byte-pin was regenerated, not weakened. +- **Next pending phase:** `193_sync_remaining_count`. diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md.status b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.validate b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.validate new file mode 100644 index 0000000..631349a --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__00_phase.a2.validate @@ -0,0 +1,150 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 97%] +........................................................................ [ 99%] +...................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 332 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7181 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.err b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md new file mode 100644 index 0000000..d75e1d2 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md @@ -0,0 +1,19 @@ +All gates green. Report: + +**Implemented (task 01 — the additive `verbatim_block` SSE frame, A1)** +- `app/rag/agent.py`: new frozen `@dataclass VerbatimBlockPiece` (`block: VerbatimBlock`); the round loop snapshots `shown_before = len(holder.shown_texts)` before `_execute_tool` and yields one piece per appended block, AFTER the round's `tool`/`tool_result` pieces and BEFORE the next model round (success-only — every refusal returns before the `shown_texts.append`); module + `run_agent` docstrings carry the emission-order contract. +- Named the widened generator union once as `type AgentPiece = StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece | VerbatimBlockPiece` (task text spelled it literally; the alias keeps `run_agent`, the chat pump and 6 test harnesses on one spelling — the literal widening broke E501/pyright in 19 annotation sites). +- `app/schemas.py`: `ChatVerbatimBlockEvent` right after `ChatToolResultEvent` — `type: Literal["verbatim_block"]` + the exact `VerbatimBlockRef` field set/caps, no `model_config` (sibling convention mirrored). +- `app/api/chat.py`: the `VerbatimBlockPiece` dispatch branch after the `ToolResultPiece` branch; done-frame `answer_text` and the `verbatim_blocks=N` log line untouched. +- Tests: 10 unit cases in `tests/unit/test_agent.py` (piece shape, success position + holder-object identity, order after a truncation piece, 5 refusal params, already-shown dedupe, kill switch), 2 in `tests/unit/test_schemas.py` (wire shape + a field/caps drift pin against `VerbatimBlockRef`), 2 in `tests/integration/test_chat_api.py` (`verbatim_block` between `tool` and `delta`/`done` with the exact key set + same block still on `done`, `verbatim_blocks=1`; kill-switch frame sequence byte-identical to the spelled-out pre-phase baseline). + +**Gates** — `uv run pytest --cov=app --cov-report=term-missing` → 4777 tests, exit 0, TOTAL 99% (chat.py 99%, agent.py 99%, schemas.py 100%); `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors. E2E in isolation `--no-cov`: `test_verbatim_show_text.py` 13 passed, `test_chat_history.py` + `test_answer_images.py` 10 passed, `test_source_name_labels.py` 7 passed. + +**Docs** — README verbatim section: new "The panel lands while the answer streams" bullet, the `done`-frame bullet's now-false "no eighth SSE event type" corrected, `BOR_ANSWER_TEXT` row notes no frame is sent; AGENTS.md rule 5 phase-182 clause + rule 10 `verbatim_blocks=N` clause updated. + +**Deviations / notes** +- `tests/e2e/test_verbatim_show_text.py` pinned "SSE type set stays inside the pre-phase seven (no eighth type)" — invalidated by this phase's design; updated to "the seven + `verbatim_block`" and extended with the mid-turn ordering pin (that suite is otherwise green unedited). +- `PLAN.md` §2 A15 / §4 still list seven event types and "no eighth SSE event type"; PLAN.md is out of my edit scope — flag for the phase-04 docs pass / owner revision note. +- No live-infra work in this task; all changes left unstaged for the harness. + +**Next pending task:** `192_verbatim_inline_midturn/02_midturn_hydration_and_append.md` (`hydrateVerbatimNow` + the `app.js` `verbatim_block` branch). diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md.status b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.validate b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.validate new file mode 100644 index 0000000..ecbb50b --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__01_verbatim_block_sse.a1.validate @@ -0,0 +1,150 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 97%] +........................................................................ [ 99%] +......................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 332 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7181 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.err b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md new file mode 100644 index 0000000..c03bcb0 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md @@ -0,0 +1,19 @@ +All gates green. Report: + +**Implemented (task 02 — mid-turn landing, A2)** +- `frontend/assets/verbatim.js`: new `hydrateVerbatimNow(root, block, opts)` — walks the done path's pending set (`.verbatim-pending[data-bor-locator]`), matches through the ONE `matchVerbatimBlock` rule with a one-block list, replaces ONLY matched nodes with `buildVerbatimPanel`, leaves every unmatched fence `.verbatim-pending`, returns the settle count, total (no `try`) via `isVerbatimBlock` + root guards. +- `frontend/assets/app.js`: the `ev.type === "verbatim_block"` branch beside `tool_result` — hydrate in place, `if (n === 0) appendUnlocatedVerbatim(wrap, [ev], VERBATIM_LINK_OPTS)`; no client-side kill-switch gate (server-only). No import line existed/changed: `verbatim.js` is a classic script whose top-level functions are globals (task text's "import site" = the existing global call sites). +- **Pulled forward phase design A3** (`verbatimPlacedKey` + `dropPlacedTwin`, called from `hydrateVerbatim`): without it, a mid-turn-appended block that the model fences in a later round painted TWO panels and the phase-182 E2E suite (`test_verbatim_show_text.py`, 5 tests) went red — the "one locator, one panel" rule (per canonical locator, `.msg`-bounded, pending/unresolved never a placement) was the only way to leave the repo green. +- Pins in `tests/unit/test_frontend_verbatim_blocks.py`: marker pins (`hydrateVerbatimNow` matched-only contract, never-throw, the app.js zero-match-append shape + asserted absence of any client gate, the twin-scan shape) plus 4 new node-driven behavioural tests on a minimal DOM (in-place settle + untouched sibling fence, server-extent fallback, malformed/no-match/null-root arms, mid-turn panel survives the done pass, one-locator-one-panel across passes incl. two ranges → two panels). +- `tests/unit/test_frontend_edit_message.py`: regenerated `PINNED_RUN_TURN` to the sanctioned phase-192 branch (verified the old pin == HEAD `runTurn` first) + docstring clause; `verbatimPlacedKey` now the single key derivation shared with `appendUnlocatedVerbatim`. + +**Gates** — `uv run pytest --cov=app --cov-report=term-missing` → **4784 passed**, TOTAL **99%**; `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors; E2E in isolation `--no-cov`: `test_verbatim_show_text.py` + `test_chat_history.py` + `test_answer_images.py` + `test_share_chat.py` + `test_chat_persistence.py` → 31 passed (the phase-182 suite green again, unedited assertions). + +**Docs** — README verbatim section: the client half of the mid-turn landing (in-place vs answer-position, twin dropped, one locator one panel, no client flag); AGENTS.md rule 5 phase-182 clause extended with the mid-turn matched-only contract + the per-locator twin scan, re-tagged "phase 182 task 07, phase 192 tasks 01–02". + +**Decisions / deviations** +- Task 02's checkbox "hydrateVerbatim untouched" is unsatisfiable together with a green suite (the duplicate is structural); resolved by implementing A3 exactly as the phase overview specifies. **Task 03 should verify/extend, not re-implement** — remaining there: the restore/shared-parity pin and its own marker set. +- Side effect, documented + pinned: a fence the model repeats for the SAME locator now shows one panel (at the last fence) instead of two. +- No live-infra work; all changes left unstaged for the harness. + +**Next pending task:** `192_verbatim_inline_midturn/03_done_and_restore_dedupe.md`. diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md.status b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.validate b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.validate new file mode 100644 index 0000000..4904d2f --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__02_midturn_hydration_and_append.a1.validate @@ -0,0 +1,150 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 97%] +........................................................................ [ 99%] +................................ [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 332 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7181 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.err b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md new file mode 100644 index 0000000..20bde19 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md @@ -0,0 +1,18 @@ +**Task 03 complete — all gates green.** + +Implemented (task 03, A3/A4): +- `frontend/assets/verbatim.js`: `hydrateVerbatim`'s twin-scan host is now `root.closest(".msg") || root` — a missing `.msg` wrap narrows the scan to the root instead of switching the per-locator dedupe off (`dropPlacedTwin` itself, `appendUnlocatedVerbatim`, `settleVerbatimPlaceholders` and the task-02 functions untouched). +- `tests/unit/test_frontend_verbatim_blocks.py`: new `_doc`/`_flow` docstring-pin helpers + `bareFence` DOM fixture; pins `test_the_twin_dedupe_docstring_keeps_the_per_locator_clause` (per-locator-not-per-identity + two-ranges-stay + in-text-wins prose), `test_the_twin_scan_bounds_to_the_msg_wrap_or_the_root`, node arms `test_the_twin_scan_bounds_to_the_message_it_hydrates` (bare-bubble dedupe, two ranges stay two, a sibling message is out of bounds) and `test_a_stored_record_restores_one_panel_per_locator` (restore/shared parity behaviour). +- `tests/integration/test_chat_api.py`: `test_done_frame_and_log_line_unchanged_by_midturn_frames` — two `show_text` rounds → two `verbatim_block` frames each inside its own round, `done.answer_text` byte-equal to the phase-182 resolution spelled from the row (new `_BACKUPS`/`_doc_title` helpers), done key set unchanged, and the ONE log line reads `verbatim_blocks=2` (never 4 — mid-turn frames counted nowhere). +- `tests/unit/test_schemas.py`: `test_chat_verbatim_block_event_dump_keys_are_the_client_contract` (dump key set == `VerbatimBlockRef` keys + `type`, client-guard keys present, ref round-trip). +- Verified the A3 code and several pins already existed from task 02; I closed only the genuine gaps (fallback, docstring pin, bound arms, the named integration/log pin, the schema key-set pin) rather than duplicating, and confirmed `app.js` restore lines and `shared.js` are untouched (`git diff`: app.js +24 lines = the new branch only). + +Gates (exact commands): +- `uv run pytest --cov=app --cov-report=term-missing` → **4790 passed**, `TOTAL 7181 16 99%` (>90%) +- `uv run pytest tests/unit/test_frontend_verbatim_blocks.py tests/integration/test_chat_api.py -v --no-cov` → **124 passed** +- `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors +- `uv run pytest tests/e2e/test_verbatim_show_text.py --no-cov` → 13 passed; `tests/e2e/test_chat_history.py` → 5 passed + +Docs: `README.md` verbatim section now states the dedupe's bound (`.msg` wrap, root fallback, never crosses into the next answer); `AGENTS.md` rule 5 `dropPlacedTwin` clause gained the bound and the attribution is now "phase 192 tasks 01–03". + +Next pending task: `.agents/phases/todo/192_verbatim_inline_midturn/04_e2e_and_gates.md`. diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md.status b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.validate b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.validate new file mode 100644 index 0000000..631349a --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__03_done_and_restore_dedupe.a1.validate @@ -0,0 +1,150 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 97%] +........................................................................ [ 99%] +...................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 332 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7181 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.err b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md new file mode 100644 index 0000000..7627dbe --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md @@ -0,0 +1,28 @@ +**Task 04 complete — all gates green.** + +Added (`tests/e2e/test_verbatim_inline_midturn.py`, NEW — 6 tests, module docstring names TODO.md:6 and the LOCKED-A4 unchanged-contract clause; ports 8209/8210, the phase-182 module-app/KB/cleanup conventions): + +| Test | Arm | Pins | +|---|---|---| +| `test_midturn_panel_lands_before_the_answer_streams` | (a) | panel in the DOM **before `done`** (in-page `MutationObserver` witness: `doneSeen=false`, `sentinelInDom=false`, pill `"Stop"`, `forLocator=1`), below the intro the model wrote pre-call, server caption/gutter/body/badge, clipboard == the server string, band order bubble → verbatim → `.msg-meta`, `done` adds nothing (count 1), frame order `delta* → tool → verbatim_block → delta* → done` with each delta segment byte-equal to the mock's text | +| `test_midturn_fence_hydrates_in_place_not_at_the_answer_tail` | (b) | placeholder exists first → `hydrateVerbatimNow` settles it **in place** mid-turn (inside `.bubble`, nothing appended), one panel after `done`, intro < panel < sentinel, no model bytes | +| `test_midturn_append_and_a_later_fence_leave_one_panel` | (c) | mid-turn append at the answer position, `done` re-homes it onto the same-locator fence and drops the twin — exactly ONE panel at the fence | +| `test_two_ranges_of_one_file_land_two_panels` | (d) | `tool → block → tool → block`, two panels in call order, own captions/gutter starts, `done.answer_text` = both | +| `test_kill_switch_off_ships_the_pre_phase_wire` | (e) | App B: `Unknown tool.` refusal, **no** `verbatim_block`, no `answer_text`, tool frame an exact dict, delta bytes == App A's answer bytes, zero panels/placeholders witnessed | +| `test_restore_repaints_one_panel_for_the_midturn_locator` | (f) | record `answer_text` unchanged, reload + server-record path (localStorage cleared) → ONE panel, caption/bytes/badge/gutter, no turn re-run | + +Mock (`tests/e2e/mock_llm.py`, additive): the `MIDTURN_TRIGGER` flow + `MIDTURN_*` constants and answer strings, `MIDTURN_GAP_S` silence between the frame and the answer's first delta, arm markers, the two-range stateless branch, the module-docstring bullet. Phase-182's pins and every other flow untouched. + +Docs: `README.md` — the mid-turn landing clause (already carried tasks 01–03) re-wrapped; `AGENTS.md` rule 5 attribution now `phase 192 tasks 01–04` and names `tests/e2e/test_verbatim_inline_midturn.py` as the owner-visible pin. `.agents/PLAN.md` needed no correction (it never claimed `done` was the only landing point). + +Gates (exact commands): +- `uv run pytest -q --cov=app --cov-report=term-missing` → **4790 passed, exit 0**, `TOTAL 7181 16 99%` (>90%) +- `uv run pytest tests/e2e/test_verbatim_inline_midturn.py -v --no-cov` → **6 passed**, exit 0 (isolated) +- `uv run pytest tests/e2e/test_verbatim_show_text.py tests/e2e/test_chat_history.py -v --no-cov` → **18 passed**, exit 0 (unedited) +- `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors + +Notes / deviations: +1. Arm (a)'s literal "ABOVE the final sentinel delta" cannot hold as sibling order — an unfenced block has no fence to return to, so `appendUnlocatedVerbatim` parks it at the phase-182 answer band (below the answer text, above the chips). The suite pins the stronger mid-turn claim (the sentinel did not exist when the panel landed) and asserts the literal between-the-two position in the fenced arms (b)/(c), where intro < panel < sentinel inside the bubble. Documented in the test docstring. +2. Two mock corrections the arms needed (documented in place): `show_text`'s `path` argument must be the COMBINED `source/path` (a bare path is the phase-72 refusal), and a round-1 fence must be padded with a blank line, because the client concatenates rounds with no separator — an unpadded closing ` ``` ` glues to the next round's first word and stops being a fence. +3. Owner-visible observation, NOT changed (out of task scope): when the fence is written *before* the call, the next `delta` re-renders the bubble from `acc` (which still holds the fence), so the node returns to `.verbatim-pending` until `done` settles it again. Both ends are asserted and the interim is the honest phase-182 state (never the model's bytes); making the per-delta renderer preserve hydrated panels would remove the interim and deserves its own phase/task. +4. Left to the harness (no index/history-mutating commands run): moving `todo/192_verbatim_inline_midturn/04_e2e.md` → `complete/` and the single atomic `--no-gpg-sign` commit. diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md.status b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.validate b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.validate new file mode 100644 index 0000000..631349a --- /dev/null +++ b/.agents/reports/192_verbatim_inline_midturn/192_verbatim_inline_midturn__04_e2e.a1.validate @@ -0,0 +1,150 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 64%] +........................................................................ [ 66%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 97%] +........................................................................ [ 99%] +...................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 332 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 865 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7181 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/AGENTS.md b/AGENTS.md index a0a8e94..7dd89a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,13 @@ caption and a `user-select: none` line-number gutter, while the plain `.code-block` wrapper carries a Copy button and no badge (phase 182); those placeholders hydrate against the frame's - `answer_text` through the ONE panel client + `answer_text` — or, since phase 192, against the additive + `verbatim_block` SSE frame (the eighth event type: one + `done.answer_text` item plus `type`) that carries a successful + `show_text` block the MOMENT the call succeeds, AFTER that round's + `tool` / `tool_result` frames and BEFORE the next model round, so the + panel lands where the model introduced the bytes instead of popping in + after the answer — through the ONE panel client (`frontend/assets/verbatim.js` — a classic script loaded AFTER `markdown.js` on the chat AND shared pages, no second locator grammar, a plain string compare against the server's canonical `locator`, the @@ -68,10 +74,23 @@ knowledge base` with no badge, no body and none of the model's bytes, no panel left at "Retrieving the exact text…" after a terminal frame (the `error` frame, the 300 s guard, Stop and the failure funnel all - settle it), the Copy button's clipboard content equal to the server - string exactly, and the shared page's panel Copy button + plain - `/document.html` link the ONE owner-directed exception to that page's - zero-controls rule) — phase 182 task 07), and the sync's peak-memory + settle it), a MID-TURN pass (`hydrateVerbatimNow`) that settles ONLY the + pending placeholders its ONE block matches — every other fence STAYS + `.verbatim-pending`, because burning one is how a still-resolvable fence + paints the honest sentence — and ONE canonical locator painting ONE panel + (`dropPlacedTwin`: the fence wins over the mid-turn appended twin, the scan + is PER LOCATOR so two honest ranges of one file stay two panels, it is + bounded by the `.msg` wrap (or the root when there is no wrap — a missing + wrap narrows the scan, it never switches the dedupe off), and a + pending/unresolved node is never a placement), the Copy button's clipboard + content equal to the server string exactly, and the shared page's panel Copy + button + plain `/document.html` link the ONE owner-directed exception to that + page's zero-controls rule) — phase 182 task 07, phase 192 tasks 01–04, the + mid-turn landing (panel on screen BEFORE `done`, the fence winning the + same-locator race, two ranges staying two panels, the `answer_text=False` + wire byte-identical) pinned through the UI by + `tests/e2e/test_verbatim_inline_midturn.py`), and + the sync's peak-memory clause is TRAILING TEXT on the result line that already exists (` · peak MB` on the RAG page's `#sync-progress` counts line or "Sync cancelled", written by the ONE `fmtSyncPeakClause` off @@ -105,7 +124,9 @@ 10. **Amply log, visibly feed back:** server-side, log the required per-turn line (PLAN §9 — phase 182 appends `verbatim_blocks=N` after `scaffold_stripped=N`: the verbatim display blocks actually SENT on - the `done` frame, `0` on every turn without them, deflected included) + the `done` frame, `0` on every turn without them, deflected included — + never the phase-192 mid-turn `verbatim_block` frames, which are not + counted anywhere else) **and** the per-response byte-budget line (`app/core/response_budget.py` — one `api method=… path=… status=… bytes=… enc=… duration_ms=…` line per diff --git a/README.md b/README.md index 1dc33f9..e231b37 100644 --- a/README.md +++ b/README.md @@ -1327,6 +1327,27 @@ The model never retypes quoted text, so a displayed quote cannot drift. each block is capped at `BOR_SHOW_TEXT_MAX_CHARS`, cut at a line boundary with a truncation marker. Image documents are refused with a pointer to `show_image` — the two display tools never convert into each other. +- **The panel lands while the answer streams.** A successful `show_text` + sends its block TWICE on purpose (phase 192): once as the additive + `verbatim_block` SSE frame — the eighth event type, + `{type, source, path, title, start_line, end_line, total_lines, text, + truncated, locator}`, i.e. exactly one `done.answer_text` item — emitted + AFTER that round's `tool` / `tool_result` frames and BEFORE the next + model round, so the panel appears where the model introduced the bytes + instead of after it finished talking; and once on the `done` frame below. + A refused call sends nothing, and `BOR_ANSWER_TEXT=0` offers no tool at + all, so such a turn's frame sequence is byte-identical to pre-phase + (pinned on the wire). Client-side, `hydrateVerbatimNow` settles the pending + fence that block names IN PLACE the moment the frame arrives and leaves every + other placeholder pending; when the answer has not fenced it (yet), the panel + lands at the `answer_image` position then — not at the end of the answer. If + the model fences that same locator in a later round, hydration drops the + appended twin, so **one canonical locator paints exactly one panel**, at the + fence (two honest ranges of one file stay two panels: the scan is per + locator, bounded by the message's `.msg` wrap — or by the root when there is + no wrap, so a missing wrap narrows the scan instead of switching the dedupe + off, and one answer never reaches into the next one's panels). There is no + client-side flag check — the server's gate is the only one. - **The bytes ride the `done` frame.** A turn that displayed text ends with one extra, additive SSE key: `done.answer_text`, the list of blocks in **display order** — answer order first (one per ` ```bor ` fence, the @@ -1338,10 +1359,13 @@ The model never retypes quoted text, so a displayed quote cannot drift. 1-based inclusive extent — the numbers a caption shows can never be the model's claim. `locator` is the canonical matching key (`source/path`, or `source/path#12-40` for a range). There is no new - bytes route and no eighth SSE event type: the bytes travel inline, and a - saved or shared chat renders its panels from its own record. The same - array persists on the saved brain record (`bor.chat.v1` `answer_text`), - so a reload and an anonymous shared page show the same blocks. + bytes route: the bytes travel inline (the only extra frame is the + phase-192 `verbatim_block` above, carrying that same item), and a saved + or shared chat renders its panels from its own record — the `done` + frame, the saved record and the restore path are UNCHANGED by phase 192. + The same array persists on the saved brain record (`bor.chat.v1` + `answer_text`), so a reload and an anonymous shared page show the same + blocks. - **Deflected turns resolve nothing.** A ` ```bor ` fence written into a deflected answer leaks no KB text — the disclaimer turn resolves no block and its frame carries no `answer_text` key. @@ -1379,8 +1403,10 @@ The model never retypes quoted text, so a displayed quote cannot drift. - **Hydration paints the panel, and the panel is the file.** The client half is one classic script, `frontend/assets/verbatim.js`, loaded after - `markdown.js` on the chat page and the shared page. At `done` (and on every - restore) it matches each placeholder's normalized `data-bor-locator` + `markdown.js` on the chat page and the shared page. Mid-turn it settles the + one block a `verbatim_block` frame carries (`hydrateVerbatimNow`); at `done` + (and on every restore) it matches each placeholder's normalized + `data-bor-locator` against the frame's `answer_text[].locator` with a plain string compare — `markdown.js::borLocator` stays the ONLY locator grammar in the client, so the two sides cannot drift — and paints: @@ -1426,8 +1452,14 @@ The model never retypes quoted text, so a displayed quote cannot drift. exists once (the owner-locked share strings are unchanged; phase 189 added the save-before-the-token upsert in front of the same success paths). - **Parity, and what stays untouched.** The live turn, a reload and an - anonymous shared page call the same three globals in the same order, so one - answer renders one way. Only the two answer renderers (plus the shared + anonymous shared page call the same three **settle-path** globals in the same + order — `hydrateVerbatim` → `appendUnlocatedVerbatim` → + `enhanceAnswerCodeBlocks` — so one answer renders one way. The phase-192 + mid-turn pass (`hydrateVerbatimNow`) is the ONE chat-only addition: a shared + page has no stream to land a panel mid-turn, so it hydrates from its record + through the same trio (which is why a reload and a shared page show the same + panel the live turn showed at the `show_text` call). Only the two answer + renderers (plus the shared page's) opt in: a question, a thinking run and the document viewer stay one-argument calls, and `document.html` does not even load the panel client. @@ -1886,7 +1918,7 @@ startup (the DB rows shadow the env, `.env` is never rewritten). |-----|---------|---------| | `BOR_AGENT_MAX_ROUNDS` | `10` | Hard cap on agent tool rounds per grounded turn (0 = no tools); every tool result carries the model's remaining count (phase 160) | | `BOR_READ_MAX_CHARS` | `128000` | Cap on a `read` tool result (chars); over-cap reads truncate + a notice that teaches the grep → `read(offset, limit)` workflow (phase 95, rewritten phase 131) | -| `BOR_ANSWER_TEXT` | `1` | Verbatim file display (phase 182): the **master switch, ON by default** — an answer's code block is typed by the server from the indexed bytes, never retyped by the model. `0` = the escape hatch: the `show_text` tool is not offered, its `` teaching clause is absent, ` ```bor ` locator fences are not resolved, and the `done` frame carries no `answer_text` key (the server surface is exactly pre-phase; a stray fence on the client still settles to the honest unresolved sentence rather than the model's copy) | +| `BOR_ANSWER_TEXT` | `1` | Verbatim file display (phase 182): the **master switch, ON by default** — an answer's code block is typed by the server from the indexed bytes, never retyped by the model. `0` = the escape hatch: the `show_text` tool is not offered, its `` teaching clause is absent, ` ```bor ` locator fences are not resolved, no `verbatim_block` frame is sent, and the `done` frame carries no `answer_text` key (the server surface — frame sequence included — is exactly pre-phase; a stray fence on the client still settles to the honest unresolved sentence rather than the model's copy) | | `BOR_SHOW_TEXT_MAX_CHARS` | `12000` | Per-block character budget for ONE displayed verbatim slice (phase 182): the slice is cut at a **line boundary**, the shared `[…truncated…]` marker is appended and the caption's line range shrinks to what is actually shown. `0` = inert (nothing displays); a negative value refuses startup naming the field | | `BOR_SHOW_TEXT_MAX_BLOCKS` | `5` | Per-answer cap on displayed verbatim blocks (phase 182), counted in answer order. `0` = inert (nothing displays); a negative value refuses startup naming the field | diff --git a/app/api/chat.py b/app/api/chat.py index 2f8a068..144656b 100644 --- a/app/api/chat.py +++ b/app/api/chat.py @@ -251,6 +251,24 @@ keeps its exact pre-phase bytes). The per-turn log line records ACTUALLY SENT (after dedupe and the ``show_text_max_blocks`` cut), 0 on every turn without them including deflected ones (the PLAN §9 additive revision). + +Mid-turn verbatim frame (phase 192, task 01, A1; ``TODO.md`` L5): the +same block ALSO reaches the client the moment the ``show_text`` call +succeeds, as the additive ``verbatim_block`` event — +``{"type": "verbatim_block", "source", "path", "title", "start_line", +"end_line", "total_lines", "text", "truncated", "locator"}`` (the exact +:class:`~app.schemas.VerbatimBlockRef` field set, so the one panel client +accepts it as-is). It lands AFTER the round's ``tool`` / ``tool_result`` frames and +BEFORE the next model round — while the answer is still streaming, so the +panel appears where the model introduced the bytes instead of popping in +after it finished talking. Success-only: every refusal (missing ``path``, +no document, an image row, the block cap, the already-shown dedupe) emits +nothing, and a ``BOR_ANSWER_TEXT=0`` turn offers no tool, so its frame +sequence is byte-identical to pre-phase. Everything downstream of the +turn keeps the phase-182 contract EXACTLY: the ``done`` frame's +``answer_text`` list, the saved record key, the restore path and the +shared page are untouched, and ``verbatim_blocks=N`` still counts the +``done``-frame blocks (never the mid-turn frames). """ from __future__ import annotations @@ -275,7 +293,9 @@ from app.models import Document, QueryLog from app.rag.agent import ( CORRECTION_INSTRUCTION, # phase 71: the harness-owned recovery line AgentHolder, + AgentPiece, # the ONE union run_agent yields (phase 192: + VerbatimBlockPiece) MalformedReplyError, # phase 71: the recovery policy's terminal signal + VerbatimBlockPiece, # phase 192: one shown verbatim block (mid-turn frame) run_agent, suggested_folder_lines, # phase 119 (D3): the suggested-folder context lines ) @@ -285,7 +305,6 @@ from app.rag.llm import ( LLMClient, LLMError, RetryPiece, # phase 67: one LLM request restart (an SSE retry frame) - StreamPiece, # type of the answer pieces streamed by the agent loop ToolCallPiece, # phase 37: one model-requested tool call ToolResultPiece, # phase 95: one truncated tool result (SSE frame = task 02) chat_stream_retried, # phase 67: the retry-before-first-piece primitive @@ -323,6 +342,7 @@ from app.schemas import ( ChatThinkingEvent, ChatToolEvent, ChatToolResultEvent, + ChatVerbatimBlockEvent, SourceRef, VerbatimBlockRef, ) @@ -974,9 +994,11 @@ async def chat( # ``agent_max_rounds=0`` ``run_agent`` is a single # ``tools=None`` request anyway (the kill switch). holder = AgentHolder() - answer_stream: AsyncIterator[ - StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece - ] + # Phase 192: ``AgentPiece`` is the ONE union ``run_agent`` + # yields (the phase-95 truncation piece plus the new + # ``VerbatimBlockPiece``); the deflected path's narrower + # ``chat_stream_retried`` union flows into it unchanged. + answer_stream: AsyncIterator[AgentPiece] deflected_filter: ScaffoldingFilter | None = None if plan.deflected: # Phase 67: the deflected stream goes through the retry @@ -1038,9 +1060,7 @@ async def chat( tools_offered = not plan.deflected async def _pump( - pieces: AsyncIterator[ - StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece - ], + pieces: AsyncIterator[AgentPiece], tools_offered: bool = True, ) -> AsyncIterator[str]: """One request's piece loop (phase 71 extraction): the @@ -1058,10 +1078,14 @@ async def chat( frame, no visible markup) and counted in a local so one debug line can name it. The grounded first pass keeps its current behavior (``tools_offered=True``; ``run_agent`` - suppresses the cap-path calls internally).""" + suppresses the cap-path calls internally). Phase 192 + (task 01): the ``VerbatimBlockPiece`` branch emits the + additive ``verbatim_block`` frame — the verbatim block a + successful ``show_text`` just displayed, so its panel + lands mid-stream instead of at ``done``.""" nonlocal thinking_chars, content_chars, retries_used dropped_calls = 0 # phase 160: tools=None requests only - async for piece in pieces: # StreamPiece | ToolCallPiece | RetryPiece + async for piece in pieces: # app.rag.agent.AgentPiece if isinstance(piece, ToolCallPiece): if not tools_offered: # Phase 160 (task 02): a tools=None request @@ -1170,6 +1194,32 @@ async def chat( ).model_dump() ) continue + if isinstance(piece, VerbatimBlockPiece): + # Phase 192 (task 01, A1): the mid-turn verbatim + # frame — the block the ``show_text`` call just + # displayed, sent at the moment it exists (the + # panel lands while the answer is still + # streaming instead of at ``done``). Additive: + # a refused call yields no piece (no frame), a + # ``answer_text=False`` turn offers no tool (no + # frame) — the frame sequence is byte-identical + # to pre-phase in both cases, and the ``done`` + # frame's ``answer_text`` list and its + # ``verbatim_blocks=N`` count are untouched. + yield sse_event( + ChatVerbatimBlockEvent( + source=piece.block.source, + path=piece.block.path, + title=piece.block.title, + start_line=piece.block.start_line, + end_line=piece.block.end_line, + total_lines=piece.block.total_lines, + text=piece.block.text, + truncated=piece.block.truncated, + locator=piece.block.locator, + ).model_dump() + ) + continue if piece.kind == "thinking": thinking_chars += len(piece.text) if settings.stream_thinking: diff --git a/app/rag/agent.py b/app/rag/agent.py index c9a2114..d10fe26 100644 --- a/app/rag/agent.py +++ b/app/rag/agent.py @@ -233,7 +233,17 @@ task 04): :data:`SHOW_TEXT_NOT_A_TEXT_DOC` refusal, the exact mirror of :data:`NOT_AN_IMAGE_DOC` — never a silent conversion), and the inline ``` ```bor ``` locator-fence form of the same contract (phase 182 D2) - is resolved at the API layer (task 05), NOT here. + is resolved at the API layer (task 05), NOT here. **Phase 192 (task + 01, A1): the block reaches the user at the CALL moment, not at the + end of the answer** — the round loop snapshots + ``len(holder.shown_texts)`` before the execution and yields one + :class:`VerbatimBlockPiece` for whatever the call appended, which the + API layer turns into the additive ``verbatim_block`` SSE frame (the + panel lands while the model is still talking). Success-only, like the + holder entry: a refusal appends nothing and yields nothing. The + ``done`` frame's ``answer_text`` list and the per-turn + ``verbatim_blocks=N`` count (the done-frame blocks) are UNCHANGED — + that log field never counts the mid-turn frames. 4. Every call the model emits is appended back to the message history as the assistant tool-call message + the tool result (refusals included), consumes one round, and the model is called again. Every tool result @@ -344,6 +354,55 @@ from app.rag.verbatim import VerbatimBlock, block_for logger = logging.getLogger("app.agent") + +@dataclass(frozen=True) +class VerbatimBlockPiece: + """One verbatim block a ``show_text`` call JUST displayed (phase 192). + + The mid-turn sibling of :class:`app.rag.llm.ToolResultPiece` (phase + 95 — the other piece the agent loop yields that does NOT come from + the model stream: it is derived from the executed call). Phase 182 + handed the block to the client only on the ``done`` frame's + ``answer_text`` list, so every panel popped in AFTER the model had + finished talking about the bytes; phase 192 (task 01, A1) sends the + very same block the moment it exists, as the additive + ``verbatim_block`` SSE frame — AFTER the round's ``tool`` / + ``tool_result`` frames and BEFORE the next model round. + + SUCCESS-ONLY by construction: the single gate is the + ``holder.shown_texts.append`` in the ``show_text`` branch — every + refusal (missing args, the ``answer_text=False`` posture, no + document, an image row, the block cap, the already-shown dedupe) + returns before it, so a refused call yields NO piece. :attr:`block` + is the exact :class:`app.rag.verbatim.VerbatimBlock` the tool put on + the holder (the server's bytes and the server's extent — LOCKED + A1/A3), never a re-derivation. Frozen like its siblings: an + immutable wire value. The ``done`` frame's ``answer_text`` + list and its ``verbatim_blocks=N`` count are UNCHANGED by this + piece (task 05's resolution still runs over the finished answer); + the mid-turn frames are not counted there. + """ + + #: The block the call displayed — the same object that rode + #: ``holder.shown_texts`` (the phase-182 task-03 lineage: the + #: server resolved it, the tool appended it on success only). + block: VerbatimBlock + + +#: The ONE union of every piece :func:`run_agent` yields — the model +#: stream's pieces (:class:`app.rag.llm.StreamPiece` / +#: :class:`~app.rag.llm.ToolCallPiece` / :class:`~app.rag.llm.RetryPiece`) +#: plus the pieces DERIVED from an executed call: the phase-95 +#: :class:`~app.rag.llm.ToolResultPiece` (a truncated ``read``) and the +#: phase-192 :class:`VerbatimBlockPiece` (a successful ``show_text``, the +#: mid-turn ``verbatim_block`` frame). Named so the wire consumers — the +#: ``app.api.chat`` pump and the test harnesses — spell the SAME union and +#: a future piece type is added in exactly one place. +type AgentPiece = ( + StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece | VerbatimBlockPiece +) + + #: The three agent tools (phase 70: the harness-aligned surface — #: ``ls`` / ``read`` / ``grep``, the pi.dev tool shapes the model was #: trained on, replacing the phase-37 list/read and phase-68 search @@ -2413,7 +2472,7 @@ async def run_agent( settings: Settings, holder: AgentHolder, history: Sequence[dict[str, Any]] = (), -) -> AsyncIterator[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece]: +) -> AsyncIterator[AgentPiece]: """Run the grounded-turn tool loop, yielding every stream piece. Every piece (``thinking`` / ``content`` / tool calls / @@ -2423,7 +2482,18 @@ async def run_agent( longer than ``settings.read_max_chars``) additionally yields one :class:`ToolResultPiece` per truncated call, AFTER the round's ``tool`` frame and BEFORE the next model round (the API layer turns it - into an SSE ``tool_result`` frame, task 02). After the loop finishes, + into an SSE ``tool_result`` frame, task 02). A successful + ``show_text`` (phase 182) additionally yields exactly ONE + :class:`VerbatimBlockPiece` per block it appended to + ``holder.shown_texts``, with the SAME emission-order contract — + AFTER the round's ``tool`` / ``tool_result`` frames and BEFORE the + next model round — so the API layer can put the verbatim panel on + screen the moment the call succeeds instead of at ``done`` (phase + 192, task 01, A1). Emission rides the ``shown_texts`` append, which + is the branch's SINGLE success gate: every refusal (missing ``path``, + the ``answer_text=False`` posture, no document, an ``is_image`` row, + the ``show_text_max_blocks`` cap, the already-shown dedupe) returns + before it and emits nothing. After the loop finishes, *holder* carries the read documents, the executed tool-call count (re-lists included), the ``read_truncations`` signal list, and the ``read_coverage`` spans (phase 141, task 01 — the lines each @@ -2640,6 +2710,12 @@ async def run_agent( # round, so at most one new entry — the loop still iterates the # tail, so a future multi-call round stays correct). trunc_before = len(holder.read_truncations) + # Phase 192 (task 01, A1): the same snapshot-then-yield shape for + # the verbatim blocks — only what THIS call appended becomes a + # ``VerbatimBlockPiece`` (a round adds at most one, the + # already-shown dedupe being a refusal, but the tail loop keeps a + # future multi-add correct). + shown_before = len(holder.shown_texts) with db_factory() as tool_db: result = _execute_tool(tool_db, call, seed_docs, holder, settings) rounds += 1 # every call the model emits consumes a round @@ -2694,6 +2770,15 @@ async def run_agent( chars_shown=shown, chars_total=total, ) + # Phase 192 (task 01, A1): surface this round's verbatim block(s) + # to the API layer — the block the ``show_text`` call JUST + # displayed, sent at the moment it exists instead of at ``done``. + # Same emission order as the truncation piece above (AFTER the + # round's ``tool`` / ``tool_result`` frames, BEFORE the next model + # round); task 02 turns it into the additive ``verbatim_block`` + # SSE frame. A refused call appended nothing, so nothing is sent. + for block in holder.shown_texts[shown_before:]: + yield VerbatimBlockPiece(block=block) if rounds >= max_rounds: logger.warning( "agent round cap reached (rounds=%d) — forcing a final " diff --git a/app/schemas.py b/app/schemas.py index 76afdd3..d8b6d97 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -347,6 +347,46 @@ class ChatToolResultEvent(BaseModel): chars_total: int = Field(ge=0) # the document's true length +class ChatVerbatimBlockEvent(BaseModel): + """SSE frame for one verbatim block a ``show_text`` call JUST displayed + (phase 192, task 01, A1). + + The MID-TURN twin of one :class:`VerbatimBlockRef` item of the ``done`` + frame's ``answer_text`` list — the same field set with the same caps, + sent the moment the block exists instead of at the end of the answer + (``TODO.md`` L5: "the verbatim tag appear[s] at the end and only when + the llm is done answering which makes the chat read strangely"). The + frame is strictly ADDITIVE (the phase-95 ``tool_result`` precedent): + it lands AFTER the round's ``tool`` / ``tool_result`` frames and + BEFORE the next model round, clients that do not know the type ignore + it, a REFUSED ``show_text`` streams NO frame (the holder append is the + single success gate), and a ``BOR_ANSWER_TEXT=0`` turn offers no tool + at all — its frame sequence is byte-identical to pre-phase. The + ``done`` frame's ``answer_text`` list, the saved record and the + restore path keep the exact phase-182 contract (the client's + idempotence keeps the same block from painting twice). + + ``isVerbatimBlock`` (``frontend/assets/verbatim.js``) accepts the + event as-is: the key set is the done-frame item's key set, so the one + panel client hydrates a mid-turn frame with no new shape. The field + caps are :class:`VerbatimBlockRef`'s verbatim — a unit pin fails the + day the two drift. + """ + + type: Literal["verbatim_block"] = "verbatim_block" + source: str = Field(max_length=120) + path: str = Field(max_length=1000) + title: str = Field(max_length=500) + start_line: int = Field(ge=0) + end_line: int = Field(ge=0) + total_lines: int = Field(ge=0) + #: The verbatim bytes themselves — SERVER-supplied, never model output + #: (the :attr:`VerbatimBlockRef.text` contract, same cap). + text: str = Field(max_length=600_000) + truncated: bool = False + locator: str = Field(max_length=2100) + + class ChatDoneEvent(BaseModel): """Final SSE event of a chat turn: metadata for the finished answer. diff --git a/frontend/assets/app.js b/frontend/assets/app.js index e69b1e7..c693409 100644 --- a/frontend/assets/app.js +++ b/frontend/assets/app.js @@ -4640,6 +4640,30 @@ async function runTurn(text, { reask = false, image = null } = {}) { } } } + } else if (ev.type === "verbatim_block") { + // Phase 192 (task 02, A2): the show_text panel lands at the moment + // the call succeeds — the fence it names hydrates IN PLACE; an + // unfenced block lands at the answer position now, not at done. + // The done pass re-runs the same functions and can never double this + // panel: hydrateVerbatim is idempotent (only .verbatim-pending is + // touched), it DROPS this appended twin when the answer fences the + // same locator later (dropPlacedTwin, phase 192 A3), and + // appendUnlocatedVerbatim skips an already-placed locator. NO + // client-side answer_text gate: the server's gate is the only one + // (a BOR_ANSWER_TEXT=0 turn sends no frame, so this branch never + // fires — a second, client-side gate would only drift). + // `wrap` always exists here (the round's `tool` frame created the + // live bubble) but the optional chain keeps a frame that arrives + // without one a silent no-op, like the `tool_result` branch. + const n = hydrateVerbatimNow( + wrap?.querySelector(".bubble"), + ev, + VERBATIM_LINK_OPTS + ); + // The append is the ZERO-match path only: a fence already on screen + // keeps its panel IN PLACE (the model's order), and only a block no + // placeholder names lands at the answer position. + if (n === 0) appendUnlocatedVerbatim(wrap, [ev], VERBATIM_LINK_OPTS); } else if (ev.type === "error") { // Phase 182 (task 07): an `error` frame is terminal — the `done` // hydrate will never run, so a placeholder that streamed in stays at diff --git a/frontend/assets/verbatim.js b/frontend/assets/verbatim.js index 150b590..1602b6a 100644 --- a/frontend/assets/verbatim.js +++ b/frontend/assets/verbatim.js @@ -6,11 +6,13 @@ * locator, and the SERVER (`app.rag.verbatim.resolve_answer_blocks`, * task 05) puts the real bytes on the `done` frame as `answer_text`. This * file is the other half of that handshake: it RESOLVES the placeholders - * against the frame's blocks (`hydrateVerbatim`), places the blocks the - * answer never located (`appendUnlocatedVerbatim`), settles leftovers when - * a turn ends without a `done` (`settleVerbatimPlaceholders`), gives every - * settled answer code block a Copy button (`enhanceAnswerCodeBlocks`) and - * owns the ONE clipboard path (`copyTextWithFallback`). + * against the frame's blocks (`hydrateVerbatim`), settles the ONE block a + * mid-turn `verbatim_block` frame carries (`hydrateVerbatimNow`, phase 192), + * places the blocks the answer never located (`appendUnlocatedVerbatim`), + * settles leftovers when a turn ends without a `done` + * (`settleVerbatimPlaceholders`), gives every settled answer code block a + * Copy button (`enhanceAnswerCodeBlocks`) and owns the ONE clipboard path + * (`copyTextWithFallback`). * * WHY IT IS A CLASSIC SCRIPT, NOT A MODULE (the `markdown.js` pattern): * `index.html` and `shared.html` load it as a plain `