From 4677d86f494186140232ef938e35230ec0322382 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Tue, 1 Sep 2026 23:51:43 -0400 Subject: [PATCH] =?UTF-8?q?feat(sources):=20real-time=20file=20progress=20?= =?UTF-8?q?for=20sync=20and=20upload=20=E2=80=94=20background=20upload=20w?= =?UTF-8?q?ith=20success=20toast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../61_stale_ui_copy/00_phase.md | 0 .../61_stale_ui_copy/01_chat_page_copy.md | 0 .../61_stale_ui_copy/02_sources_pages_copy.md | 0 .../61_stale_ui_copy/03_footer_defaults.md | 0 .../61_stale_ui_copy/04_tests_and_commit.md | 0 .../62_ui_customization/00_phase.md | 0 .../62_ui_customization/01_config_settings.md | 0 .../02_brand_layer_customization.md | 0 .../62_ui_customization/03_theme_assets.md | 0 .../62_ui_customization/04_docs.md | 0 .../05_e2e_customization.md | 0 .../00_phase.md | 54 + .../01_agent_list_format.md | 75 ++ .../02_mock_and_unit_tests.md | 85 ++ .../03_e2e_document_tools.md | 44 + .../04_docs.md | 44 + .../01_importer_progress_hook.md | 0 .../02_sync_status_current_file.md | 0 .../03_upload_background_202.md | 0 .../04_sync_button_live_file.md | 0 .../05_upload_toast_progress.md | 0 .../61_stale_ui_copy__00_phase.a1.err | 0 .../61_stale_ui_copy__00_phase.a1.md | 19 + .../61_stale_ui_copy__00_phase.a1.validate | 71 ++ ...61_stale_ui_copy__01_chat_page_copy.a1.err | 0 .../61_stale_ui_copy__01_chat_page_copy.a1.md | 17 + ...ale_ui_copy__01_chat_page_copy.a1.validate | 71 ++ ...tale_ui_copy__02_sources_pages_copy.a1.err | 0 ...stale_ui_copy__02_sources_pages_copy.a1.md | 19 + ...ui_copy__02_sources_pages_copy.a1.validate | 71 ++ ...1_stale_ui_copy__03_footer_defaults.a1.err | 0 ...61_stale_ui_copy__03_footer_defaults.a1.md | 14 + ...le_ui_copy__03_footer_defaults.a1.validate | 71 ++ ..._stale_ui_copy__04_tests_and_commit.a1.err | 0 ...1_stale_ui_copy__04_tests_and_commit.a1.md | 17 + ...e_ui_copy__04_tests_and_commit.a1.validate | 71 ++ .../62_ui_customization__00_phase.a1.err | 0 .../62_ui_customization__00_phase.a1.md | 14 + .../62_ui_customization__00_phase.a1.validate | 71 ++ ...i_customization__01_config_settings.a1.err | 0 ...ui_customization__01_config_settings.a1.md | 19 + ...tomization__01_config_settings.a1.validate | 71 ++ ...ation__02_brand_layer_customization.a1.err | 0 ...zation__02_brand_layer_customization.a1.md | 23 + ...__02_brand_layer_customization.a1.validate | 71 ++ ...2_ui_customization__03_theme_assets.a1.err | 0 ...62_ui_customization__03_theme_assets.a1.md | 21 + ...customization__03_theme_assets.a1.validate | 71 ++ .../62_ui_customization__04_docs.a1.err | 0 .../62_ui_customization__04_docs.a1.md | 12 + .../62_ui_customization__04_docs.a1.validate | 71 ++ ...customization__05_e2e_customization.a1.err | 0 ..._customization__05_e2e_customization.a1.md | 16 + ...mization__05_e2e_customization.a1.validate | 71 ++ ...mbiguous_document_listing__00_phase.a1.err | 0 ...ambiguous_document_listing__00_phase.a1.md | 17 + ...ous_document_listing__00_phase.a1.validate | 71 ++ ...ument_listing__01_agent_list_format.a1.err | 0 ...cument_listing__01_agent_list_format.a1.md | 11 + ..._listing__01_agent_list_format.a1.validate | 71 ++ ...ent_listing__02_mock_and_unit_tests.a1.err | 0 ...ment_listing__02_mock_and_unit_tests.a1.md | 19 + ...isting__02_mock_and_unit_tests.a1.validate | 71 ++ ...ment_listing__03_e2e_document_tools.a1.err | 0 ...ument_listing__03_e2e_document_tools.a1.md | 16 + ...listing__03_e2e_document_tools.a1.validate | 71 ++ ...ambiguous_document_listing__04_docs.a1.err | 0 ...nambiguous_document_listing__04_docs.a1.md | 18 + ...uous_document_listing__04_docs.a1.validate | 71 ++ ...progress__01_importer_progress_hook.a1.err | 0 ..._progress__01_importer_progress_hook.a1.md | 15 + ...ess__01_importer_progress_hook.a1.validate | 71 ++ ...ogress__02_sync_status_current_file.a1.err | 0 ...rogress__02_sync_status_current_file.a1.md | 17 + ...s__02_sync_status_current_file.a1.validate | 72 ++ ..._progress__03_upload_background_202.a1.err | 0 ...d_progress__03_upload_background_202.a1.md | 20 + ...ress__03_upload_background_202.a1.validate | 72 ++ ..._progress__04_sync_button_live_file.a1.err | 0 ...d_progress__04_sync_button_live_file.a1.md | 14 + ...ress__04_sync_button_live_file.a1.validate | 72 ++ ..._progress__05_upload_toast_progress.a1.err | 0 ...d_progress__05_upload_toast_progress.a1.md | 19 + ...ress__05_upload_toast_progress.a1.validate | 72 ++ ...ogress__06_e2e_sync_upload_progress.a1.err | 0 .agent/user_stories/agent-document-tools.md | 3 +- app/api/git_sources.py | 427 +++++--- app/api/sync.py | 44 +- app/rag/importer.py | 32 + app/schemas.py | 22 +- frontend/assets/git-sources.js | 316 +++++- frontend/assets/sources.js | 187 +++- frontend/assets/styles.css | 13 +- frontend/git-sources.html | 31 +- frontend/sources.html | 10 +- tests/e2e/slow_llm.py | 93 ++ tests/e2e/test_archive_upload_sources.py | 208 +++- tests/e2e/test_sync_upload_progress.py | 758 ++++++++++++++ tests/integration/test_git_sources_upload.py | 932 ++++++++++++++---- tests/integration/test_sync_api.py | 16 +- tests/unit/test_frontend_sync_upload.py | 663 +++++++++++++ tests/unit/test_importer.py | 138 +++ tests/unit/test_sync_button.py | 323 +++++- 103 files changed, 5914 insertions(+), 456 deletions(-) rename .agent/phases/{todo => complete}/61_stale_ui_copy/00_phase.md (100%) rename .agent/phases/{todo => complete}/61_stale_ui_copy/01_chat_page_copy.md (100%) rename .agent/phases/{todo => complete}/61_stale_ui_copy/02_sources_pages_copy.md (100%) rename .agent/phases/{todo => complete}/61_stale_ui_copy/03_footer_defaults.md (100%) rename .agent/phases/{todo => complete}/61_stale_ui_copy/04_tests_and_commit.md (100%) rename .agent/phases/{todo => complete}/62_ui_customization/00_phase.md (100%) rename .agent/phases/{todo => complete}/62_ui_customization/01_config_settings.md (100%) rename .agent/phases/{todo => complete}/62_ui_customization/02_brand_layer_customization.md (100%) rename .agent/phases/{todo => complete}/62_ui_customization/03_theme_assets.md (100%) rename .agent/phases/{todo => complete}/62_ui_customization/04_docs.md (100%) rename .agent/phases/{todo => complete}/62_ui_customization/05_e2e_customization.md (100%) create mode 100644 .agent/phases/complete/63_unambiguous_document_listing/00_phase.md create mode 100644 .agent/phases/complete/63_unambiguous_document_listing/01_agent_list_format.md create mode 100644 .agent/phases/complete/63_unambiguous_document_listing/02_mock_and_unit_tests.md create mode 100644 .agent/phases/complete/63_unambiguous_document_listing/03_e2e_document_tools.md create mode 100644 .agent/phases/complete/63_unambiguous_document_listing/04_docs.md rename .agent/phases/{todo => complete}/64_sync_upload_progress/01_importer_progress_hook.md (100%) rename .agent/phases/{todo => complete}/64_sync_upload_progress/02_sync_status_current_file.md (100%) rename .agent/phases/{todo => complete}/64_sync_upload_progress/03_upload_background_202.md (100%) rename .agent/phases/{todo => complete}/64_sync_upload_progress/04_sync_button_live_file.md (100%) rename .agent/phases/{todo => complete}/64_sync_upload_progress/05_upload_toast_progress.md (100%) create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.err create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.md create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.validate create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.err create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.md create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.validate create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.err create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.md create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.validate create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.err create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.md create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.validate create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.err create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.md create mode 100644 .agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.validate create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.err create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.md create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.validate create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.err create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.md create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.validate create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.err create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.md create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.validate create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.err create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.md create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.validate create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.err create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.md create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.validate create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.err create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.md create mode 100644 .agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.validate create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.err create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.md create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.validate create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.err create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.md create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.validate create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.err create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.md create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.validate create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.err create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.md create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.validate create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.err create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.md create mode 100644 .agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.validate create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.err create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.md create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.validate create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.err create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.md create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.validate create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.err create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.md create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.validate create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.err create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.md create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.validate create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.err create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.md create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.validate create mode 100644 .agent/reports/64_sync_upload_progress/64_sync_upload_progress__06_e2e_sync_upload_progress.a1.err create mode 100644 tests/e2e/slow_llm.py create mode 100644 tests/e2e/test_sync_upload_progress.py create mode 100644 tests/unit/test_frontend_sync_upload.py diff --git a/.agent/phases/todo/61_stale_ui_copy/00_phase.md b/.agent/phases/complete/61_stale_ui_copy/00_phase.md similarity index 100% rename from .agent/phases/todo/61_stale_ui_copy/00_phase.md rename to .agent/phases/complete/61_stale_ui_copy/00_phase.md diff --git a/.agent/phases/todo/61_stale_ui_copy/01_chat_page_copy.md b/.agent/phases/complete/61_stale_ui_copy/01_chat_page_copy.md similarity index 100% rename from .agent/phases/todo/61_stale_ui_copy/01_chat_page_copy.md rename to .agent/phases/complete/61_stale_ui_copy/01_chat_page_copy.md diff --git a/.agent/phases/todo/61_stale_ui_copy/02_sources_pages_copy.md b/.agent/phases/complete/61_stale_ui_copy/02_sources_pages_copy.md similarity index 100% rename from .agent/phases/todo/61_stale_ui_copy/02_sources_pages_copy.md rename to .agent/phases/complete/61_stale_ui_copy/02_sources_pages_copy.md diff --git a/.agent/phases/todo/61_stale_ui_copy/03_footer_defaults.md b/.agent/phases/complete/61_stale_ui_copy/03_footer_defaults.md similarity index 100% rename from .agent/phases/todo/61_stale_ui_copy/03_footer_defaults.md rename to .agent/phases/complete/61_stale_ui_copy/03_footer_defaults.md diff --git a/.agent/phases/todo/61_stale_ui_copy/04_tests_and_commit.md b/.agent/phases/complete/61_stale_ui_copy/04_tests_and_commit.md similarity index 100% rename from .agent/phases/todo/61_stale_ui_copy/04_tests_and_commit.md rename to .agent/phases/complete/61_stale_ui_copy/04_tests_and_commit.md diff --git a/.agent/phases/todo/62_ui_customization/00_phase.md b/.agent/phases/complete/62_ui_customization/00_phase.md similarity index 100% rename from .agent/phases/todo/62_ui_customization/00_phase.md rename to .agent/phases/complete/62_ui_customization/00_phase.md diff --git a/.agent/phases/todo/62_ui_customization/01_config_settings.md b/.agent/phases/complete/62_ui_customization/01_config_settings.md similarity index 100% rename from .agent/phases/todo/62_ui_customization/01_config_settings.md rename to .agent/phases/complete/62_ui_customization/01_config_settings.md diff --git a/.agent/phases/todo/62_ui_customization/02_brand_layer_customization.md b/.agent/phases/complete/62_ui_customization/02_brand_layer_customization.md similarity index 100% rename from .agent/phases/todo/62_ui_customization/02_brand_layer_customization.md rename to .agent/phases/complete/62_ui_customization/02_brand_layer_customization.md diff --git a/.agent/phases/todo/62_ui_customization/03_theme_assets.md b/.agent/phases/complete/62_ui_customization/03_theme_assets.md similarity index 100% rename from .agent/phases/todo/62_ui_customization/03_theme_assets.md rename to .agent/phases/complete/62_ui_customization/03_theme_assets.md diff --git a/.agent/phases/todo/62_ui_customization/04_docs.md b/.agent/phases/complete/62_ui_customization/04_docs.md similarity index 100% rename from .agent/phases/todo/62_ui_customization/04_docs.md rename to .agent/phases/complete/62_ui_customization/04_docs.md diff --git a/.agent/phases/todo/62_ui_customization/05_e2e_customization.md b/.agent/phases/complete/62_ui_customization/05_e2e_customization.md similarity index 100% rename from .agent/phases/todo/62_ui_customization/05_e2e_customization.md rename to .agent/phases/complete/62_ui_customization/05_e2e_customization.md diff --git a/.agent/phases/complete/63_unambiguous_document_listing/00_phase.md b/.agent/phases/complete/63_unambiguous_document_listing/00_phase.md new file mode 100644 index 0000000..4899f07 --- /dev/null +++ b/.agent/phases/complete/63_unambiguous_document_listing/00_phase.md @@ -0,0 +1,54 @@ +# Phase 63 — Unambiguous Document Listing for Agent Tools + +**Source:** Live troubleshooting report (2026-09-01) — the `brain-of-reese` agent at `https://brain.experimental.reeseapps.com/` cannot read documents because the `list_documents` tool output uses an ambiguous `source/path — title` format that the LLM cannot reliably parse when paths contain `/` characters. + +**Story:** `.agent/user_stories/agent-document-tools.md` (phase 37, extended) + +**Context:** The agent's grounded-turn loop uses two tools: `list_documents` (returns the catalog) and `read_document` (reads one document). The catalog output is formatted as `source/path — title` (one line per document), but when a file lives inside a subdirectory of its source, the output looks like `brain-of-reese-main/homelab/aws-route53.md — aws-route53`. The LLM cannot reliably determine where `source` ends and `path` begins because both contain `/` separators. The thinking trace shows the LLM cycling through multiple failed attempts, each time guessing a different split. + +The fix changes the catalog format to unambiguous `source: X | path: Y | title: Z` lines and updates the tool descriptions to tell the LLM how to parse them. + +## Objective + +The agent can always extract the correct `(source, path)` pair from the `list_documents` output, regardless of how many `/` characters the path contains. + +## Dependencies + +- `62_ui_customization` (todo, preceding — functional: no code changes in this phase touch UI or config) + +## Tasks + +1. `01_agent_list_format.md` — change `list_catalog` output from `source/path — title` to `source: X | path: Y | title: Z`; update the `read_document` tool descriptions to reference the new format. +2. `02_mock_and_unit_tests.md` — update `tests/e2e/mock_llm.py::_catalog_docs` to parse the new format; update `tests/unit/test_agent.py` (the "No document at" refusal line is unchanged — it names the lookup failure, not the list format). +3. `03_e2e_document_tools.md` — validate that `tests/e2e/test_agent_document_tools.py` and `tests/e2e/test_agent_unlimited_tools.py` still pass (they assert on SSE frames and UI rendering, not on the catalog text format — the tool call `argument` field `source/path` is unchanged). +4. `04_docs.md` — update any inline documentation that references the old format. + +## Testing & Quality + +- Unit: `tests/unit/test_agent.py` — the "No document at" refusal message is unchanged (it reports the lookup failure, not the list format). The `_catalog_docs` mock parsing must produce identical `(source, path)` tuples for all existing test catalogs. +- Integration: `tests/integration/test_agent_tools.py` — `list_catalog` ordering assertions are unchanged (the function returns `[(source, path, title), ...]` tuples; only the string formatting in `_execute_tool` changes). +- E2E (mandatory, house rule): `tests/e2e/test_agent_document_tools.py` and `tests/e2e/test_agent_unlimited_tools.py` run in isolation — the SSE `argument` field (`source/path`) is unchanged; the mock's `_catalog_docs` parsing produces identical results; the tool lines and source chips render identically. +- Coverage: **>90%** on `app/` (validate.sh gate). + +## Completion Criteria + +- [ ] `list_catalog` output uses `source: X | path: Y | title: Z` format — verifiable by a unit test that patches `list_catalog` and checks the formatted string. +- [ ] The `read_document` tool descriptions tell the LLM to extract `source` and `path` from the `source:` / `path:` labels. +- [ ] `tests/e2e/mock_llm.py::_catalog_docs` parses the new format and produces identical `(source, path)` tuples for all existing test catalogs. +- [ ] `uv run pytest` green; coverage TOTAL >90%; `uv run ruff check . && uv run pyright` clean. +- [ ] `uv run pytest tests/e2e/test_agent_document_tools.py tests/e2e/test_agent_unlimited_tools.py -v --no-cov` green in isolation (DB up). +- [ ] Regression E2E suites green in isolation: `test_chat_api.py`, `test_sse_events.py`, `test_whole_document_context.py`. +- [ ] One `--no-gpg-sign` commit; phase dir moved to `.agent/phases/complete/` (`.agent/` stays untracked). + +## Locked decisions + +- **Owner-locked (2026-09-01, troubleshooting report, A1):** the new format uses `source: X | path: Y | title: Z` — one line per document, pipe-delimited fields, no ambiguity even with deeply nested paths. +- **Owner-locked (2026-09-01, troubleshooting report, A2):** the `read_document` tool descriptions are updated to reference the new format (tell the LLM how to parse `source:` and `path:` labels). +- **Owner-locked (2026-09-01, troubleshooting report, A3):** the SSE `argument` field (`source/path`) is unchanged — it is the tool call argument, not the list output. The frontend tool-line rendering is unchanged. +- **Owner-locked (2026-09-01, troubleshooting report, A4):** the "No document at …" refusal message is unchanged — it reports the lookup failure, not the list format. + +## Commit + +```bash +git add app/ tests/ && git commit --no-gpg-sign -m "fix(agent): unambiguous document listing format for LLM parsing" +``` diff --git a/.agent/phases/complete/63_unambiguous_document_listing/01_agent_list_format.md b/.agent/phases/complete/63_unambiguous_document_listing/01_agent_list_format.md new file mode 100644 index 0000000..46da683 --- /dev/null +++ b/.agent/phases/complete/63_unambiguous_document_listing/01_agent_list_format.md @@ -0,0 +1,75 @@ +# Task 01 — Change `list_catalog` output format + update tool descriptions + +**Goal:** Make the `list_documents` catalog output unambiguous for LLM parsing, and update the `read_document` tool descriptions to tell the LLM how to extract `source` and `path`. + +## Changes + +### `app/rag/agent.py` + +#### 1. Change `list_catalog` output format + +In `_execute_tool`, the `list_documents` handler formats the catalog as: + +```python +# BEFORE (ambiguous when path contains /) +listing = f"{len(rows)} documents:\n" + "\n".join( + f"{source}/{path} — {title}" for source, path, title in rows +) +``` + +Change to: + +```python +# AFTER (unambiguous — each field is labeled) +listing = f"{len(rows)} documents:\n" + "\n".join( + f"source: {source} | path: {path} | title: {title}" + for source, path, title in rows +) +``` + +#### 2. Update `read_document` tool descriptions + +In `AGENT_TOOLS`, update the `source` and `path` parameter descriptions to reference the new format: + +```python +# BEFORE +"source": { + "description": ( + "The document's source (a directory basename, e.g. 'Homelab')." + ), +}, +"path": { + "description": ( + "The document's path relative to its source directory." + ), +}, + +# AFTER +"source": { + "description": ( + "The document's source, as shown after 'source: ' in the " + "list_documents output (e.g. 'Homelab' from " + "'source: Homelab | path: homelab/aws-route53.md')." + ), +}, +"path": { + "description": ( + "The document's path, as shown after 'path: ' in the " + "list_documents output (e.g. 'homelab/aws-route53.md' from " + "'source: Homelab | path: homelab/aws-route53.md')." + ), +}, +``` + +### `app/rag/prompts.py` + +No changes needed — the prompt text references the tool names but not the output format. + +## Files changed + +- `app/rag/agent.py` — `list_catalog` formatting + `AGENT_TOOLS` descriptions + +## Verification + +- `uv run pytest tests/unit/test_agent.py -v --no-cov` — the tool shape tests (`test_agent_tools_names_and_parameters`) check the description strings, so they must be updated to match. +- `uv run ruff check app/rag/agent.py && uv run pyright app/rag/agent.py` — lint + types clean. diff --git a/.agent/phases/complete/63_unambiguous_document_listing/02_mock_and_unit_tests.md b/.agent/phases/complete/63_unambiguous_document_listing/02_mock_and_unit_tests.md new file mode 100644 index 0000000..041f55f --- /dev/null +++ b/.agent/phases/complete/63_unambiguous_document_listing/02_mock_and_unit_tests.md @@ -0,0 +1,85 @@ +# Task 02 — Update mock parsing + unit tests for the new format + +**Goal:** Update `tests/e2e/mock_llm.py::_catalog_docs` to parse the new `source: X | path: Y | title: Z` format, and update `tests/unit/test_agent.py` to match the new tool descriptions. + +## Changes + +### `tests/e2e/mock_llm.py` — `_catalog_docs` function + +The current parsing uses `head.rpartition("/")` on the `source/path` head: + +```python +# BEFORE +def _catalog_docs(body: dict[str, Any]) -> list[tuple[str, str]]: + docs: list[tuple[str, str]] = [] + for m in _messages(body): + if m.get("role") != "tool": + continue + content = str(m.get("content") or "") + if content.startswith(_READ_RESULT_PREFIX): + continue + for line in content.splitlines(): + head = line.split(" — ", 1)[0].strip() + if "/" in head: + source, _, path = head.rpartition("/") + if source and path: + docs.append((source, path)) + return docs +``` + +Change to parse the new labeled format: + +```python +# AFTER +def _catalog_docs(body: dict[str, Any]) -> list[tuple[str, str]]: + docs: list[tuple[str, str]] = [] + for m in _messages(body): + if m.get("role") != "tool": + continue + content = str(m.get("content") or "") + if content.startswith(_READ_RESULT_PREFIX): + continue + for line in content.splitlines(): + # New format: "source: X | path: Y | title: Z" + if not line.startswith("source: "): + continue + parts = line.split(" | ", 2) + if len(parts) < 3: + continue + source = parts[0].removeprefix("source: ").strip() + path_part = parts[1] + if not path_part.startswith("path: "): + continue + path = path_part.removeprefix("path: ").strip() + if source and path: + docs.append((source, path)) + return docs +``` + +### `tests/unit/test_agent.py` — tool description assertions + +The test `test_agent_tools_names_and_parameters` asserts the exact description strings: + +```python +# BEFORE (line ~109) +assert by_name["read_document"]["function"]["description"] == ( + "Add the full content of one more indexed document to your context" +) +``` + +This assertion is about the function description (not parameter descriptions), so it stays the same. But the test file may have inline assertions about the `source` and `path` parameter descriptions — check and update if any exist. + +### `tests/unit/test_agent.py` — refusal message + +The refusal message `"No document at S/ghost.md — check the list_documents output."` is unchanged — it reports the lookup failure, not the list format. No changes needed here. + +## Files changed + +- `tests/e2e/mock_llm.py` — `_catalog_docs` parsing +- `tests/unit/test_agent.py` — only if inline assertions reference the old format (verify) + +## Verification + +- `uv run pytest tests/unit/test_agent.py -v --no-cov` — all agent unit tests pass. +- `uv run pytest tests/e2e/test_agent_document_tools.py -v --no-cov` — the mock's `_catalog_docs` must produce identical `(source, path)` tuples for the test catalogs. +- `uv run ruff check tests/ && uv run pyright tests/` — lint + types clean. diff --git a/.agent/phases/complete/63_unambiguous_document_listing/03_e2e_document_tools.md b/.agent/phases/complete/63_unambiguous_document_listing/03_e2e_document_tools.md new file mode 100644 index 0000000..f8975ff --- /dev/null +++ b/.agent/phases/complete/63_unambiguous_document_listing/03_e2e_document_tools.md @@ -0,0 +1,44 @@ +# Task 03 — E2E document tools validation + +**Goal:** Run the E2E suites for agent document tools and verify they still pass with the new catalog format. + +## What to verify + +The E2E tests assert on: +1. **SSE frames** — the `argument` field of `tool` events is `source/path` (e.g., `"Deployments/example-record-file.json"`). This is the tool call argument, NOT the list output format. **Unchanged.** +2. **UI rendering** — tool lines show "Listing documents" and "Reading ". **Unchanged.** +3. **Source chips** — display `${s.source}/${s.path}`. **Unchanged.** +4. **Answer content** — the mock's deterministic answer quoting the read document. **Unchanged** (the mock reads the same `(source, path)` tuples from the catalog). +5. **Query log** — sources field stores `"source/path, source/path"`. **Unchanged.** + +The only thing that changes is the **text of the catalog** that the mock parses. The mock's `_catalog_docs` function must produce identical `(source, path)` tuples for the test catalogs, which means the tool flow classification (`_tool_flow`) and the read step will work identically. + +## Run commands + +```bash +# DB must be up: podman compose up -d db +uv run pytest tests/e2e/test_agent_document_tools.py -v --no-cov +uv run pytest tests/e2e/test_agent_unlimited_tools.py -v --no-cov +``` + +## Regression suites to run + +```bash +uv run pytest tests/e2e/test_chat_api.py -v --no-cov +uv run pytest tests/e2e/test_whole_document_context.py -v --no-cov +uv run pytest tests/e2e/test_sse_events.py -v --no-cov +``` + +## Expected outcome + +All tests pass without modification (aside from the mock parsing change in task 02). The catalog text format is an internal detail of the agent loop — the tool call arguments, SSE frames, UI rendering, and answer content are all unchanged. + +## If tests fail + +1. Check that `_catalog_docs` produces the same `(source, path)` tuples as before for the test catalogs. +2. Verify the mock's `_tool_flow` classification still works (it calls `_catalog_docs`). +3. Check that the SSE `argument` field is still `source/path` (it should be — `_execute_tool` builds it from `call.arguments["source"] + "/" + call.arguments["path"]`, which is unchanged). + +## Files changed + +- None (verification only — all changes were in tasks 01 and 02) diff --git a/.agent/phases/complete/63_unambiguous_document_listing/04_docs.md b/.agent/phases/complete/63_unambiguous_document_listing/04_docs.md new file mode 100644 index 0000000..7af68e6 --- /dev/null +++ b/.agent/phases/complete/63_unambiguous_document_listing/04_docs.md @@ -0,0 +1,44 @@ +# Task 04 — Update documentation + +**Goal:** Update all documentation that references the old `source/path — title` catalog format to the new `source: X | path: Y | title: Z` format. + +## Files to update + +### `README.md` — Agent document tools section (~L168) + +```markdown +# BEFORE +* **`list_documents`** — lists every indexed document, one + `source/path — title` line each (the same order as the Sources page); + +# AFTER +* **`list_documents`** — lists every indexed document, one + `source: X | path: Y | title: Z` line each (the same order as the + Sources page); +``` + +### `.agent/user_stories/agent-document-tools.md` — Acceptance criteria (~L38) + +```markdown +# BEFORE +3. `app/rag/agent.py`: the loop — budgets from + `BOR_AGENT_LIST_CALLS` / `BOR_AGENT_READ_CALLS` (default 1/1); the + `list_documents` tool returns the DB catalog + (`source/path — title` lines, `/api/docs` order); + +# AFTER +3. `app/rag/agent.py`: the loop — budgets from + `BOR_AGENT_LIST_CALLS` / `BOR_AGENT_READ_CALLS` (default 1/1); the + `list_documents` tool returns the DB catalog + (`source: X | path: Y | title: Z` lines, `/api/docs` order); +``` + +## Verification + +- `grep -n "source/path — title\|source/path - title" README.md .agent/user_stories/*.md` should return no results. +- `uv run ruff check README.md .agent/` — lint clean (markdown files are not linted by ruff, but the command should not error). + +## Files changed + +- `README.md` — Agent document tools section +- `.agent/user_stories/agent-document-tools.md` — Acceptance criteria item 3 diff --git a/.agent/phases/todo/64_sync_upload_progress/01_importer_progress_hook.md b/.agent/phases/complete/64_sync_upload_progress/01_importer_progress_hook.md similarity index 100% rename from .agent/phases/todo/64_sync_upload_progress/01_importer_progress_hook.md rename to .agent/phases/complete/64_sync_upload_progress/01_importer_progress_hook.md diff --git a/.agent/phases/todo/64_sync_upload_progress/02_sync_status_current_file.md b/.agent/phases/complete/64_sync_upload_progress/02_sync_status_current_file.md similarity index 100% rename from .agent/phases/todo/64_sync_upload_progress/02_sync_status_current_file.md rename to .agent/phases/complete/64_sync_upload_progress/02_sync_status_current_file.md diff --git a/.agent/phases/todo/64_sync_upload_progress/03_upload_background_202.md b/.agent/phases/complete/64_sync_upload_progress/03_upload_background_202.md similarity index 100% rename from .agent/phases/todo/64_sync_upload_progress/03_upload_background_202.md rename to .agent/phases/complete/64_sync_upload_progress/03_upload_background_202.md diff --git a/.agent/phases/todo/64_sync_upload_progress/04_sync_button_live_file.md b/.agent/phases/complete/64_sync_upload_progress/04_sync_button_live_file.md similarity index 100% rename from .agent/phases/todo/64_sync_upload_progress/04_sync_button_live_file.md rename to .agent/phases/complete/64_sync_upload_progress/04_sync_button_live_file.md diff --git a/.agent/phases/todo/64_sync_upload_progress/05_upload_toast_progress.md b/.agent/phases/complete/64_sync_upload_progress/05_upload_toast_progress.md similarity index 100% rename from .agent/phases/todo/64_sync_upload_progress/05_upload_toast_progress.md rename to .agent/phases/complete/64_sync_upload_progress/05_upload_toast_progress.md diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.err b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.md b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.md new file mode 100644 index 0000000..a4c08c1 --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.md @@ -0,0 +1,19 @@ +All verification passes green. Phase 61 is fully complete — final verification pass only, no code changes needed. + +**Phase 61 (stale_ui_copy) — final verification report** + +Verified (all 4 task files already in `complete/`; no work remaining, no defects found): +- Chat page: meta description, empty-state sub, composer placeholder, and the 4 default chips in `app/config.py` match the locked (A1/A2) copy exactly +- KB page-sub (`sources.html`) reads the locked current-source-model text (git repos + local dirs + uploaded archives, Sync pulls/re-imports); git-sources placeholder is `https://github.com/you/your-repo.git` (A3) +- All 9 pages carry `Powered by self-hosted models` (1 each); `grep -rn "Reese's" frontend/` empty; only remaining "homelab" is the A3-exempt L176 code comment in `index.html` +- Tests: `tests/unit/test_stale_ui_copy.py` + `tests/e2e/test_stale_ui_copy.py` exist with real pins (body-text scan, DOM assertions) + +Results: +- `uv run pytest --cov=app --cov-report=term-missing`: **1201 passed**, coverage TOTAL **99%** (>90% ✓) +- `uv run ruff check . && uv run pyright`: clean (0 errors) +- `uv run pytest tests/e2e/test_stale_ui_copy.py -v --no-cov`: **5 passed** in isolation (DB up) +- Regression E2E in isolation: `test_smoke` 3 ✓, `test_configurable_brand` 5 ✓, `test_header_consistency` 3 ✓, `test_dark_tech_theme` 6 ✓ +- Commit `baefcde` `--no-gpg-sign` exists, staged paths only `app/ frontend/ tests/` ✓ + +Completion criteria: all 7 met (E2E body scan, KB page-sub, 9 footers, suite/coverage/lint, phase E2E, regression suites, single commit). +Next pending phase: **62_ui_customization**. diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.validate b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.validate new file mode 100644 index 0000000..c85b8df --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__00_phase.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 101 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2458 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.err b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.md b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.md new file mode 100644 index 0000000..6fc9f03 --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.md @@ -0,0 +1,17 @@ +**Task 01 complete — report** + +Implemented: +- `frontend/index.html`: meta description, `.empty-state-sub`, and `#message-input` placeholder → locked neutral copy (A1); brand-embedding strings left untouched for the phase-39 brand layer. +- `app/config.py`: `suggestions` default → the four locked chips (A2); `BOR_SUGGESTIONS` override path unchanged. +- Defect fix (pre-existing — 3 `tests/integration/test_api.py` failures even on the pristine tree): the operator's local gitignored `.env` leaked into test apps via pydantic `env_file=".env"`. Made the test env hermetic — `tests/conftest.py` + `tests/e2e/conftest.py` now force `BOR_DOCS_REPO=""` and `BOR_SUGGESTIONS`=code default (derived from `Settings.model_fields` so it can't drift); `test_response_to_docs.py` "unconfigured" spawn forces empty docs repo (dotenv leak re-enabled the docs feature there too). + +Results: +- `uv run pytest --cov=app` → 1193 passed; TOTAL 99% (>90%) +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors +- `bash .agent/validate.sh` → validation OK +- `uv run pytest tests/e2e/test_smoke.py -v --no-cov` → 3 passed (validates the modified shared E2E server) +- Task grep criteria verified: `homelab` in `index.html` only at the L176 code comment (A3); no "deployment" text; one user-visible "Ask me anything" + +Decisions: conftest changes are a required defect fix (suite was red before this task on this machine); no commit made (task 04 owns it). + +Next pending: `02_sources_pages_copy.md` (KB page-sub + remaining page-subs + git-sources example URL). diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.validate b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.validate new file mode 100644 index 0000000..d8efed0 --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__01_chat_page_copy.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 6%] +........................................................................ [ 12%] +........................................................................ [ 18%] +........................................................................ [ 24%] +........................................................................ [ 30%] +........................................................................ [ 36%] +........................................................................ [ 42%] +........................................................................ [ 48%] +........................................................................ [ 54%] +........................................................................ [ 60%] +........................................................................ [ 66%] +........................................................................ [ 72%] +........................................................................ [ 78%] +........................................................................ [ 84%] +........................................................................ [ 90%] +........................................................................ [ 96%] +......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 101 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2458 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.err b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.md b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.md new file mode 100644 index 0000000..49e23c4 --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.md @@ -0,0 +1,19 @@ +All gates green. Task 02 complete. + +**Implemented** +- `frontend/sources.html` KB page-sub → locked (A1) copy: "Every file indexed from your configured sources — git repositories, local directories, and uploaded archives. Press **Sync sources** to pull the latest and re-import." +- `frontend/git-sources.html` example placeholder → `https://github.com/you/your-repo.git` (A3) +- `tests/unit/test_sync_button.py` — updated the stale copy-pin (it asserted the retired "clone the repos and re-import") to pin the new locked copy incl. the source-model wording +- Verification read-through, all **no change** (accurate today): `sources.js` sync result/error strings (server-driven, incl. "no sources configured (git or local)" from `app/api/sync.py:190`); page-subs in `tuning.html`, `history.html`, `doc-edit.html`, `git-sources.html`; `#sources-gate` copy (verified `GET /api/docs` admin-only, `GET /api/documents/content` public) + +**Results** +- `uv run pytest --cov=app --cov-report=term` → 1193 passed, TOTAL **99%** (>90%) +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors +- `uv run pytest tests/e2e/test_smoke.py -v --no-cov` → 3 passed; `tests/e2e/test_header_consistency.py -v --no-cov` → 3 passed +- `grep "Homelab\|Deployments" frontend/sources.html` and `grep "homelab" frontend/git-sources.html` both empty; only A3-exempt comments remain + +**Decisions** +- One pre-existing test pinned the retired copy (defect in previously completed work) — updated to the locked replacement rather than the HTML reverted +- No commit made: the phase's single atomic commit belongs to task 04 + +**Next pending task:** `.agent/phases/todo/61_stale_ui_copy/03_footer_defaults.md` diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.validate b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.validate new file mode 100644 index 0000000..d8efed0 --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__02_sources_pages_copy.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 6%] +........................................................................ [ 12%] +........................................................................ [ 18%] +........................................................................ [ 24%] +........................................................................ [ 30%] +........................................................................ [ 36%] +........................................................................ [ 42%] +........................................................................ [ 48%] +........................................................................ [ 54%] +........................................................................ [ 60%] +........................................................................ [ 66%] +........................................................................ [ 72%] +........................................................................ [ 78%] +........................................................................ [ 84%] +........................................................................ [ 90%] +........................................................................ [ 96%] +......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 101 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2458 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.err b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.md b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.md new file mode 100644 index 0000000..11d2afb --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.md @@ -0,0 +1,14 @@ +Task 03 is complete and green. + +**Task 03 — Neutral footer default on all 9 pages (`.footer-text` hook)** + +- Replaced `Powered by Reese's self-hosted models` → `Powered by self-hosted models` on all 9 pages (`index`, `sources`, `tuning`, `document`, `git-sources`, `history`, `login`, `shared`, `doc-edit`), exactly at the audited line numbers; `.footer-version` siblings untouched +- Verified `grep -c 'class="footer-text"' frontend/*.html` → 1 per file, 9 total; `grep -rn "Reese's" frontend/` → empty +- No CSS/JS changes — the class is a selector-only hook for phase 62's `BOR_FOOTER_TEXT`; no test pinned the old footer text +- Tests: `uv run pytest --cov=app` → 1193 passed (exit 0), coverage TOTAL **99%** (>90% gate); `uv run ruff check . && uv run pyright` → clean +- Brand layer unaffected: `uv run pytest tests/unit/test_frontend_brand.py tests/e2e/test_configurable_brand.py -v --no-cov` → 15 passed (DB up, so run now rather than deferred to task 04) +- The 9-page unit pin is deliberately deferred to task 04 per the task file; no new tests added in this task (no logic changes) +- Not committed — the single atomic phase commit is task 04's job; working tree carries tasks 01–03 changes +- Note: `.agent/PLAN.md` is absent from the repo; owner-locked copy was taken from the phase's `00_phase.md` locked decisions (A1) + +**Next pending task:** `.agent/phases/todo/61_stale_ui_copy/04_tests_and_commit.md` diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.validate b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.validate new file mode 100644 index 0000000..d8efed0 --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__03_footer_defaults.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 6%] +........................................................................ [ 12%] +........................................................................ [ 18%] +........................................................................ [ 24%] +........................................................................ [ 30%] +........................................................................ [ 36%] +........................................................................ [ 42%] +........................................................................ [ 48%] +........................................................................ [ 54%] +........................................................................ [ 60%] +........................................................................ [ 66%] +........................................................................ [ 72%] +........................................................................ [ 78%] +........................................................................ [ 84%] +........................................................................ [ 90%] +........................................................................ [ 96%] +......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 101 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2458 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.err b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.md b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.md new file mode 100644 index 0000000..34f1c07 --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.md @@ -0,0 +1,17 @@ +Task 04 complete. Report: + +**Implemented** +- `tests/unit/test_stale_ui_copy.py` (8 tests, house text-pin pattern): old homelab-era strings GONE from `index.html`/`sources.html`/`git-sources.html`; locked A1 strings present exactly once; `class="footer-text"` + neutral footer on all 9 pages; A2 chips in `app/config.py` in order; old chips absent from all of `app/` + `frontend/` (retired, not relocated). +- `tests/e2e/test_stale_ui_copy.py` (5 tests, shared conftest server, DB up): chat placeholder/meta/footer pins, full body scan for `homelab`/`deployment`, A2 chip list in order, sources `.page-sub` (no Homelab/Deployments, has "configured sources"), git-sources `#git-source-url` placeholder (A3) — all from the rendered DOM. + +**Verification (all green)** +- `uv run pytest --cov=app` → 1201 passed, coverage TOTAL **99%** (>90% gate) +- `uv run pytest tests/e2e/test_stale_ui_copy.py -v --no-cov` → 5 passed (isolation, DB up) +- Regression E2E in isolation: `test_smoke.py` 3 ✓, `test_configurable_brand.py` 5 ✓, `test_header_consistency.py` 3 ✓, `test_dark_tech_theme.py` 6 ✓ +- `uv run ruff check . && uv run pyright` → clean + +**Decisions/deviations** +- Old git example URL also pinned (A3, "where they lived"); old-chip absence scan scoped to `app/`+`frontend/` (a repo-wide scan would hit the pin file itself). +- Atomic commit `baefcde` on `main` — staged `frontend/ tests/ app/` only, `--no-gpg-sign`, body carries the task-02 read-through outcome (all 6 checked locations: no change needed). + +**Next pending task:** `62_ui_customization/01_config_settings.md` diff --git a/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.validate b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.validate new file mode 100644 index 0000000..c85b8df --- /dev/null +++ b/.agent/reports/61_stale_ui_copy/61_stale_ui_copy__04_tests_and_commit.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 101 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2458 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.err b/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.md b/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.md new file mode 100644 index 0000000..ad4e024 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.md @@ -0,0 +1,14 @@ +All completion criteria verified. Report: + +**Phase 62 — UI Customization: final verification pass (all tasks already complete, commit `c738105` present)** + +- Verified implementation: 3 new `Settings` fields + fail-loud `theme` validator, six-key `/api/config`, `brand.js` placeholder/footer/theme-link passes, `themes/indigo.css` + README, Containerfile L29 `cp -r ./assets/themes`, `.env.example` + README docs. +- Live spot-checks: default `/api/config` = exactly the six keys with phase-61 defaults; `/assets/themes/indigo.css` served 200 from dev static dir; `../evil.css`, `a/b.css`, `INDIGO.CSS`, `indigo` all refused at startup, each naming the value. +- `uv run pytest --cov=app` — all green; **coverage TOTAL 99%** (>90% gate). +- `uv run ruff check . && uv run pyright` — clean (0 errors). +- `uv run pytest tests/e2e/test_ui_customization.py -v --no-cov` — **5 passed** in isolation (DB up). +- Regression E2E in isolation: `test_configurable_brand` 5✓, `test_stale_ui_copy` 5✓, `test_smoke` 3✓, `test_dark_tech_theme` 6✓. +- Commit criterion: single `--no-gpg-sign` commit `c738105 feat(web): customizable placeholder, footer text, and color theme via BOR_* env vars` (stages only `app/ frontend/ tests/ .env.example README.md Containerfile`); `.agent/` phase files already filed under `complete/` by the harness. +- All 9 completion criteria: **met**. No defects found; no fixes needed. +- Note: `.agent/PLAN.md` referenced by AGENTS.md does not exist in the repo — verified against the phase overview's locked decisions (A4–A7), all honored. +- Next pending phase: **`63_unambiguous_document_listing`**. diff --git a/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.validate b/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.validate new file mode 100644 index 0000000..4121954 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__00_phase.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +.................................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.err b/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.md b/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.md new file mode 100644 index 0000000..1ed8710 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.md @@ -0,0 +1,19 @@ +All green. Task complete — final verification summary: + +**Implemented (task 01 — Settings + `/api/config` keys + validator):** +- `app/config.py`: `input_placeholder` / `footer_text` / `theme` (phase-61 neutral defaults) + `theme` field_validator — empty ok, bare `^[a-z0-9_-]+\.css$` only, malformed fails startup naming the value (phase-56 style) +- `app/api/config.py`: `GET /api/config` now returns exactly the six locked keys (values verbatim, same public posture) +- Contract tests moved to six keys: `tests/integration/test_api.py` (both key-set tests + new `test_config_serves_ui_customization_overrides` + default-value pins), `tests/e2e/test_configurable_brand.py` (both instances) +- Unit pins: `tests/unit/test_config.py` — validator accept/reject cases (each rejection names the value), env-override + `BOR_THEME=../evil.css` startup failure + +**Test / lint / coverage results:** +- `uv run pytest` → **1211 passed** +- `uv run pytest --cov=app --cov-report=term` → **TOTAL 99%** (>90% gate) +- `uv run ruff check .` → clean · `uv run pyright` → 0 errors +- `bash .agent/validate.sh` → "validation OK" · e2e file collects (5 tests; run lands in task 05) + +**Notable decisions:** +- Also updated `tests/unit/test_save_as_doc_button.py` L54 key-set pin (not in the task's inventory — suite would go red without it) and added the phase-61-style `.env`-leak guard for the three new vars in `tests/conftest.py` +- No commit made — the phase's single atomic commit belongs to task 05 + +**Next pending task:** `02_brand_layer_customization.md` (brand.js applies placeholder/footer/theme from the same boot fetch) diff --git a/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.validate b/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.validate new file mode 100644 index 0000000..8de124a --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__01_config_settings.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +........................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.err b/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.md b/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.md new file mode 100644 index 0000000..d78dcfe --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.md @@ -0,0 +1,23 @@ +Task 02 complete. All gates green. + +**Implemented** +- `frontend/assets/brand.js`: the three phase-62 keys apply from the SAME settled-config `.then` (no second fetch), placed after the app_name block and independent of it (the name passes are now wrapped in `if (name)` — behavior byte-identical to the old early-return): + - `input_placeholder` → `#message-input` placeholder (null-guard no-op off the chat page) + - `footer_text` → every `.footer-text` node via `textContent` (no markup injection) + - `theme` → `` inserted immediately after the `styles.css` link (cascade wins), idempotent via `#theme-override`, A5 degradation: `onerror` → `console.warn`, built-in theme stands +- Header contract comment: new "Phase 62 (owner-locked 2026-09-01, TODO L3)" items 5–7 + updated no-op property (all 4 env vars unset ⇒ byte-identical) +- `tests/unit/test_frontend_brand.py`: new pin `test_brand_js_applies_phase_62_customization_from_the_same_fetch` — single-fetch marker, all three application markers, empty-skip guards, phase-62-after-app_name ordering, app_name default literal still holds +- Existing app_name passes 1–4 and `BOR_DOCS_REPO_CONFIGURED` logic untouched (re-indent only) + +**Results** +- `uv run pytest` → 1212 passed +- `uv run pytest --cov=app --cov-report=term-missing` → TOTAL 99% (>90%) +- `uv run ruff check . && uv run pyright` → clean (0 errors) +- `bash .agent/validate.sh` → validation OK +- `node --check frontend/assets/brand.js` → syntax OK + +**Decisions** +- Keyed the empty-skip guards on `typeof … === "string"` + truthiness, so a fetch failure (`cfg` null) is also a clean no-op (house style: page never breaks). +- No defects found in prior work — baseline was green before my changes. + +**Next pending task:** `.agent/phases/todo/62_ui_customization/03_theme_assets.md` diff --git a/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.validate b/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.validate new file mode 100644 index 0000000..7cbbc3b --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__02_brand_layer_customization.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +............................................................ [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.err b/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.md b/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.md new file mode 100644 index 0000000..6838f09 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.md @@ -0,0 +1,21 @@ +All done and green. Final report: + +**Implemented (task 03 — theme assets)** +- `frontend/assets/themes/indigo.css` — WCAG-AA dark indigo/slate theme, single `:root` block, exactly the 8 identity overrides (semantic accent/ok/err untouched) +- `frontend/assets/themes/README.md` — authoring guide (load path, variable table w/ built-in refs, filename regex, 4.5:1 bar, never-white-on-brand trap, A7 deployment) +- `Containerfile` — `&& cp -r ./assets/themes /out/assets/themes \` added in stage 1, after the styles.css minify (line 29) +- `tests/unit/test_themes.py` — 6 new pins: files exist, single leading `:root` block + nothing else, exactly the 8 locked overrides, 5 identity-pair ratios ≥4.5:1 computed from the file's own hex values, Containerfile cp line (position + no esbuild), guide contract markers + +**Test / lint / coverage** +- `uv run pytest --cov=app --cov-report=term -q` → rc=0, 1218 tests, TOTAL **99%** (>90%) +- `uv run ruff check .` → All checks passed · `uv run pyright` → 0 errors +- `uv run pytest tests/unit/test_themes.py tests/integration/test_containerfile_assets.py -v --no-cov` → 12 passed +- Ratios (WCAG calc): ink/bg 15.84 · ink/surface 14.71 · ink-soft/surface 8.27 · bg-on-brand 6.46 · brand-ink/surface 11.99 — all ≥4.5:1 +- Dev sanity: `BOR_THEME=indigo.css` → `GET /assets/themes/indigo.css` 200, `/api/config` theme="indigo.css", served HTML has 0 theme refs; default env → theme="", built-in `--brand: #f43f5e` stands (opt-in via env var, no dir scanning) + +**Decisions / notes** +- Task's comment ratios were slightly off (14.6/8.2/6.4/11.9); wrote the computed values (14.7/8.3/6.5/12.0) — no palette values changed +- Required grep spot-check: `f43f5e|#f0e6e6|#0f0a0a` matches only the `:root` block (L9/11/14) + one comment (L1271) as expected; broader scan found decorative `rgb()` literals (bg grid, glow spots, `.send-btn` steps) — noted, not refactored per task +- Commit deferred to task 05's atomic phase commit (per phase plan) + +**Next pending task:** `.agent/phases/todo/62_ui_customization/04_docs.md` diff --git a/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.validate b/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.validate new file mode 100644 index 0000000..4121954 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__03_theme_assets.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +.................................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.err b/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.md b/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.md new file mode 100644 index 0000000..a0ba117 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.md @@ -0,0 +1,12 @@ +All gates green. Task complete. + +**Report** + +- `.env.example` — added the three commented vars (`BOR_INPUT_PLACEHOLDER`, `BOR_FOOTER_TEXT`, `BOR_THEME`) in the `# --- App ---` block directly after the `BOR_APP_NAME` line, task's exact text/defaults, phase-cited. +- `README.md` env table — three rows added immediately after the `BOR_APP_NAME` row (what it affects, default, unset ⇒ no-op contract; `BOR_THEME` row documents the startup validator regex + missing-file degradation). +- `README.md` — new `### Customizing the look` subsection (4 sentences) placed at the end of "Configuration reference", directly **below** the env table, before "## Troubleshooting" — mirrors the house pattern of `###` feature subsections under a parent `##`; links to `frontend/assets/themes/README.md` (verified exists, link resolves) and names `indigo.css` as the example. +- No code changes (task is docs-only, per spec). +- Tests/lint: `uv run pytest` → 1218 passed; `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90% gate); `uv run ruff check .` → all checks passed; `uv run pyright` → 0 errors. +- No commit made — the phase's single atomic commit is defined in task 05 (harness/handoff per phase overview). + +Next pending task: `.agent/phases/todo/62_ui_customization/05_e2e_customization.md` (E2E suite + regression suites + atomic commit). diff --git a/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.validate b/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.validate new file mode 100644 index 0000000..4121954 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__04_docs.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +.................................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.err b/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.md b/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.md new file mode 100644 index 0000000..281c259 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.md @@ -0,0 +1,16 @@ +All gates green and the atomic commit landed. Final report: + +## Phase 62 / Task 05 — complete + +**Implemented** +- `tests/e2e/test_ui_customization.py` — 5 mapped tests: `/api/config` six-key overrides + dev static serving of the theme; chat page custom placeholder/footer/`link#theme-override` + computed `--brand` `#818cf8`; multi-page footer (login, no composer); default server byte-identical (settled-fetch wait before the absence assert); malformed `BOR_THEME="../evil.css"` refuses boot via real uvicorn subprocess, stderr names the value. Own instance on `APP_PORT+2`, brand suite's env-block pattern. +- **Defect fix (task 02 work):** the phase-33/54 cache-bust rewrite serves `styles.css?v=`, so brand.js's `el.href.endsWith("styles.css")` never matched and the theme link was never inserted — now matches the raw attribute path with query/fragment stripped; unit pin added for the exact regression. +- E2E conftest hardening: shared `app_server` pins the three new `BOR_` vars to code defaults (same leak-class fix as phase 61). + +**Results** +- `uv run pytest tests/e2e/test_ui_customization.py -v --no-cov` → 5 passed (isolation, DB up) +- Regressions in isolation: `test_configurable_brand` 5✓, `test_stale_ui_copy` 5✓, `test_smoke` 3✓, `test_dark_tech_theme` 6✓ +- `.agent/validate.sh` → unit+integration green, coverage TOTAL **99%** (>90%), ruff clean, pyright 0 errors +- Commit `c738105` on `main`, staged exactly `app/ frontend/ tests/ .env.example README.md Containerfile`, `--no-gpg-sign` + +**Decisions:** `.agent/` left unstaged per task file (harness moves the phase dir). Next pending: `63_unambiguous_document_listing/01` (or the harness's queue order). diff --git a/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.validate b/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.validate new file mode 100644 index 0000000..4121954 --- /dev/null +++ b/.agent/reports/62_ui_customization/62_ui_customization__05_e2e_customization.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +.................................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.err b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.md b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.md new file mode 100644 index 0000000..48ac1ca --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.md @@ -0,0 +1,17 @@ +All verification complete. The one remaining `source/path —` match is an explanatory comment in a test describing the historical bug (introduced by this phase's own commit) — not stale. + +**Phase 63 — Final verification pass: ALL GREEN** (all 4 tasks already in `complete/`; no code changes needed) + +- Verified `app/rag/agent.py`: `_execute_tool` emits `source: X | path: Y | title: Z` lines; `AGENT_TOOLS` `read_document` param descriptions reference the `source:`/`path:` labels; "No document at …" refusal unchanged +- Verified `tests/e2e/mock_llm.py::_catalog_docs` parses the new format via labeled-field regex; README + user story updated to new format; no stale old-format references remain +- `uv run pytest --cov=app --cov-report=term-missing` → **1219 passed**, TOTAL coverage **99%** (>90% gate) +- `uv run ruff check . && uv run pyright` → **All checks passed / 0 errors** +- `uv run pytest tests/e2e/test_agent_document_tools.py tests/e2e/test_agent_unlimited_tools.py -v --no-cov` → **8 passed** (DB up) +- Regression in isolation: `test_whole_document_context.py` **3 passed**; `tests/integration/test_chat_api.py` **17 passed**; `tests/unit/test_sse_events.py` **12 passed**; `tests/integration/test_agent_tools.py` **4 passed** +- Commit `15a16a8 fix(agent): unambiguous document listing format for LLM parsing` already present (`--no-gpg-sign`); harness has moved task files to `complete/` + +Completion criteria: (1) new format + unit test — ✓ (2) tool descriptions — ✓ (3) mock parsing — ✓ (4) pytest/coverage/lint — ✓ (5) E2E document tools — ✓ (6) regression suites — ✓ (7) commit + phase move — ✓ (commit exists; `00_phase.md` move handled by harness) + +Deviation: `test_chat_api.py`/`test_sse_events.py` don't exist under `tests/e2e/`; ran their actual locations (integration/unit) in isolation — all green. + +Next pending phase: none — `todo/` holds only phase 63 itself. diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.validate b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.validate new file mode 100644 index 0000000..22439e4 --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__00_phase.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 64%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.err b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.md b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.md new file mode 100644 index 0000000..de3996b --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.md @@ -0,0 +1,11 @@ +**Task 01 complete — report** + +- `app/rag/agent.py`: `list_documents` output now `source: X | path: Y | title: Z` (labeled, pipe-delimited — owner-locked A1); `read_document` `source`/`path` param descriptions now point the LLM at the `source:`/`path:` labels (A2, verbatim from task file); `list_documents` tool description + module docstring updated to match the new format (old wording would have misdescribed the output). "No document at …" refusal and SSE `argument` unchanged (A3/A4). +- `tests/unit/test_agent.py`: tool-shape test now asserts the new param descriptions; all three listing-format assertions updated. +- `tests/unit/test_mock_tool_flow.py`: `CATALOG_*` wire fixtures updated to the new format (they must stay byte-identical to `agent.py` output). +- Deviation: `tests/e2e/mock_llm.py::_catalog_docs` (nominally task 02) was pulled into this task — the old `rsplit("/", 1)` parse would have mis-read the new lines and broken the phase's mandatory E2E gate; now regex-parses the labeled fields, producing identical `(source, path)` tuples (verified round-trip incl. the `brain-of-reese-main/homelab/aws-route53.md` case from the bug report). +- `uv run pytest --cov=app` → 1218 passed, TOTAL coverage **99%** (>90%) +- `uv run ruff check . && uv run pyright` → clean (0 errors) +- `uv run pytest tests/e2e/test_agent_document_tools.py tests/e2e/test_agent_unlimited_tools.py -v --no-cov` → 8 passed (DB up); regression `test_whole_document_context.py` → 3 passed. (`test_chat_api.py`/`test_sse_events.py` listed in the phase don't exist in this repo.) +- No commit made — the phase (tasks 02–04 remaining) gets one atomic commit at phase completion per AGENTS.md rule 8. +- Next pending task: `63_unambiguous_document_listing/02_mock_and_unit_tests.md` (its mock/unit work is already done here — expect verification only). diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.validate b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.validate new file mode 100644 index 0000000..4121954 --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__01_agent_list_format.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +.................................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.err b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.md b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.md new file mode 100644 index 0000000..b87851f --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.md @@ -0,0 +1,19 @@ +All gates green. Task 02 complete. + +**Task 02 report — mock + unit tests for the new catalog format** + +- **State found:** task 01's session had already applied the task-02 code (noted as a deviation in its report); this session verified it end-to-end and closed the one test gap. +- `tests/e2e/mock_llm.py::_catalog_docs` — verified: regex-parses the labeled `source: X | path: Y | title: Z` lines; produces identical `(source, path)` tuples for all test catalogs. +- `tests/unit/test_agent.py` — verified: `test_agent_tools_names_and_parameters` asserts the new A2 `source:`/`path:` parameter descriptions; all three listing-format assertions use the new format; "No document at S/ghost.md" refusal unchanged (A4). +- **Added** `test_read_step_nested_path_stays_intact` in `tests/unit/test_mock_tool_flow.py` — pins the actual bug: the `brain-of-reese-main` / `homelab/aws-route53.md` nested path (multiple `/`) parses intact; the old `rpartition` parse misread it. + +**Gates (exact commands → outcomes)** +- `uv run pytest tests/unit/test_agent.py -v --no-cov` → 22 passed +- `uv run pytest tests/e2e/test_agent_document_tools.py -v --no-cov` → 4 passed (DB up) +- `uv run pytest tests/e2e/test_agent_document_tools.py tests/e2e/test_agent_unlimited_tools.py -v --no-cov` → 8 passed in isolation +- `uv run pytest --cov=app --cov-report=term-missing` → **1219 passed**, coverage TOTAL **99%** (>90%) +- `uv run ruff check tests/ && uv run pyright tests/` → clean; `uv run ruff check . && uv run pyright` → clean (0 errors) + +**Decisions/deviations:** none — mock and unit work was already in the working tree from task 01's session (its report documents the pull-in); no commit made (one atomic commit lands at phase completion per AGENTS.md rule 8). + +**Next pending task:** `63_unambiguous_document_listing/03_e2e_document_tools.md` diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.validate b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.validate new file mode 100644 index 0000000..22439e4 --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__02_mock_and_unit_tests.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 64%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.err b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.md b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.md new file mode 100644 index 0000000..a23ce6e --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.md @@ -0,0 +1,16 @@ +All gates green. Task 03 is verification-only (no code changes needed, consistent with its "Files changed: None"). Following the phase convention (one atomic commit at phase completion, per AGENTS.md rule 8), no commit is made here. + +**Task 03 complete — E2E document tools validation** + +- **Verified (no code changes):** SSE `tool` frames carry `argument` = `source/path`; UI renders "Listing documents"/"Reading …"; source chips `(source, path)` pairs; mock's deterministic answer content; query log `sources` = `"source/path, source/path"` — all asserted by the passing E2E suites against the new `source: X | path: Y | title: Z` catalog format. +- **Gates (DB up, all in isolation):** + - `uv run pytest tests/e2e/test_agent_document_tools.py -v --no-cov` → 4 passed + - `uv run pytest tests/e2e/test_agent_unlimited_tools.py -v --no-cov` → 4 passed + - `uv run pytest tests/e2e/test_whole_document_context.py -v --no-cov` → 3 passed + - `uv run pytest tests/e2e/test_chat_rag.py -v --no-cov` → 3 passed + - `uv run pytest --cov=app --cov-report=term-missing` → **1219 passed**, coverage TOTAL **99%** (>90%) + - `uv run ruff check . && uv run pyright` → clean (0 errors) +- **Deviations:** `test_chat_api.py` / `test_sse_events.py` named in the task never existed in this repo (verified via `git log --all`); ran `test_chat_rag.py` as the chat-API/SSE regression equivalent — same finding as task 01's report. +- **No defects found** in prior work; no commit (phase-level commit at completion). + +**Next pending task:** `63_unambiguous_document_listing/04_docs.md` diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.validate b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.validate new file mode 100644 index 0000000..22439e4 --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__03_e2e_document_tools.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 64%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.err b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.md b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.md new file mode 100644 index 0000000..96eae39 --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.md @@ -0,0 +1,18 @@ +All green. Phase 63 is now fully complete. + +**Task 04 — documentation update** + +- `README.md` — "Agent document tools" section now documents the `list_documents` output as `source: X | path: Y | title: Z` lines +- `.agent/user_stories/agent-document-tools.md` — acceptance criterion 3 updated to the labeled format (reflowed, meaning unchanged) +- Verified: `grep "source/path — title\|source/path - title"` over `README.md` + `.agent/user_stories/*.md` → no matches; `uv run ruff check README.md .agent/` → no error. Other `source/path` hits (SSE `argument`, source chips, `app/config.py` KB-overview note) are locked unchanged per A3 / are a different format +- Committed phase work (tasks 01–04) as `15a16a8 fix(agent): unambiguous document listing format for LLM parsing` (`--no-gpg-sign`, staged `app/ tests/ README.md`; `.agent/` tree left for the harness, matching phases 61/62) + +**Test / lint / coverage results** +- `uv run pytest --cov=app --cov-report=term-missing` → exit 0, 1231 results, 0 failures; coverage TOTAL **99%** (>90% gate) +- `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors, 0 warnings +- E2E isolated (DB up): `test_agent_document_tools.py` 4 passed · `test_agent_unlimited_tools.py` 4 passed · `test_whole_document_context.py` 3 passed +- Regression isolated: `tests/integration/test_chat_api.py` 17 passed · `tests/unit/test_sse_events.py` 12 passed (the phase's `tests/e2e/` paths for these two don't exist — actual locations used) + +**Deviations:** none — phase commit command's `app/ tests/` extended with `README.md` (task 04's deliverable; phase-62 precedent). + +**Next pending task:** none — task 04 was the last task of phase 63; `todo/` is now empty aside from this phase's harness-managed files. diff --git a/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.validate b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.validate new file mode 100644 index 0000000..22439e4 --- /dev/null +++ b/.agent/reports/63_unambiguous_document_listing/63_unambiguous_document_listing__04_docs.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 64%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 173 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2469 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.err b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.md b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.md new file mode 100644 index 0000000..dac06b5 --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.md @@ -0,0 +1,15 @@ +All green. Task complete. + +**Report — phase 64, task 01 (importer progress hook)** + +- `app/rag/importer.py`: `import_sources` gains optional kwarg `progress: Callable[[str, str, int, int], None] | None`; called once per importable file immediately before `_index_file` with `(source, rel_posix_path, done, total)` — `done` = 1-based index across all sources (`summary.files`), `total` = combined count from a pre-walk via the existing `iter_importable_files` (stats only, no file reads), done **only when `progress` is provided**; `limit` path unchanged (hook fires per processed file, `total` stays full); module docstring notes the hook (phase 64). +- `tests/unit/test_importer.py`: 4 new tests — exact multi-root sequence (`total` = combined), no-pre-walk sentinel (monkeypatched counting wrapper: 1 walk without hook vs 2 with), unchanged + `EmbeddingError` files still reported as current, and `limit=` (2 events, `total` = 3). + +**Results** +- `uv run pytest --cov=app --cov-report=term`: **1223 passed**, `app/` coverage **99%** (>90% gate) +- `uv run ruff check . && uv run pyright`: clean (0 errors) +- Existing callers (`scripts/import_docs.py`, `app/api/sync.py`, `app/api/git_sources.py`) untouched and byte-identical in behavior — all pre-existing tests green. + +**Notes:** `.agent/PLAN.md` no longer exists in the repo (removed in commit `281f355`); the phase files served as binding context — no deviations. No commit made (phase commit belongs to task 06). + +**Next pending task:** `64_sync_upload_progress/02_sync_status_current_file.md` diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.validate b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.validate new file mode 100644 index 0000000..81c3713 --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__01_importer_progress_hook.a1.validate @@ -0,0 +1,71 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 52%] +........................................................................ [ 58%] +........................................................................ [ 64%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 94%] +....................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 89 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2476 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.err b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.md b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.md new file mode 100644 index 0000000..c87cc96 --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.md @@ -0,0 +1,17 @@ +Everything is green and the diff is clean. Task complete. + +## Report + +**Implemented (task 02 — sync status reports the current file):** +- `app/api/sync.py`: `SyncStatus` gains `current_file`/`files_done`/`files_total`; `GET /api/sync/status` returns them (null/0/0 idle, all pre-existing keys unchanged); `_run_sync` resets them at start, passes the task-01 hook to `import_sources(..., prune=True, progress=_hook)` assigning `f"{source}/{rel}"` + counts, and clears `current_file` in both terminal states while keeping final counts; module + endpoint + dataclass docstrings updated (clone/pull phase reports no file per A4). +- `tests/unit/test_sync_button.py`: 3 new unit tests — full idle dict pinned (new keys null/0/0), mid-run status via the runner's real hook closure (worker-thread + gated clone/import, also pins A4 no-file-during-clone and success-terminal behavior), failed-terminal clears `current_file` keeping counts with sanitized error. +- `tests/integration/test_sync_api.py` (defect fix for prior work): `FakeImportSources`/`failing_import` now accept + record the `progress` kwarg, idle-shape pin extended with the three keys, success test pins the hook wiring. + +**Results:** +- `uv run pytest` → 1226 passed +- `uv run pytest --cov=app --cov-report=term-missing` → 1226 passed, TOTAL 99% (`app/api/sync.py` 100%) +- `uv run ruff check . && uv run pyright` → clean (0 errors) + +**Decisions:** unit tests drive `_run_sync` directly (no DB/HTTP; seams monkeypatched on `app.api.sync` — the house mock-import pattern), since the task named `test_sync_button.py` though the API coverage lives in the integration suite; in-memory `GitSource` needs explicit `kind="git"` (Python default applies at INSERT flush). No UI changes — "Syncing…" stays until task 04. No commit (phase commit belongs to task 06). + +**Next pending task:** `.agent/phases/todo/64_sync_upload_progress/03_upload_background_202.md` diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.validate b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.validate new file mode 100644 index 0000000..2e7e264 --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__02_sync_status_current_file.a1.validate @@ -0,0 +1,72 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 52%] +........................................................................ [ 58%] +........................................................................ [ 64%] +........................................................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 93%] +........................................................................ [ 99%] +.. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 143 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 198 0 100% +----------------------------------------------- +TOTAL 2488 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.err b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.md b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.md new file mode 100644 index 0000000..581a444 --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.md @@ -0,0 +1,20 @@ +All green. Final report: + +## Task 03 complete — background upload (202 + status endpoint) + +**Implemented** +- `app/schemas.py`: new `UploadAccepted` (`detail` + safe `name`); `UploadOut` kept — now the shape of the status `detail` on success. +- `app/api/git_sources.py`: `POST /api/git-sources/upload` → **202** the moment the archive is on disk (`db` dependency dropped; inline gates unchanged: name 422, flag 409 set *before* the streamed receive, cap 413 with temp deleted + flag released on the pre-task exception path); unpack → swap → row upsert (short-lived `SessionLocal`, `IntegrityError` → failed naming the path) → `check_models` → `import_sources(..., progress=_hook)` → gated overview → moved INFO log line → `success` with `UploadOut` fields, all in `_run_upload` (phase-32 `_run_sync` shape; failures land sanitized in `failed`; `finally` cleans temps + clears the flag; `CancelledError` not caught). New `UploadStatus` dataclass + `GET /api/git-sources/upload/status` (identical key set to `/api/sync/status`, admin-only). +- `tests/integration/test_git_sources_upload.py`: all 29 scenarios kept, observation point moved to the status endpoint (polled to terminal) — every background branch has a dedicated test (unpack/zero-entry/swap/IntegrityError/model-down/import-fail/success/cancel-cleanup), mid-run `current_file`/counts via the task-02 hook seam, flag-not-state 409 pins, temp-on-disk + 202-before-scan pins, TRUNCATE regression. + +**Results** +- `uv run pytest --cov=app --cov-report=term` → **1233 passed**, app/ **99%** (`git_sources.py` 100%) +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors, 0 warnings +- `bash .agent/validate.sh` → validation OK; upload suite alone: 29 passed in ~1 s + +**Notable decisions** +- The task's "adapt `tests/unit/test_archive_upload.py`" bullet misreferences the file: that suite covers the unchanged `archive_upload` *utility*; the 202-contract HTTP scenarios live in `tests/integration/test_git_sources_upload.py` (adapted there; unit file untouched, still green). +- Discovered: starlette's TestClient delivers the response only after the app's task-completion callback runs on the loop — a sync park inside `unpack_archive` stalls even the 202. So the temp-on-disk pin drives `_run_upload` directly (worker loop), and the HTTP "202 while in flight" pin uses the async import gate. +- `test_archive_upload_sources.py` E2E expected to fail until task 06 (noted, not fixed here). + +**Next pending task:** `04_sync_button_live_file.md` diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.validate b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.validate new file mode 100644 index 0000000..d46dfaa --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__03_upload_background_202.a1.validate @@ -0,0 +1,72 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 40%] +........................................................................ [ 46%] +........................................................................ [ 52%] +........................................................................ [ 58%] +........................................................................ [ 64%] +........................................................................ [ 70%] +........................................................................ [ 75%] +........................................................................ [ 81%] +........................................................................ [ 87%] +........................................................................ [ 93%] +........................................................................ [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 181 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 201 0 100% +----------------------------------------------- +TOTAL 2529 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.err b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.md b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.md new file mode 100644 index 0000000..d48083a --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.md @@ -0,0 +1,14 @@ +Task 04 complete. All changes are confined to `frontend/` + the two test files (the `app/`/integration diffs were pre-existing uncommitted work from tasks 01–03, untouched by me). + +**Implemented** +- `frontend/assets/sources.js`: `fmtSyncLabel(kind, currentFile, done, total)` (sync → "Syncing…", upload → "Importing", file appended only when present, counts only when `total > 0`); `enterSyncRunningState(kind, file, done, total)` writes the full untruncated path to button `title` (removed when null) + `#sync-result` aria-live; unified tick fetches both `/api/sync/status` and `/api/git-sources/upload/status` (upload 403 = "no upload", blips retry) with the 7-branch decision tree (sync running → upload running → sync success → sync failed → upload success → upload failed → idle); upload success settles + `loadDocs()` with no counts in `#sync-result` (A3); 202/409 click branch → `enterSyncRunningState("sync", null, 0, 0)`; `initSyncButton` re-attaches a running upload when sync is idle; header comment documents the contract +- `frontend/assets/styles.css`: `.sync-label` ellipsis (inline-block, `min(16rem, 40vw)`, ellipsis, nowrap) +- `frontend/sources.html`: comment block updated (no structural change) +- New `tests/unit/test_frontend_sync_upload.py` (9 pins); adapted 2 now-stale pins in `tests/unit/test_sync_button.py` + +**Results** +- `uv run pytest --cov=app --cov-report=term` → 1242 passed, `app/` 99% (>90% gate) +- `uv run ruff check . && uv run pyright` → clean (0 errors) +- E2E regressions in isolation: `test_sync_button.py` 3✓, `test_sync_model_down.py` 4✓, `test_git_sources_admin.py` 6✓, `test_local_directory_sources.py` 3✓; `node --check` + runtime `fmtSyncLabel` checks pass + +**Notes**: no deviations from the task spec; no commit (task 06 owns it). Next pending task: `.agent/phases/todo/64_sync_upload_progress/05_upload_toast_progress.md`. diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.validate b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.validate new file mode 100644 index 0000000..440b45d --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__04_sync_button_live_file.a1.validate @@ -0,0 +1,72 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 28%] +........................................................................ [ 34%] +........................................................................ [ 40%] +........................................................................ [ 46%] +........................................................................ [ 52%] +........................................................................ [ 57%] +........................................................................ [ 63%] +........................................................................ [ 69%] +........................................................................ [ 75%] +........................................................................ [ 81%] +........................................................................ [ 86%] +........................................................................ [ 92%] +........................................................................ [ 98%] +.................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 181 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 201 0 100% +----------------------------------------------- +TOTAL 2529 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.err b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.md b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.md new file mode 100644 index 0000000..bcc3841 --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.md @@ -0,0 +1,19 @@ +Everything is verified. All deletions in `git-sources.js` are confined to the phase-49 upload doc/paragraph (marked superseded) and the synchronous submit handler; load/add/remove behavior is untouched. + +## Report — Task 05 complete + +**Implemented** (`frontend/assets/git-sources.js` + `frontend/git-sources.html`): +- `showUploadToast` — page-local phase-55 toast: single lazy node, `role="status" aria-live="polite"`, ~5 s auto-dismiss, new toast replaces pending one; `.toast` CSS reused as-is +- Submit rework: 202 → toast `Successfully uploaded — ` (UploadAccepted body, file-name fallback) → clear input → processing state → poll; 409 → re-attach, NO error banner; other non-2xx/network keep phase-49 banners + kept selection; `finally` restores only when no poll is active (§7.4) +- `startUploadPolling` — 2 s single-timer poll of `GET /api/git-sources/upload/status`: running → live `"Processing… (n/m)"` + full-path `title`; success → result line + announce + `loadSources()` (no second toast); failed → sanitized error banner + `loadSources()`; idle → defensive restore +- `initUploadStatus` boot re-attach (admin branch): running → processing + poll; success → result line only; failed → banner; idle → nothing; header/form comments rewritten (phase-49 paragraph superseded) +- `tests/unit/test_frontend_sync_upload.py` extended with 10 source pins (all task-04 pins intact) + +**Results:** +- `uv run pytest --cov=app --cov-report=term-missing` → 1251 passed, coverage **99%** (>90 ✓) +- `uv run ruff check . && uv run pyright` → clean (0 errors) +- Ad-hoc Playwright sanity (temp files, deleted): toast-on-202 + auto-dismiss + single node, Processing→settle, reload re-render, 409 re-attach without banner, mid-scan reload (160 files) re-attaches to `Processing…` and settles — zero uncaught JS errors + +**Notes:** `tests/e2e/test_archive_upload_sources.py` still expects the old 200 flow — expected-fail until task 06 (noted in task 03); no commit (phase commit lands with task 06). + +**Next pending task:** `06_e2e_sync_upload_progress.md` diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.validate b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.validate new file mode 100644 index 0000000..6239700 --- /dev/null +++ b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__05_upload_toast_progress.a1.validate @@ -0,0 +1,72 @@ +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 28%] +........................................................................ [ 34%] +........................................................................ [ 40%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 57%] +........................................................................ [ 63%] +........................................................................ [ 69%] +........................................................................ [ 74%] +........................................................................ [ 80%] +........................................................................ [ 86%] +........................................................................ [ 92%] +........................................................................ [ 97%] +........................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 131 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 181 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 112 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 78 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 189 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 201 0 100% +----------------------------------------------- +TOTAL 2529 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__06_e2e_sync_upload_progress.a1.err b/.agent/reports/64_sync_upload_progress/64_sync_upload_progress__06_e2e_sync_upload_progress.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/user_stories/agent-document-tools.md b/.agent/user_stories/agent-document-tools.md index 5f3bdff..95946c1 100644 --- a/.agent/user_stories/agent-document-tools.md +++ b/.agent/user_stories/agent-document-tools.md @@ -35,7 +35,8 @@ addition to "thinking". 3. `app/rag/agent.py`: the loop — budgets from `BOR_AGENT_LIST_CALLS` / `BOR_AGENT_READ_CALLS` (default 1/1); the `list_documents` tool returns the DB catalog - (`source/path — title` lines, `/api/docs` order); `read_document` + (`source: X | path: Y | title: Z` lines, `/api/docs` order); + `read_document` appends the **full** document text (A7-revised: never truncated); after both budgets are spent the tools are dropped and the model must answer; safety cap on model rounds; the deflection (LOW) path is byte-identical. diff --git a/app/api/git_sources.py b/app/api/git_sources.py index 9d26d96..87c4e14 100644 --- a/app/api/git_sources.py +++ b/app/api/git_sources.py @@ -17,13 +17,19 @@ Routes: ``GET`` (DB rows oldest-first, or the env list with ``path``, git rows — and env rows — report ``path: null``), ``POST`` (201, validated create; ``kind`` selects the validation: git → exactly the phase-35 URL contract, local → an existing absolute directory, else -422 naming the path), ``POST /upload`` (phase 49 — admin archive upload: -``.tar``/``.tar.gz``/``.tgz``/``.zip`` streamed with a size cap, safely -unpacked, atomically swapped in over an existing folder of the same -name, row upserted, then the synchronous single-source scan — see -:func:`upload_archive`), ``DELETE /{source_id}`` (204). The whole -router sits behind :func:`app.core.auth.require_admin` — anonymous -callers get 403 on every route. +422 naming the path), ``POST /upload`` (phase 49, backgrounded in phase +64 task 03 — admin archive upload: the ``.tar``/``.tar.gz``/``.tgz``/ +``.zip`` name/format gate + the 1 MiB-chunk receive with the +``upload_max_mb`` cap run **inline** and answered 202 the moment the +archive is safely on disk; unpack → swap → row upsert → model check → +single-source scan → change-gated overview then run in a **background +task** — see :func:`upload_archive` and :func:`_run_upload`), +``GET /upload/status`` (the phase-32 ``SyncStatus``-shaped in-memory +state of that run — incl. the phase-64 ``current_file`` / +``files_done`` / ``files_total`` progress fields; navigating away from +the page mid-scan no longer aborts anything), ``DELETE /{source_id}`` +(204). The whole router sits behind :func:`app.core.auth.require_admin` +— anonymous callers get 403 on every route. No credential-echo path: git URLs may embed ``user:pass@`` (phase 32's masking discipline), so every git 409/422 detail is a fixed generic @@ -33,21 +39,26 @@ owner sees exactly which directory failed. Scope boundary (phase locked decisions): the CRUD routes do NOT clone, import, or prune anything — the existing Sync button performs -that (a removal prunes on the next sync, ``prune=True``). The phase-49 -upload route is the exception: it unpacks the archive and then scans -the single source synchronously in the request (``import_sources`` -with ``prune=True`` + the change-gated overview refresh) and answers -with the sync-style counts. +that (a removal prunes on the next sync, ``prune=True``). The upload +route is the exception (phase 64, task 03): after the 202 receive +answer, its background task unpacks the archive, swaps it in, upserts +the row, probes the models, scans the single source +(``import_sources`` with ``prune=True`` + the change-gated overview +refresh), and lands the sync-style counts (the ``UploadOut`` fields) +in the status ``detail``. """ from __future__ import annotations +import asyncio import logging import re import shutil import time import uuid +from dataclasses import dataclass, field +from datetime import UTC, datetime from pathlib import Path -from typing import Literal, cast +from typing import Any, Literal, cast from fastapi import APIRouter, Depends, File, HTTPException, Response, UploadFile from sqlalchemy import select @@ -57,7 +68,7 @@ from sqlalchemy.orm import Session from app.api.sync import _sanitize_error from app.config import get_settings from app.core.auth import require_admin -from app.db import get_db +from app.db import SessionLocal, get_db from app.models import GitSource from app.rag.archive_upload import ( ARCHIVE_SUFFIXES, @@ -67,9 +78,15 @@ from app.rag.archive_upload import ( unpack_archive, ) from app.rag.importer import import_sources -from app.rag.llm import LLMClient, ModelUnavailableError, check_models +from app.rag.llm import LLMClient, check_models from app.rag.overview import regenerate_overview -from app.schemas import GitSourceIn, GitSourceList, GitSourceOut, GitSourceRow, UploadOut +from app.schemas import ( + GitSourceIn, + GitSourceList, + GitSourceOut, + GitSourceRow, + UploadAccepted, +) logger = logging.getLogger("app.api.git_sources") @@ -79,18 +96,56 @@ router = APIRouter( dependencies=[Depends(require_admin)], # phase 16 pattern: admin-only surface ) -#: One upload at a time (phase 49, task 02 — the phase-32 ``_task`` -#: spirit): the flag is held from the name gate through the scan -#: response. A plain bool, not an ``asyncio.Lock`` — it is checked and -#: set with no await in between (a single app loop can never enter +#: One upload at a time (phase 49, backgrounded in phase 64 task 03): +#: the flag is checked and set with **no await in between, BEFORE the +#: streaming receive** — the handler now awaits (the 1 MiB-chunk stream) +#: long before the background task exists, so a task-done check alone +#: would let a concurrent POST slip through the receive window and start +#: a second run. A plain bool, not an ``asyncio.Lock``: it is checked +#: and set with no await in between (a single app loop can never enter #: twice), and it stays correct across requests that run on separate -#: event loops (the TestClient convention). +#: event loops (the TestClient convention). Held until +#: :func:`_run_upload`'s ``finally`` (end of the background run) — or +#: cleared on the inline exception path where the task was never +#: created. _upload_in_progress = False #: Streaming read size while counting compressed upload bytes (1 MiB #: chunks — the task-02 cap check granularity). _STREAM_CHUNK = 1 << 20 + +@dataclass +class UploadStatus: + """In-memory state of the (at most one) in-flight upload run. + + Mirrors :class:`app.api.sync.SyncStatus` (the phase-32 pattern, + phase 64 task 03): ``state`` is the same four-state machine + (``idle`` / ``running`` / ``success`` / ``failed``); terminal states + carry the run's ``detail`` (success — the ``UploadOut`` fields) or + ``error`` (failure — sanitized) so the UI can render the last result + after a page reload (the re-attach behavior, task 05). + + Phase 64 (task 03) progress fields: ``current_file`` is the + ``source/relative/path`` the scan is processing right now (null + outside the import phase — unpack/swap/row/model-check first — and + in terminal states); ``files_done`` / ``files_total`` carry the + hook's done/total position and survive a terminal state (the run's + last position is useful context next to the error). + """ + + state: Literal["idle", "running", "success", "failed"] = "idle" + started_at: datetime | None = None + finished_at: datetime | None = None + current_file: str | None = None + files_done: int = 0 + files_total: int = 0 + detail: dict[str, Any] = field(default_factory=dict) + error: str | None = None + + +_upload_status = UploadStatus() + #: Accepted git URL shapes — the trimmed URL must *start* with one of them. #: Covers the phase-28 real URLs (HTTPS + ``git@`` SSH); scp-style #: ``host:repo`` is deliberately rejected (422). ASSUMPTION (task 02): the @@ -227,46 +282,55 @@ def _create_local_row(payload: GitSourceIn, db: Session) -> GitSource: ) -@router.post("/upload", response_model=UploadOut) +@router.post("/upload", response_model=UploadAccepted, status_code=202) async def upload_archive( file: UploadFile = File(...), # noqa: B008 - db: Session = Depends(get_db), # noqa: B008 -) -> UploadOut: - """Upload a source archive and scan it (phase 49, task 02). +) -> UploadAccepted: + """Receive a source archive; scan it in the background (phase 49, + backgrounded in phase 64 task 03 — owner-locked A1/A2). - The scan is **synchronous in the request** (phase locked decisions, - owner-confirmed) and mirrors the admin sync pipeline: + The **inline (request) work is exactly three gates** — steps 1–3 — + everything else runs in a background task behind + ``GET /upload/status`` (the phase-32 ``SyncStatus`` pattern), so + navigating away mid-scan no longer aborts anything: 1. name/format gate — only ``.tar``/``.tar.gz``/``.tgz``/``.zip`` (422 naming the accepted set) and a safe source name (``archive_source_name`` — its message is the 422 detail); - 2. one at a time — 409 ``an upload is already in progress``; + 2. one at a time — 409 ``an upload is already in progress`` while + the flag is held (checked and set with no await in between, + BEFORE the receive — see ``_upload_in_progress``); 3. stream the upload in 1 MiB chunks into a dotfile temp with the - ``upload_max_mb`` cap — 413 naming the cap, temp deleted; + ``upload_max_mb`` cap — 413 naming the cap, temp deleted. + + Then the archive is **safely on disk** — 202 + ``UploadAccepted`` + (the "successfully uploaded" moment the UI toasts on, A2) and + :func:`_run_upload` runs the rest on the app's event loop: + 4. unpack to a temp sibling (traversal/symlink/device/corrupt/ - over-cap all 422 with the task-01 user-safe message, temps - deleted); a zero-entry archive is 422 ``the archive contains no - files`` — an archive with only non-A9 files is a VALID - replacement (the scan indexes nothing, prune removes the + over-cap → ``failed`` with the task-01 user-safe message, temps + deleted); a zero-entry archive is ``failed`` ``the archive + contains no files`` — an archive with only non-A9 files is a + VALID replacement (the scan indexes nothing, prune removes the source's docs); 5. atomic swap-in — a same-name re-upload replaces the previous folder in place; a failure leaves the previous folder/row/KB - untouched (422); + untouched; 6. upsert the row by ``path`` (``kind='local'``; an existing row is left as-is — ``added_at`` preserved — and the unique index is - the 409 backstop); - 7. fail-fast ``check_models`` — 503 with the sanitized - model-unavailable message; the folder/row are already committed, - so the next sync/re-upload retries idempotently; - 8. ``import_sources([folder], llm, prune=True)`` + the change-gated - ``regenerate_overview``; - 9. one INFO log line (PLAN §9 / AGENTS.md rule 10); - 10. 200 with the sync-detail count keys (``UploadOut``). + the backstop: a concurrent insert lands ``failed`` with + ``a local source with this path already exists: ``); + 7. fail-fast ``check_models`` — ``ModelUnavailableError`` → + ``failed`` with the sanitized message (the phase-49 503 becomes + a status state, A5); the folder/row are already committed, so + the next sync/re-upload retries idempotently; + 8. ``import_sources([folder], llm, prune=True, progress=)`` + + the change-gated ``regenerate_overview`` — the hook feeds the + status ``current_file`` / ``files_done`` / ``files_total``; + 9. one INFO log line (PLAN §9 / AGENTS.md rule 10 — ``total_ms`` is + the background run's duration); + 10. ``success`` — ``detail`` = the ``UploadOut`` fields. """ - started = time.monotonic() - settings = get_settings() - total = 0 - # 1. Name/format gate — the accepted formats first (the 422 names # them), then the task-01 safe-name derivation. A BARE suffix # ("tar.gz") is an accepted format with no usable stem — it @@ -288,12 +352,15 @@ async def upload_archive( raise HTTPException(status_code=422, detail=str(e)) from None # 2. One at a time — the flag is checked and set with no await - # between, so the single app loop can never enter twice. + # between, BEFORE the streaming receive: the background task + # does not exist yet, so the flag (not a task-done check) is the + # gate (see ``_upload_in_progress``). global _upload_in_progress if _upload_in_progress: raise HTTPException(status_code=409, detail="an upload is already in progress") _upload_in_progress = True + settings = get_settings() upload_root = Path(settings.upload_dir).expanduser() upload_root.mkdir(parents=True, exist_ok=True) max_bytes = settings.upload_max_mb * 1024 * 1024 @@ -302,77 +369,192 @@ async def upload_archive( try: # 3. Stream with the compressed-size cap — dotfile temps are # hidden from the upload dir's listing. - try: - with open(temp_upload, "wb") as out: - while chunk := await file.read(_STREAM_CHUNK): - total += len(chunk) - if total > max_bytes: - raise HTTPException( - status_code=413, - detail=f"the upload exceeds the {settings.upload_max_mb} MiB limit", - ) - out.write(chunk) - except HTTPException: - temp_upload.unlink(missing_ok=True) - raise - # 4. Unpack to a temp sibling; the compressed bytes are no - # longer needed once unpacked (phase locked decision: only - # the unpacked content is kept). - try: - unpack_archive(temp_upload, temp_unpack, max_bytes) - except ArchiveUploadError as e: - temp_upload.unlink(missing_ok=True) - shutil.rmtree(temp_unpack, ignore_errors=True) - raise HTTPException(status_code=422, detail=str(e)) from None + total = 0 + with open(temp_upload, "wb") as out: + while chunk := await file.read(_STREAM_CHUNK): + total += len(chunk) + if total > max_bytes: + raise HTTPException( + status_code=413, + detail=f"the upload exceeds the {settings.upload_max_mb} MiB limit", + ) + out.write(chunk) + # The archive is safely on disk — 202 is the "successfully + # uploaded" moment (A2). Steps 4–10 run in the background: + asyncio.create_task( + _run_upload(name, filename, total, upload_root, temp_upload, temp_unpack) + ) + except BaseException: + # The background task was never created (cap 413, a broken + # pipe, cancellation, or create_task itself): release the flag + # so the next upload is not refused, and make sure no temp + # survives the failed receive. + temp_upload.unlink(missing_ok=True) + _upload_in_progress = False + raise + return UploadAccepted(name=name) + + +@router.get("/upload/status") +def upload_status() -> dict[str, Any]: + """Current upload state (the UI polls this — the phase-32 + ``GET /api/sync/status`` contract, identical key set). + + ``started_at`` / ``finished_at`` are ISO-8601 strings or null. + ``current_file`` (phase 64) is the ``source/relative/path`` the + scan is processing right now — null during the unpack/swap/row/ + model phases and in terminal states; ``files_done`` / ``files_total`` + carry the hook's position (0/0 idle). The router dependency makes + it admin-only like every other route here. + """ + return { + "state": _upload_status.state, + "started_at": ( + _upload_status.started_at.isoformat() if _upload_status.started_at else None + ), + "finished_at": ( + _upload_status.finished_at.isoformat() if _upload_status.finished_at else None + ), + "detail": _upload_status.detail, + "error": _upload_status.error, + "current_file": _upload_status.current_file, + "files_done": _upload_status.files_done, + "files_total": _upload_status.files_total, + } + + +async def _run_upload( + name: str, + filename: str, + total_bytes: int, + upload_root: Path, + temp_upload: Path, + temp_unpack: Path, +) -> None: + """The post-202 upload pipeline, one in-process background task + (the phase-32 ``_run_sync`` shape — A1). + + Every failure mode (unpack, zero entries, swap, row, models, + import, anything else) lands in the ``failed`` state with a + sanitized ``error`` string — a background task must die in state, + never as an unobserved exception (A5: post-202 failures are status + states, never HTTP errors). ``CancelledError`` is deliberately *not* + caught: app shutdown cancels the task, and swallowing that would + mask a real stop. The ``finally`` cleans both temps (defensive — + each step already cleans its own) and clears ``_upload_in_progress``. + """ + global _upload_in_progress + started = time.monotonic() + _upload_status.state = "running" + _upload_status.started_at = datetime.now(UTC) + _upload_status.finished_at = None + _upload_status.current_file = None + _upload_status.files_done = 0 + _upload_status.files_total = 0 + _upload_status.detail = {} + _upload_status.error = None + try: + settings = get_settings() + max_bytes = settings.upload_max_mb * 1024 * 1024 + # Step 4 — unpack to a temp sibling; the compressed bytes are + # no longer needed once unpacked (phase 49 locked decision: + # only the unpacked content is kept). + unpack_archive(temp_upload, temp_unpack, max_bytes) temp_upload.unlink(missing_ok=True) if not any(temp_unpack.iterdir()): # Zero entries = a user error. (Only non-A9 files is NOT an # error — it still has entries and is a valid replacement.) - shutil.rmtree(temp_unpack, ignore_errors=True) - raise HTTPException(status_code=422, detail="the archive contains no files") - # 5. Swap in — a same-name re-upload replaces the previous - # folder atomically; a failure leaves it, the row, and the - # KB untouched. + raise ArchiveUploadError("the archive contains no files") + # Step 5 — swap in — a same-name re-upload replaces the + # previous folder atomically; a failure leaves it, the row, + # and the KB untouched (the ``failed`` state carries the + # user-safe message). final_dir = upload_root / name - try: - swap_in(temp_unpack, final_dir) - except ArchiveUploadError as e: - shutil.rmtree(temp_unpack, ignore_errors=True) - raise HTTPException(status_code=422, detail=str(e)) from None - # 6. Upsert the row by path — no duplicates: an existing row is - # left exactly as it is (``added_at`` preserved); the unique - # index is the 409 backstop for a concurrent insert the - # pre-check missed. + swap_in(temp_unpack, final_dir) + # Step 6 — upsert the row by path in a SHORT-LIVED session + # (open/close around it — the ``effective_sources`` / + # ``bump_sources_version`` pattern in ``app.api.sync``): the + # background task has no request session to leak locks from + # (the old inline ``db.close()`` discipline, now structural). + # No duplicates: an existing row is left exactly as it is + # (``added_at`` preserved); the unique index is the backstop + # for a concurrent insert the pre-check missed. path = str(final_dir) - if db.scalar(select(GitSource).where(GitSource.path == path)) is None: - _commit_new( - GitSource(url=path, kind="local", path=path), - f"a local source with this path already exists: {path}", - db, - ) - # Release the request session NOW — the handler never touches - # ``db`` again (the scan below uses its own sessions). If the - # session stayed open, its uncommitted transaction (the - # ``_commit_new`` refresh SELECT) would hold ``git_sources`` - # locks for the whole scan, and any concurrent TRUNCATE of the - # KB tables (the E2E isolation fixtures) would deadlock against - # the scan's own document locks — a cycle Postgres cannot see. - # ``get_db``'s teardown close() is idempotent. - db.close() - # 7. Fail-fast models (phase 41) — 503 with the sanitized - # message; nothing else is rolled back (the folder/row are - # committed and the next sync/re-upload retries idempotently). - llm = LLMClient() + db = SessionLocal() try: - await check_models(llm) - except ModelUnavailableError as e: - raise HTTPException(status_code=503, detail=_sanitize_error(str(e))) from None - # 8. Scan — single source, prune (dropped files leave the KB), - # then the change-gated overview refresh (phases 31/32). - summary = await import_sources([final_dir], llm, prune=True) + if db.scalar(select(GitSource).where(GitSource.path == path)) is None: + db.add(GitSource(url=path, kind="local", path=path)) + try: + db.commit() + except IntegrityError: + db.rollback() + raise ValueError( + f"a local source with this path already exists: {path}" + ) from None + finally: + db.close() + # Step 7 — fail-fast models (phase 41): ``ModelUnavailableError`` + # lands in the ``failed`` state sanitized (the phase-49 503 + # becomes a status state, A5). Nothing is rolled back — the + # folder/row are committed and the next sync/re-upload retries + # idempotently. + llm = LLMClient() + await check_models(llm) + # Step 8 — scan — single source, prune (dropped files leave + # the KB), with the phase-64 progress hook feeding the status, + # then the change-gated overview refresh (phases 31/32). The + # closure captures the module ``_upload_status`` exactly like + # the state assignments above. + def _hook(source: str, rel: str, done: int, total: int) -> None: + _upload_status.current_file = f"{source}/{rel}" + _upload_status.files_done = done + _upload_status.files_total = total + + summary = await import_sources([final_dir], llm, prune=True, progress=_hook) overview = False if summary.added + summary.updated > 0: overview = await regenerate_overview(llm) + # Step 9 — per-upload log line (PLAN §9 / AGENTS.md rule 10) + # — moved with the scan: ``total_ms`` is the background run's + # duration. + logger.info( + "upload: name=%s file=%s bytes_in=%d files=%d added=%d updated=%d " + "unchanged=%d pruned=%d errors=%d overview=%s total_ms=%d", + name, + filename, + total_bytes, + summary.files, + summary.added, + summary.updated, + summary.unchanged, + summary.pruned, + summary.errors, + overview, + round((time.monotonic() - started) * 1000), + ) + # Step 10 — success: the ``UploadOut`` fields ride in the + # status ``detail`` (the UI renders the same result line from + # the status that the sync button renders from its own). + _upload_status.state = "success" + _upload_status.finished_at = datetime.now(UTC) + _upload_status.current_file = None # phase 64: keep the final counts + _upload_status.detail = { + "source": name, + "files": summary.files, + "added": summary.added, + "updated": summary.updated, + "unchanged": summary.unchanged, + "pruned": summary.pruned, + "errors": summary.errors, + "chunks": summary.chunks, + "overview": overview, + } + except Exception as e: # noqa: BLE001 — a background task dies in state, see above + logger.exception("upload: failed") + _upload_status.state = "failed" + _upload_status.finished_at = datetime.now(UTC) + _upload_status.error = _sanitize_error(str(e)) + _upload_status.current_file = None # phase 64: keep the final counts finally: _upload_in_progress = False # No temp may survive any failure path (defensive — each step @@ -380,35 +562,6 @@ async def upload_archive( temp_upload.unlink(missing_ok=True) shutil.rmtree(temp_unpack, ignore_errors=True) - # 9. Per-upload log line (PLAN §9 / AGENTS.md rule 10). - logger.info( - "upload: name=%s file=%s bytes_in=%d files=%d added=%d updated=%d " - "unchanged=%d pruned=%d errors=%d overview=%s total_ms=%d", - name, - filename, - total, - summary.files, - summary.added, - summary.updated, - summary.unchanged, - summary.pruned, - summary.errors, - overview, - round((time.monotonic() - started) * 1000), - ) - # 10. Respond 200 with the sync-style counts. - return UploadOut( - source=name, - files=summary.files, - added=summary.added, - updated=summary.updated, - unchanged=summary.unchanged, - pruned=summary.pruned, - errors=summary.errors, - chunks=summary.chunks, - overview=overview, - ) - @router.delete("/{source_id}", status_code=204) def delete_git_source( diff --git a/app/api/sync.py b/app/api/sync.py index 700e09c..0a1b904 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -51,7 +51,12 @@ decisions): run aborts in the ``failed`` state before this step. Status is in memory: a restart mid-sync loses the running state -(accepted — the next click re-syncs idempotently). +(accepted — the next click re-syncs idempotently). The status also +carries the phase-64 per-file progress — ``current_file`` (the +``source/relative/path`` the import is processing right now) plus +``files_done`` / ``files_total`` — null/0/0 before the import starts +(clone/pull reports no file yet) and in terminal states, which clear +``current_file`` but keep the run's final counts. """ from __future__ import annotations @@ -107,6 +112,13 @@ class SyncStatus: ``running``, ``success``, ``failed``. Terminal states carry the run's ``detail`` (success) or ``error`` (failure) so the UI can render the last result after a page reload (task 02's re-attach behavior). + + Phase 64 (task 02) progress fields: ``current_file`` is the + ``source/relative/path`` the import is processing right now (null + outside the import phase — clone/pull first, terminal states + after); ``files_done`` / ``files_total`` carry the hook's + done/total position and survive a terminal state (the run's last + position is useful context next to the error). """ state: Literal["idle", "running", "success", "failed"] = "idle" @@ -114,6 +126,11 @@ class SyncStatus: finished_at: datetime | None = None detail: dict[str, Any] = field(default_factory=dict) error: str | None = None + # Phase 64 (task 02): per-file progress — the file the import is + # processing right now and the hook's done/total position. + current_file: str | None = None + files_done: int = 0 + files_total: int = 0 _status = SyncStatus() @@ -125,6 +142,10 @@ def sync_status() -> dict[str, Any]: """Current sync state (the UI polls this every 2 s — task 02). ``started_at`` / ``finished_at`` are ISO-8601 strings or null. + ``current_file`` (phase 64) is the ``source/relative/path`` the + import is processing right now — null during the clone/pull phase + and in terminal states; ``files_done`` / ``files_total`` carry the + hook's position (0/0 idle). """ return { "state": _status.state, @@ -132,6 +153,9 @@ def sync_status() -> dict[str, Any]: "finished_at": _status.finished_at.isoformat() if _status.finished_at else None, "detail": _status.detail, "error": _status.error, + "current_file": _status.current_file, + "files_done": _status.files_done, + "files_total": _status.files_total, } @@ -165,6 +189,11 @@ async def _run_sync() -> None: _status.finished_at = None _status.detail = {} _status.error = None + # Phase 64 (task 02): the progress fields reset with the run — no + # current file until the import starts (the clone/pull phase). + _status.current_file = None + _status.files_done = 0 + _status.files_total = 0 try: settings = get_settings() # Step 1 (phase 41): fail fast — verify both models the sync @@ -208,7 +237,16 @@ async def _run_sync() -> None: if not path.is_dir(): raise GitSyncError(f"local source missing: {path}") sources.append(path) - summary: ImportSummary = await import_sources(sources, llm, prune=True) + # Phase 64 (task 02): the per-file progress hook — the status + # endpoint reports the file being processed right now. The + # closure captures the module ``_status`` exactly like the state + # assignments above. + def _hook(source: str, rel: str, done: int, total: int) -> None: + _status.current_file = f"{source}/{rel}" + _status.files_done = done + _status.files_total = total + + summary: ImportSummary = await import_sources(sources, llm, prune=True, progress=_hook) overview = False if summary.added + summary.updated > 0: overview = await regenerate_overview(llm) @@ -234,6 +272,7 @@ async def _run_sync() -> None: db.close() _status.state = "success" _status.finished_at = datetime.now(UTC) + _status.current_file = None # phase 64: keep the final counts _status.detail = { "files": summary.files, "added": summary.added, @@ -253,3 +292,4 @@ async def _run_sync() -> None: _status.state = "failed" _status.finished_at = datetime.now(UTC) _status.error = _sanitize_error(str(e)) + _status.current_file = None # phase 64: keep the final counts diff --git a/app/rag/importer.py b/app/rag/importer.py index 294e987..f3f3c82 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -26,11 +26,15 @@ no longer exist **or no longer match the format filter** — this is how previously-imported junk (e.g. dot-dir READMEs) leaves the index. Per-file logging uses the verbs ``added | updated | unchanged | pruned`` plus a summary line with per-format counts (PLAN §9). + +``import_sources`` accepts an optional per-file ``progress`` callback +(phase 64, task 01) reporting the file being processed right now. """ from __future__ import annotations import hashlib import logging +from collections.abc import Callable from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -148,12 +152,26 @@ async def import_sources( prune: bool = False, limit: int | None = None, session: Session | None = None, + progress: Callable[[str, str, int, int], None] | None = None, ) -> ImportSummary: """Import every A9-format file under *sources* (see module docstring). ``session`` may be supplied (tests); a private one is opened and closed otherwise. ``limit`` caps the number of files processed (debug only) and disables pruning, since an incomplete walk must not drive deletions. + + ``progress`` (phase 64, task 01) is an optional per-file hook called + once per importable file, immediately before that file's + ``_index_file`` — with ``(source, rel_posix_path, done, total)``: the + same POSIX *rel* the document rows use, ``done`` = the 1-based index of + the current file **across all sources**, and ``total`` = the combined + pre-walk count of importable files across all *sources* roots. The + pre-walk (same extension/exclusion rules, directory stats only, no file + reads) happens **only when *progress* is provided**: callers passing + nothing pay no extra walk and behave exactly as before. Under + ``limit``, the hook still fires per processed file only — ``done`` never + exceeds the limit, but ``total`` stays the full pre-walk count (an + incomplete walk must not misreport the denominator). """ if limit is not None and limit <= 0: raise ValueError("limit must be >= 1") @@ -163,6 +181,14 @@ async def import_sources( session = SessionLocal() seen: set[tuple[str, str]] = set() source_names: set[str] = set() + # phase 64 (task 01): the hook's combined denominator, walked with the + # exact same rules as the processing loop below (directory stats only, + # no file reads). Skipped entirely for ``progress=None`` callers — no + # extra pass, byte-identical behaviour and cost. + total = 0 + if progress is not None: + for root in sources: + total += len(iter_importable_files(root, llm.settings.import_extension_set)) try: for root in sources: if not root.is_dir(): @@ -180,6 +206,12 @@ async def import_sources( summary.files += 1 ext = path.suffix.lower().lstrip(".") or "unknown" summary.formats[ext] = summary.formats.get(ext, 0) + 1 + if progress is not None: + # phase 64: report the file *before* indexing it — a + # file that then errors or turns out unchanged was + # already the "current file". No try/except around the + # call: the hooks in this repo only assign fields. + progress(source, rel, summary.files, total) try: await _index_file( session, source=source, rel=rel, full_path=path, llm=llm, diff --git a/app/schemas.py b/app/schemas.py index c7d2a60..12bb6c1 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -304,9 +304,12 @@ class GitSourceList(BaseModel): class UploadOut(BaseModel): - """``POST /api/git-sources/upload`` response (phase 49, task 02). + """The upload run's result fields (phase 49, task 02; phase 64, task 03). - The uploaded source's name (filename minus the archive suffix) plus + Phase 64 (task 03): ``POST /api/git-sources/upload`` answers 202 the + moment the archive is on disk; these fields become the shape of + ``GET /api/git-sources/upload/status`` ``detail`` on ``success`` — + the uploaded source's name (filename minus the archive suffix) plus the SAME count keys as the admin sync's success ``detail`` (``files``, ``added``, ``updated``, ``unchanged``, ``pruned``, ``errors``, ``chunks`` — ``app.api.sync._run_sync``) and the @@ -325,6 +328,21 @@ class UploadOut(BaseModel): overview: bool +class UploadAccepted(BaseModel): + """``POST /api/git-sources/upload`` 202 response (phase 64, task 03). + + The archive is **safely on disk** — this is the "successfully + uploaded" moment the Sources page toasts on (owner-locked A2). The + scan itself (unpack → swap → row upsert → model check → import → + overview) runs in a background task behind + ``GET /api/git-sources/upload/status``, whose ``success`` ``detail`` + carries the :class:`UploadOut` fields. + """ + + detail: str = "upload received" + name: str + + class ToolCall(BaseModel): """One agent tool-call record (the phase-37 ``tools`` record shape). diff --git a/frontend/assets/git-sources.js b/frontend/assets/git-sources.js index 5de6d17..a3ed0e8 100644 --- a/frontend/assets/git-sources.js +++ b/frontend/assets/git-sources.js @@ -34,20 +34,38 @@ * shape-aware like the tuning forms) and keeps the input — the * instruction survives. 409/422 details are fixed generic strings * (credential safety — the URL is never echoed). - * • upload — #archive-upload-form submit (phase 49) → POST + * • upload — #archive-upload-form submit (phase 49, reworked to the + * phase-64 202 contract in task 05 — the phase-49 synchronous + * 200 paragraph is superseded): POST * /api/git-sources/upload with a FormData file (NO manual * Content-Type — the browser sets the multipart boundary). The - * SAME §7.4 never-stale lifecycle: the button disables + - * relabels "Uploading…" while the request is out and is restored - * on success AND failure. 200 clears the file input, shows the - * sync-style count line ("2 added · 1 pruned" — fmtUploadResult, - * sources.js's fmtSyncResult convention) in the role=status - * result line, announces "Archive uploaded: …" and reloads the - * list (the new/updated row lands with the Local badge; a - * re-upload simply refreshes the row — no duplicate). Non-2xx - * inlines the server detail (422 format/name/traversal, 413 - * size, 409 busy — the messages are already user-safe) and KEEPS - * the file selection — the fix is one re-pick, not a re-type. + * §7.4 never-stale lifecycle keeps its shape — the button + * disables + relabels "Uploading…" while the request is out — + * but the transfer is now short: the 202 arrives the moment the + * archive is safely on disk (A1). 202 → the page-local + * "Successfully uploaded — " toast fires (showUploadToast, + * the phase-55 share-toast pattern; A2: safe to navigate away), + * the file input clears, and the button hands over to the scan — + * the processing state ("Processing…", disabled, title cleared) + * plus startUploadPolling(): a 2 s poll of + * GET /api/git-sources/upload/status renders the live + * "Processing… (n/m)" label (A4 — bare during unpack; the + * full path rides the button title) and settles it: success → + * the sync-style count line (fmtUploadResult, the role=status + * result line) + the "Archive uploaded: …" announce + + * loadSources (the new/updated row lands with the Local badge; + * a re-upload refreshes the row — no duplicate; NO second toast + * — A2); failure → the sanitized server error in the role=alert + * banner + loadSources, the file selection KEPT for a one-click + * re-upload. 409 (an upload is already in progress) raises NO + * error banner — it re-attaches to the in-flight run (processing + * state + poll, never stale). Other non-2xx (422 format/name, + * 413 cap, 5xx) keep the phase-49 error banner + the kept file + * selection. The submit finally restores the button ONLY when no + * poll is active (§7.4). Boot re-attach (initUploadStatus, admin + * branch): a running scan re-enters the processing state + poll + * (a reload mid-scan re-attaches — no second upload), a terminal + * run re-renders its result line / error banner. * • remove — a row's Remove button asks window.confirm first * (removal prunes the documents only on the NEXT sync — the * confirm says so). Cancel → nothing; ok → the row button @@ -64,8 +82,8 @@ * removing a source does NOT clone, import, or prune — the sync * service (server-side) performs that; the page's hint box says so. * The phase-49 upload is the exception: it unpacks and scans the - * single source in place, and its response counts render as the - * result line. + * single source in place (the phase-64 background task — 202 + + * status endpoint), and its counts render as the result line. * * The shared header module loads through this script's own relative * import ("./header.js") — a hoisted import evaluated before this body @@ -354,22 +372,42 @@ wireAddForm({ idleLabel: "Add source", }); -/* ---------- upload (POST /api/git-sources/upload) — phase 49 ------- - * The archive upload form: the file input's selection is posted as - * FormData (the browser sets the multipart boundary — no manual - * Content-Type). §7.4 never-stale: "Uploading…" while in flight, - * restored in the finally block on success AND failure. 200 → the - * input clears, the sync-style counts land in the role=status result - * line, the announcer confirms, and loadSources() re-renders the row - * (Local badge; a re-upload refreshes the existing row — no - * duplicate). Non-2xx → the server detail inline (role=alert; 422 - * format/name/traversal, 413 size, 409 busy — user-safe as-is) with - * the file selection KEPT; network failure → the fixed line. */ +/* ---------- upload (POST /api/git-sources/upload) — phase 64 (task 05) ------- + * The archive upload form follows the phase-64 202 contract (A1): + * the file input's selection is posted as FormData (the browser sets + * the multipart boundary — no manual Content-Type), and the 202 + * answers the moment the archive is safely on disk — the "Uploading…" + * label covers only that short receive. Then the button HANDS OVER to + * the scan: 202 → the page-local "Successfully uploaded — " + * toast (showUploadToast — A2, safe to navigate away), the file input + * clears, and the processing state ("Processing…", disabled, title + * cleared) + startUploadPolling() own it — a 2 s poll of + * GET /api/git-sources/upload/status renders the live "Processing… + * (n/m)" label (A4 — bare during unpack; the full path rides + * the button title) and settles it: success → the sync-style count + * line (fmtUploadResult) in the role=status result line + the + * "Archive uploaded: …" announce + loadSources (NO second toast — it + * already fired at the 202, A2); failure → the sanitized server + * error in the role=alert banner + loadSources, the file selection + * KEPT for a one-click re-upload. 409 (an upload is already in + * progress) raises NO error banner — it re-attaches to the in-flight + * run (processing state + poll, never stale); the phase-49 "server + * detail inline for 409" branch is superseded. Other non-2xx (422 + * format/name, 413 cap, 5xx) keep the phase-49 error banner + the + * kept file selection; a network failure keeps the fixed line. The + * submit finally restores the button ONLY when no poll is active + * (PLAN §7.4 — while startUploadPolling owns the button it stays + * disabled / "Processing…"). Boot re-attach (initUploadStatus, the + * admin branch): a running scan re-enters the processing state + poll + * (no second upload, no error); a terminal run re-renders its result + * line (success) or error banner (failed); idle does nothing. + * (The phase-49 synchronous 200 paragraph is superseded by phase 64.) */ /* The success line's text — the sync-result shape (sources.js's fmtSyncResult convention): "N added" always leads, then updated / unchanged / pruned — zero parts omitted (unchanged is shown - when nothing was added or updated). */ + when nothing was added or updated). Reads exactly the keys the + upload status's detail carries (task 03's UploadOut-shaped dict). */ function fmtUploadResult(detail) { const d = detail || {}; const added = d.added || 0; @@ -383,6 +421,183 @@ function fmtUploadResult(detail) { return parts.join(" · "); } +/* Upload-success toast (phase 64 task 05, A2 — owner-locked): the + * "successfully uploaded" confirmation, the phase-55 share-toast + * pattern (frontend/assets/app.js) made page-local. A SINGLE node — + * lazy-created on the first 202 and reused (toasts never stack): a + * new toast replaces a pending one (clear the prior dismiss timer, + * re-run the entry). role="status" aria-live="polite" — on THIS page + * the toast IS the a11y announcer for the 202 (there is no other + * live-region line for it). SUCCESS-ONLY (A2): failures are the + * #archive-upload-error banner, never a toast. The .toast CSS ships + * as-is (styles.css, phase 55). */ +const UPLOAD_TOAST_MS = 5000; // ~5 s auto-dismiss (A2) +let uploadToastEl = null; // the single toast node — lazy-created, reused +let uploadToastTimer = 0; // the pending auto-dismiss (replaced by a new toast) + +function showUploadToast(message) { + if (!uploadToastEl) { + uploadToastEl = document.createElement("div"); + uploadToastEl.className = "toast"; + uploadToastEl.setAttribute("role", "status"); + uploadToastEl.setAttribute("aria-live", "polite"); + document.body.appendChild(uploadToastEl); + } + uploadToastEl.textContent = message; // XSS-safe text assignment + // Re-trigger the entry even when a toast is already up (a second + // upload accepted while the first toast is showing): clear the + // pending dismiss, drop the visible state, force a reflow + // (restarts the CSS transition), then show again. + clearTimeout(uploadToastTimer); + uploadToastEl.classList.remove("is-visible"); + void uploadToastEl.offsetWidth; // force reflow — the entry transition restarts + uploadToastEl.classList.add("is-visible"); + uploadToastTimer = setTimeout(() => { + uploadToastEl.classList.remove("is-visible"); // auto-dismiss ~5 s + }, UPLOAD_TOAST_MS); +} + +/* The scan poll (phase 64 task 05): a 2 s cadence — the SYNC_POLL_MS + * house value. Single timer, one loop at a time (the guard makes a + * double-start a no-op, and the submit finally reads this same + * variable to know whether the poll OWNS the button). Each tick + * fetches GET /api/git-sources/upload/status: running → the live + * "Processing… (n/m)" label (A4 — bare "Processing…" during + * the unpack phase, before any file is indexed; the full untruncated + * path rides the button title) + reschedule; success → stop + the + * result line + the announcement + the row reload (NO toast — it + * fired at the 202, A2); failed → stop + the sanitized server error + * banner + the row reload (a post-swap failure keeps the row — the + * list state may have changed), the file selection kept for a + * one-click re-upload; idle → stop + the button restored + * (defensive — a started run never returns to idle). A network blip + * retries next tick. */ +const UPLOAD_POLL_MS = 2000; // the SYNC_POLL_MS house value +let uploadPollTimer = null; // null = no poll active (the finally's guard) + +function stopUploadPolling() { + if (uploadPollTimer !== null) { + clearTimeout(uploadPollTimer); + uploadPollTimer = null; + } +} + +/* The button's processing entry (the 202 + the 409 re-attach): from + * here the poll OWNS it — disabled, "Processing…", title cleared (a + * live file lands on it at the first tick). */ +function enterUploadProcessingState() { + uploadBtn.disabled = true; + uploadBtn.textContent = "Processing…"; + uploadBtn.title = ""; +} + +/* The idle restore (the poll's terminal branches + the submit + * finally, which calls this ONLY when no poll is active — PLAN §7.4). */ +function restoreUploadButton() { + uploadBtn.disabled = false; // never stale — success OR failure + uploadBtn.textContent = "Upload & scan"; + uploadBtn.removeAttribute("title"); +} + +function startUploadPolling() { + if (uploadPollTimer !== null) return; // one poll loop at a time + const tick = async () => { + let status = null; + try { + const r = await fetch("/api/git-sources/upload/status"); + if (r.ok) status = await r.json(); + } catch { /* network blip — retry next tick */ } + if (!status) { + uploadPollTimer = setTimeout(tick, UPLOAD_POLL_MS); + return; + } + // running: the live file label (A4 — bare "Processing…" during + // the unpack phase, before any file is indexed). + if (status.state === "running") { + uploadBtn.textContent = + "Processing…" + + (status.current_file ? ` ${status.current_file}` : "") + + (status.files_total > 0 ? ` (${status.files_done}/${status.files_total})` : ""); + uploadBtn.title = status.current_file || ""; // full path on hover + uploadPollTimer = setTimeout(tick, UPLOAD_POLL_MS); + return; + } + stopUploadPolling(); + if (status.state === "success") { + // The scan finished: the result line (the existing helper reads + // exactly these keys), the announcement, the row lands. NO toast + // here — it already fired at the 202 (A2). + const detail = status.detail || {}; + if (uploadResult) { + uploadResult.textContent = fmtUploadResult(detail); + uploadResult.hidden = false; + } + announce(`Archive uploaded: ${detail.source}.`); + uploadFileInput.value = ""; + restoreUploadButton(); + loadSources(); // the row lands / refreshes + return; + } + if (status.state === "failed") { + // Sanitized server-side (task 03's _sanitize_error): the banner + // is the failure UI (A2), the selection stays KEPT for a + // one-click re-upload, and the list reloads (a post-swap failure + // keeps the row — the list state may have changed). + if (uploadError) { + uploadError.textContent = status.error || "The upload scan failed."; + uploadError.hidden = false; + } + restoreUploadButton(); + loadSources(); // the list state may have changed + return; + } + // idle: defensive — a started run never returns to idle; just + // settle the button (the poll stopped above). + restoreUploadButton(); + }; + uploadPollTimer = setTimeout(tick, UPLOAD_POLL_MS); +} + +/* Boot re-attach (phase 64 task 05, the admin branch only): fetch the + * upload status ONCE — a running scan re-enters the processing state + * + the poll (a reload mid-scan re-attaches instead of dead-ending — + * no second upload, no error); a finished run re-renders its result + * line ONLY (no announce, no toast — the toast fired at the 202, A2); + * a failed run re-renders its error banner; idle does nothing (and a + * blip is a no-op — the page boots honest either way). */ +async function initUploadStatus() { + if (!uploadBtn) return; + let status; + try { + const r = await fetch("/api/git-sources/upload/status"); + if (!r.ok) return; + status = await r.json(); + } catch { + /* network blip — boot without the re-attach */ + } + if (!status) return; + if (status.state === "running") { + enterUploadProcessingState(); + startUploadPolling(); // the first tick carries the live file + return; + } + if (status.state === "success") { + // The last run's result line only — no announce, no toast (A2). + if (uploadResult) { + uploadResult.textContent = fmtUploadResult(status.detail); + uploadResult.hidden = false; + } + return; + } + if (status.state === "failed") { + if (uploadError) { + uploadError.textContent = status.error || "The upload scan failed."; + uploadError.hidden = false; + } + } + // idle: nothing to re-attach. +} + if (uploadFormEl && uploadFileInput && uploadBtn) { uploadFormEl.addEventListener("submit", async (e) => { e.preventDefault(); @@ -399,7 +614,7 @@ if (uploadFormEl && uploadFileInput && uploadBtn) { if (uploadError) uploadError.hidden = true; if (uploadResult) uploadResult.hidden = true; // a new attempt starts clean uploadBtn.disabled = true; // §7.4: one upload per click - uploadBtn.textContent = "Uploading…"; + uploadBtn.textContent = "Uploading…"; // the transfer is now short — the 202 try { // Multipart from the form itself (the file input's name is // "file") — the browser sets the boundary; NO manual @@ -408,23 +623,35 @@ if (uploadFormEl && uploadFileInput && uploadBtn) { method: "POST", body: new FormData(uploadFormEl), }); - if (r.ok) { - let data = {}; + if (r.status === 202) { + // The archive is safely on disk (A1) — the "successfully + // uploaded" moment: the toast fires NOW (A2), the file input + // clears, and the scan's poll takes over the button. The 202 + // body (UploadAccepted) carries the safe source name; a body + // parse failure degrades to the picked file's name. + let name = file.name; try { - data = await r.json(); + const data = await r.json(); + if (data && typeof data.name === "string" && data.name) name = data.name; } catch { - /* the body is advisory — the counts line degrades gracefully */ + /* body parse failure — the picked file's name degrades fine */ } - uploadFileInput.value = ""; // 200: the archive is unpacked + scanned - if (uploadResult) { - uploadResult.textContent = fmtUploadResult(data); - uploadResult.hidden = false; - } - announce(`Archive uploaded: ${data.source || file.name}.`); - await loadSources(); // the new/updated row lands (Local badge) + showUploadToast(`Successfully uploaded — ${name}`); + uploadFileInput.value = ""; // 202: the archive is on the server + enterUploadProcessingState(); + startUploadPolling(); return; } - // 422 (format/name/traversal), 413 (size), 409 (busy): the server + // 409 (an upload is already in progress): NO error banner — + // re-attach to the in-flight run (never stale): the processing + // state + the poll track it to the terminal. The phase-49 + // "server detail inline" branch does not apply to 409 anymore. + if (r.status === 409) { + enterUploadProcessingState(); + startUploadPolling(); + return; + } + // Other non-2xx (422 format/name, 413 cap, 5xx): the server // detail inline, the file selection KEPT — the fix is one // re-pick, not a re-type. if (uploadError) { @@ -437,8 +664,11 @@ if (uploadFormEl && uploadFileInput && uploadBtn) { uploadError.hidden = false; } } finally { - uploadBtn.disabled = false; // never stale — success OR failure - uploadBtn.textContent = "Upload & scan"; + // Never stale (PLAN §7.4) — but ONLY when no poll owns the + // button: while startUploadPolling tracks the scan (202 / 409) + // it stays disabled / "Processing…", so a finally restore here + // would race the poll. No poll → the button is ours to restore. + if (uploadPollTimer === null) restoreUploadButton(); } }); } @@ -474,4 +704,8 @@ if (retryBtn) retryBtn.addEventListener("click", () => loadSources()); if (gateEl) gateEl.hidden = true; if (contentEl) contentEl.hidden = false; await loadSources(); + // Phase 64 (task 05): re-attach a running scan (a reload mid-scan + // resumes the Processing state) or re-render a terminal run's + // result line / error banner. + await initUploadStatus(); })(); diff --git a/frontend/assets/sources.js b/frontend/assets/sources.js index ce8b809..995f904 100644 --- a/frontend/assets/sources.js +++ b/frontend/assets/sources.js @@ -14,6 +14,13 @@ * page "new chat" means going to the chat, fresh — the module clears * the phase-14 conversation key and navigates to "/"). * + * Phase 64 (task 04): the sync button is also the live progress face + * of the BACKGROUND archive scan (task 03) — while an upload import + * is in flight the button animates with the upload's current file + * ("Importing "), settles + refreshes the catalog when it + * finishes, and re-attaches to it on page load. See the sync-button + * block below for the full two-job contract. + * * Phase 26: the table's path links open the document in the * almost-fullscreen modal overlay (assets/document-modal.js) on the * SAME page — no new tab, no navigation. The link keeps its @@ -36,6 +43,33 @@ import { fetchIsAdmin, initSharedHeader } from "./header.js"; * opens an error modal (same as the former header.js module — recreated * here since the navbar button is gone). * + * Phase 64 (task 04): the button reports the FILE being processed, not + * just "Syncing…" — TWO jobs drive it. Each poll tick fetches BOTH + * status endpoints — GET /api/sync/status + + * GET /api/git-sources/upload/status — and applies this decision + * tree, in order (startSyncPolling): + * 1. sync running → "Syncing… (n/m)" — bare "Syncing…" until + * the import's first file (clone/pull, A4); + * 2. upload running → "Importing (n/m)" — the background + * archive scan (the "clicked upload, then opened + * sources" contract, A3); + * 3. sync success → the phase-32 settle (counts + catalog refresh); + * 4. sync failed → the phase-32 failure (banner + modal); + * 5. upload success → settle "Sync sources" + catalog refresh + * (loadDocs — the new documents must appear); the + * upload's counts live on the Sources page, never + * in #sync-result (A3); + * 6. upload failed → settle "Sync sources" — the failure is the + * Sources page's error banner, never this page's (A3); + * 7. both idle → retry-ready idle. + * The live label is the status endpoint's full source/relative/path + * (A4): CSS ellipsizes #sync-label; the full untruncated path also + * rides the button title (hover) and #sync-result (the aria-live + * announcer — screen readers hear it). The load-time re-attach + * (initSyncButton) re-enters a RUNNING upload the same way; a terminal + * upload is a no-op there (the boot-time loadDocs() already shows the + * current catalog). + * * Elements: #sync-btn (the button), #sync-label (the text), * #sync-icon (the spinner icon), #sync-result (aria-live result * line), #sync-error-banner / #sync-error-text (error banner). @@ -70,6 +104,22 @@ function fmtSyncTime(iso) { return `${pad(d.getHours())}:${pad(d.getMinutes())}`; } +/* Phase 64 (task 04): the live-file label. `kind` picks the prefix — + * "sync" → "Syncing…", "upload" → "Importing" (the background scan's + * word, A3). The current file — the status endpoint's full + * source/relative/path (A4) — is appended while one is being processed; + * the BARE prefix shows during the clone/pull (sync) or unpack (upload) + * phase, before any file is indexed. The counts appear only once the + * import has started (total > 0). CSS ellipsizes the button label; the + * same untruncated text goes to the button title + #sync-result (the + * aria-live announcer). */ +function fmtSyncLabel(kind, currentFile, done, total) { + const prefix = kind === "upload" ? "Importing" : "Syncing…"; + let label = currentFile ? `${prefix} ${currentFile}` : prefix; + if (total > 0) label += ` (${done}/${total})`; + return label; +} + function fmtSyncResult(detail) { const d = detail || {}; const added = d.added || 0; @@ -88,15 +138,25 @@ function sanitizeSyncError(message) { return text.length > 200 ? `${text.slice(0, 200)}…` : text; } -function enterSyncRunningState() { +/* The single running-state entry point (phase 64 task 04: the label + * carries the live file — `kind` "sync" | "upload", the status + * endpoint's current_file + done/total). Same mechanics as before + * (disabled, aria-busy, spinning icon, no is-error) plus: the full + * untruncated path on the button title (removed when null — no file + * yet) and in #sync-result (the aria-live announcer reads the full + * live path; CSS ellipsizes the button's label span only). */ +function enterSyncRunningState(kind, currentFile, done, total) { if (!syncBtn) return; syncBtn.disabled = true; syncBtn.setAttribute("aria-busy", "true"); - syncBtn.removeAttribute("title"); + if (currentFile) syncBtn.title = currentFile; + else syncBtn.removeAttribute("title"); syncBtn.setAttribute("aria-label", "Sync sources"); syncBtn.classList.remove("is-error"); if (syncIcon) syncIcon.classList.add("is-spinning"); - if (syncLabel) syncLabel.textContent = "Syncing…"; + const label = fmtSyncLabel(kind, currentFile, done, total); + if (syncLabel) syncLabel.textContent = label; + if (syncResult) syncResult.textContent = label; } function settleSyncButton(label) { @@ -193,15 +253,22 @@ function applySyncIdle(status) { emitSyncStatus(status || { state: "idle" }); } +/* The 2 s poll (phase 64 task 04): each tick fetches BOTH jobs — the + * sync AND the background upload scan — and applies the two-job + * decision tree in order (see the section header). The 403 on the SYNC + * fetch hides the button (the whoami backstop); a 403 on the UPLOAD + * fetch is simply "no upload" (never a hide), and a network blip on + * either fetch retries next tick. */ function startSyncPolling() { if (syncPollTimer !== null) return; const tick = async () => { - let status = null; + let syncStatus = null; + let uploadStatus = null; let notAdmin = false; try { const r = await fetch("/api/sync/status"); if (r.status === 403) notAdmin = true; - else if (r.ok) status = await r.json(); + else if (r.ok) syncStatus = await r.json(); } catch { /* network blip — retry next tick */ } if (notAdmin) { stopSyncPolling(); @@ -209,28 +276,66 @@ function startSyncPolling() { applySyncIdle(); return; } - if (!status) { + // The SECOND job: the background upload scan (admin-only surface). + try { + const ur = await fetch("/api/git-sources/upload/status"); + if (ur.ok) uploadStatus = await ur.json(); + } catch { /* network blip — retry next tick */ } + if (!syncStatus) { syncPollTimer = setTimeout(tick, SYNC_POLL_MS); return; } - if (status.state === "success") { - stopSyncPolling(); - applySyncSuccess(status); + // 1. sync running: the live sync file (bare "Syncing…" until the + // import's first file — A4). + if (syncStatus.state === "running") { + enterSyncRunningState( + "sync", syncStatus.current_file, syncStatus.files_done, syncStatus.files_total + ); + syncPollTimer = setTimeout(tick, SYNC_POLL_MS); return; } - if (status.state === "failed") { - stopSyncPolling(); - applySyncFailure(status); + // 2. upload running: the same animation, the upload's file (A3). + if (uploadStatus && uploadStatus.state === "running") { + enterSyncRunningState( + "upload", uploadStatus.current_file, uploadStatus.files_done, uploadStatus.files_total + ); + syncPollTimer = setTimeout(tick, SYNC_POLL_MS); return; } - if (status.state === "idle") { + if (syncStatus.state === "success") { stopSyncPolling(); - applySyncIdle(status); + applySyncSuccess(syncStatus); return; } - // Still running: keep button state honest and re-schedule. - enterSyncRunningState(); - syncPollTimer = setTimeout(tick, SYNC_POLL_MS); + if (syncStatus.state === "failed") { + stopSyncPolling(); + applySyncFailure(syncStatus); + return; + } + // 5. upload success: settle + catalog refresh (A3 — the upload's + // counts live on the Sources page; #sync-result stays empty). + if (uploadStatus && uploadStatus.state === "success") { + stopSyncPolling(); + settleSyncButton("Sync sources"); + if (syncResult) syncResult.textContent = ""; + hideSyncError(); + emitSyncStatus({ state: "idle" }); + loadDocs(); + return; + } + // 6. upload failed: settle only — the failure is the Sources page's + // error banner, never this page's (A3). + if (uploadStatus && uploadStatus.state === "failed") { + stopSyncPolling(); + settleSyncButton("Sync sources"); + if (syncResult) syncResult.textContent = ""; + hideSyncError(); + emitSyncStatus({ state: "idle" }); + return; + } + // 7. both idle: settle retry-ready. + stopSyncPolling(); + applySyncIdle(syncStatus); }; syncPollTimer = setTimeout(tick, SYNC_POLL_MS); } @@ -253,8 +358,10 @@ async function startSync() { return; } if (r.status === 202 || r.status === 409) { - enterSyncRunningState(); - if (syncResult) syncResult.textContent = ""; + // Phase 64: the run is just starting (model check / clone-pull) — + // bare "Syncing…" until the first polled file (A4); entering the + // running state also clears #sync-result with the same label. + enterSyncRunningState("sync", null, 0, 0); hideSyncError(); if (lastSyncState !== "running") emitSyncStatus({ state: "running" }); startSyncPolling(); @@ -268,8 +375,12 @@ async function startSync() { }); } -/* Load-time re-attach (ADMIN ONLY): a running run re-enters running state, - * a terminal run renders its last result. */ +/* Load-time re-attach (ADMIN ONLY): a running run re-enters running + * state, a terminal run renders its last result. Phase 64 (A3): with + * the sync IDLE, an in-flight background upload scan adopts the button + * the same way — the "user clicked upload, then opened sources" case; + * a terminal upload is a no-op (the boot-time loadDocs() already shows + * the current catalog). */ async function initSyncButton() { if (!syncBtn) return; if (!(await fetchIsAdmin())) return; @@ -281,16 +392,36 @@ async function initSyncButton() { status = await r.json(); } catch { return; } if (status.state === "running") { - enterSyncRunningState(); + enterSyncRunningState( + "sync", status.current_file, status.files_done, status.files_total + ); emitSyncStatus(status); startSyncPolling(); - } else if (status.state === "success") { - applySyncSuccess(status); - } else if (status.state === "failed") { - applySyncFailure(status); - } else { - applySyncIdle(status); + return; } + if (status.state === "success") { + applySyncSuccess(status); + return; + } + if (status.state === "failed") { + applySyncFailure(status); + return; + } + // Sync idle: check the SECOND job — an in-flight upload scan re-attaches. + let upload; + try { + const ur = await fetch("/api/git-sources/upload/status"); + if (ur.ok) upload = await ur.json(); + } catch { /* network blip — the idle settle below is still honest */ } + if (upload && upload.state === "running") { + enterSyncRunningState( + "upload", upload.current_file, upload.files_done, upload.files_total + ); + emitSyncStatus({ state: "running" }); + startSyncPolling(); + return; + } + applySyncIdle(status); } if (syncBtn) { diff --git a/frontend/assets/styles.css b/frontend/assets/styles.css index a30136c..39bacd4 100644 --- a/frontend/assets/styles.css +++ b/frontend/assets/styles.css @@ -1496,7 +1496,18 @@ details.thinking .thinking-text ul { margin: 0 0 0.5rem; } @media (prefers-reduced-motion: reduce) { .sync-btn .sync-icon.is-spinning { animation: none; } } -.sync-label { display: inline; } +/* Phase 64: the live-file label ("Syncing… (n/m)" / "Importing + (n/m)") — a long source/relative/path ellipsizes inside the + pill; the full path lives in the button title + #sync-result (the + aria-live announcer). */ +.sync-label { + display: inline-block; + max-width: min(16rem, 40vw); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + vertical-align: bottom; +} /* Last-result announcer — soft ink, small mono */ .sync-result { display: block; diff --git a/frontend/git-sources.html b/frontend/git-sources.html index da92de0..8843839 100644 --- a/frontend/git-sources.html +++ b/frontend/git-sources.html @@ -179,16 +179,27 @@ - +
- +