feat(sources): admin page to add and remove git sources (TODO.md L4)

This commit is contained in:
2026-08-26 18:42:28 -04:00
parent b2d8696741
commit 1925bb66a8
30 changed files with 2673 additions and 110 deletions
+572
View File
@@ -0,0 +1,572 @@
"""Phase 35 story E2E (Playwright): the admin page to add / remove git
sources (``/git-sources.html``).
Story: ``.agent/user_stories/git-sources-admin.md``
Run in isolation (DB must be up: ``podman compose up -d db``):
uv run pytest tests/e2e/test_git_sources_admin.py -v --no-cov
The story gate for the admin-managed git source list. **No git, no
network** — this suite is UI + API only (the clone/import pipeline is
mocked at the integration level, phase 35 task 03; a real sync is
deliberately never triggered here). The stored list (``git_sources``
table, migration 0006) is exercised through the real page and the real
admin-only CRUD API (phase 35 task 02).
Per-module app env (the conftest pattern, module-scoped — as in
``test_sync_button.py``): this story's app boots with a fixed
``BOR_GIT_SOURCES`` CSV (two deterministic URLs that are NEVER cloned —
nothing in this suite triggers a sync) so the empty-table env fallback
(the phase's locked decision: ``BOR_GIT_SOURCES`` only applies while the
``git_sources`` table is empty) can be asserted against real env rows.
The session app (no git sources) is never started in this isolated run,
so no port clash.
Contract under test:
* anonymous: the sign-in gate (the exact ``#sources-gate`` pattern), the
manager hidden (list + add form inert), NO ``/api/git-sources`` call,
403 on all three routes (asserted with the page context's request
client, the ``test_admin_auth.py``/``test_sync_button.py`` pattern),
and ``#nav-git-sources`` hidden on all five pages;
* admin: ``#nav-git-sources`` visible on all five pages (revealed by the
shared header on the cached whoami), clicking it from the chat lands
on ``/git-sources.html`` with the link ``is-active``; the stored rows
render as a full-width table (mono URL, added date, per-row Remove)
with the env note hidden while the DB has rows; add (201 → row, input
cleared, button re-enabled), duplicate (inline role=alert, no new
row), invalid shape (inline 422, no new row), remove (confirm → gone;
cancel → stays); with the table truncated the env rows render with
"from .env" tags and ``#git-sources-env-note`` visible;
* the page a11y / no-CDN basics (UI Structure Check, AGENTS.md rule 5):
landmarks, labeled form control, full-width table, ≥44px targets,
3px focus-visible outline, the aria-live list announcer, same-origin
assets only (rule 6).
Test → story mapping (Playwright Mapping Rule):
1. ``test_anonymous_gate_no_api_calls_and_403s``
2. ``test_admin_nav_link_on_all_five_pages_and_click_navigates``
3. ``test_admin_sees_seeded_rows_with_dates_and_no_env_note``
4. ``test_admin_add_then_remove_lifecycle``
5. ``test_admin_env_fallback_rows_and_note``
6. ``test_admin_page_a11y_and_no_cdn``
"""
from __future__ import annotations
import os
import re
import subprocess
import sys
import uuid
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import pytest
from playwright.sync_api import Dialog, Page, expect
from sqlalchemy import text
from app.db import SessionLocal
from app.models import GitSource
from e2e.auth_helpers import login
from e2e.conftest import (
ADMIN_PASSWORD,
APP_PORT,
SESSION_SECRET,
USE_REAL_LLM,
_wait_http,
)
REPO = Path(__file__).resolve().parents[2]
APP_URL = f"http://127.0.0.1:{APP_PORT}"
#: The five pages that ship the header (phase 34 contract) — the pages
#: task 05 gave the admin-only "Git sources" nav link.
CHAT_URL = "/"
SOURCES_URL = "/sources.html"
VIEWER_URL = "/document.html?source=docs&path=homelab%2Fkubernetes.md"
TUNING_URL = "/tuning.html"
GIT_SOURCES_URL = "/git-sources.html"
FIVE_PAGES = (
("chat", CHAT_URL),
("sources", SOURCES_URL),
("viewer", VIEWER_URL),
("tuning", TUNING_URL),
("git-sources", GIT_SOURCES_URL),
)
#: The module app's ``BOR_GIT_SOURCES`` — two deterministic URLs that
#: are NEVER cloned (no sync is triggered in this suite; the fallback
#: list is what is under test).
ENV_SOURCE_A = "https://github.com/reese/env-alpha.git"
ENV_SOURCE_B = "https://github.com/reese/env-beta.git"
ENV_SOURCES_CSV = ",".join((ENV_SOURCE_A, ENV_SOURCE_B))
#: ``is-active`` as a word-boundary regex — to_have_class() matches the
#: WHOLE class string, so the current-page marker is asserted the same
#: way test_tuning_nav_link.py does it.
IS_ACTIVE = re.compile(r"\bis-active\b")
#: Deterministic URLs for the UI-driven assertions.
SEED_URL = "https://gitlab.example.com/reese/seeded.git"
NEW_REPO_URL = "https://example.com/reese/new-repo.git"
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def app_server(mock_llm: int) -> Iterator[str]:
"""The real app under test — per-module env: a fixed two-URL
``BOR_GIT_SOURCES`` CSV so the empty-table env fallback (the phase's
locked decision) is live. No ``BOR_SOURCES_DIR``, no git: nothing in
this suite clones or syncs."""
env = dict(os.environ)
env.pop("DEBUGPY", None)
env["BOR_ENVIRONMENT"] = "e2e"
env["BOR_STATIC_DIR"] = str(REPO / "frontend")
env["BOR_LLM_BASE_URL"] = (
"https://aipi.reeseapps.com/v1"
if USE_REAL_LLM
else f"http://127.0.0.1:{mock_llm}/v1"
)
# Mock-calibrated threshold (conftest pattern) — no chat turn is
# ever sent in this suite, but the app boots with the same env shape.
env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
env.setdefault(
"BOR_DATABASE_URL",
"postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese",
)
# Phase 16: admin auth must be set or create_app() refuses to boot.
env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD
env["BOR_SESSION_SECRET"] = SESSION_SECRET
# Phase 35: the empty-table fallback list (never cloned here).
env["BOR_GIT_SOURCES"] = ENV_SOURCES_CSV
proc = subprocess.Popen(
[sys.executable, "-m", "uvicorn", "app.main:app",
"--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"],
cwd=REPO,
env=env,
)
try:
_wait_http(f"{APP_URL}/api/health")
yield APP_URL
finally:
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
@pytest.fixture(scope="module")
def app_url(app_server: str) -> str:
return app_server
def _truncate_git_sources() -> None:
with SessionLocal() as db:
db.execute(text("TRUNCATE git_sources"))
db.commit()
@pytest.fixture(autouse=True)
def _clean_git_sources(db_ready: None) -> Iterator[None]:
"""Fresh ``git_sources`` table per test — this suite owns the table
(the E2E isolation pattern); the KB tables are irrelevant here and
are left untouched (the viewer page may show its not-found state —
no document assertion is made about it). The table is emptied BOTH
before and after every test: suites run in isolation but share one
Postgres, and a leftover row would flip another suite's app from the
``BOR_GIT_SOURCES`` env fallback to the DB list (the sync story's
``effective_git_sources`` resolver reads the DB first)."""
_truncate_git_sources()
yield
_truncate_git_sources()
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _seed_rows(urls: list[str]) -> None:
"""Store rows directly (deterministic, ordered added_at) — the task
allows seeding via the API *or* SessionLocal; direct inserts keep
the list-rendering test independent of the add/remove lifecycle
test."""
base = datetime(2026, 8, 26, 9, 0, 0, tzinfo=UTC)
with SessionLocal() as db:
for i, url in enumerate(urls):
db.add(GitSource(url=url, added_at=base + timedelta(minutes=i)))
db.commit()
def _admin_git_sources_page(page: Page, app_url: str) -> None:
"""Real form login landing on the git sources page (admin settled:
Sign out visible, the manager revealed by the page module)."""
login(page, app_url, next=GIT_SOURCES_URL)
expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000)
expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000)
expect(page.locator("#git-sources-gate")).to_be_hidden()
expect(page.locator("#git-sources-content")).to_be_visible()
def _row(page: Page, url: str) -> Any:
"""The table row whose mono URL cell shows ``url``."""
return page.locator("#git-sources-tbody tr", has_text=url)
def _assert_settled(page: Page, admin: bool) -> None:
"""The shared header's whoami toggle has landed (one of Sign in /
Sign out visible) — the same settled-state gate as
test_nav_consistency.py."""
if admin:
expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000)
expect(page.locator("#sign-in-link")).to_be_hidden()
else:
expect(page.locator("#sign-in-link")).to_be_visible(timeout=15_000)
expect(page.locator("#sign-out-btn")).to_be_hidden()
# ---------------------------------------------------------------------------
# 1. Anonymous: gate, inert manager, no API calls, 403s, hidden link
# ---------------------------------------------------------------------------
def test_anonymous_gate_no_api_calls_and_403s(
page: Page, app_url: str, db_ready: None
) -> None:
page.set_default_timeout(30_000)
# Track every /api/git-sources request the page itself makes — the
# gate must be reached WITHOUT touching the admin API (the
# test_admin_auth.py pattern for the Sources page's /api/docs).
api_calls: list[str] = []
page.on(
"request",
lambda r: api_calls.append(r.url) if "/api/git-sources" in r.url else None,
)
page.goto(app_url + GIT_SOURCES_URL)
_assert_settled(page, admin=False)
# The sign-in gate (the #sources-gate pattern, phase 16)…
gate = page.locator("#git-sources-gate")
expect(gate).to_be_visible()
expect(gate).to_contain_text("Sign in to manage the git sources")
link = gate.locator("a[href='/login.html?next=/git-sources.html']")
expect(link).to_have_count(1)
box = link.bounding_box()
assert box is not None and box["height"] >= 44, f"gate link too small: {box}"
# …and the manager is absent/inert: list, add form, env note — all
# inside the hidden #git-sources-content.
expect(page.locator("#git-sources-content")).to_be_hidden()
expect(page.locator("#git-sources-table")).to_be_hidden()
expect(page.locator("#git-source-form")).to_be_hidden()
expect(page.locator("#git-sources-env-note")).to_be_hidden()
# The admin-only nav link is hidden on this page…
expect(page.locator("#nav-git-sources")).to_be_hidden()
# …and on the other four pages (all five ship it hidden by default).
for _name, path in FIVE_PAGES:
if path == GIT_SOURCES_URL:
continue
page.goto(app_url + path)
_assert_settled(page, admin=False)
expect(page.locator("#nav-git-sources")).to_be_hidden()
# The gate never called the admin API…
assert api_calls == [], f"anonymous page called the git sources API: {api_calls}"
# …and the API 403s anonymous callers on all three routes (the
# test_sync_button.py pattern — the page context has no cookie).
assert page.request.get(f"{app_url}/api/git-sources").status == 403
assert (
page.request.post(
f"{app_url}/api/git-sources", data={"url": NEW_REPO_URL}
).status
== 403
)
assert page.request.delete(f"{app_url}/api/git-sources/{uuid.uuid4()}").status == 403
# ---------------------------------------------------------------------------
# 2. Admin: the nav link on all five pages; the click lands + is-active
# ---------------------------------------------------------------------------
def test_admin_nav_link_on_all_five_pages_and_click_navigates(
page: Page, app_url: str, db_ready: None
) -> None:
page.set_default_timeout(30_000)
login(page, app_url, next=CHAT_URL)
expect(page).to_have_url(app_url + CHAT_URL, timeout=30_000)
# The link is revealed (admin) on every one of the five pages,
# pointing at the git sources page, labeled "Git sources" — and it
# is NOT the current page on the four non-git-sources pages.
for _name, path in FIVE_PAGES:
if path != CHAT_URL:
page.goto(app_url + path)
_assert_settled(page, admin=True)
link = page.locator("#nav-git-sources")
expect(link).to_be_visible(timeout=15_000)
expect(link).to_have_attribute("href", GIT_SOURCES_URL)
expect(link).to_have_text("Git sources")
if path != GIT_SOURCES_URL:
expect(link).not_to_have_class(IS_ACTIVE)
# From the chat: a real click navigates…
page.goto(app_url + CHAT_URL)
_assert_settled(page, admin=True)
page.click("#nav-git-sources")
expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000)
# …where the page's own link is the current-page marker.
link = page.locator("#nav-git-sources")
expect(link).to_be_visible(timeout=15_000)
expect(link).to_have_class(IS_ACTIVE)
expect(link).to_have_attribute("aria-current", "page")
# ---------------------------------------------------------------------------
# 3. Admin: two seeded rows render (mono URL + added date); env note
# hidden while the DB has rows
# ---------------------------------------------------------------------------
def test_admin_sees_seeded_rows_with_dates_and_no_env_note(
page: Page, app_url: str, db_ready: None
) -> None:
page.set_default_timeout(30_000)
_seed_rows([SEED_URL, ENV_SOURCE_A]) # ordered by added_at
_admin_git_sources_page(page, app_url)
# Both rows render, oldest first…
expect(page.locator("#git-sources-tbody tr")).to_have_count(2)
first = _row(page, SEED_URL)
second = _row(page, ENV_SOURCE_A)
expect(first.locator("td.git-source-url-cell code")).to_have_text(SEED_URL)
expect(second.locator("td.git-source-url-cell code")).to_have_text(ENV_SOURCE_A)
expect(page.locator("#git-sources-tbody tr").first).to_contain_text(SEED_URL)
expect(page.locator("#git-sources-tbody tr").last).to_contain_text(ENV_SOURCE_A)
# …each with the mono URL cell, a rendered added date (not the "—"
# env-fallback placeholder), and a labeled per-row Remove button.
for row, url in ((first, SEED_URL), (second, ENV_SOURCE_A)):
added = row.locator("td").nth(1)
expect(added).not_to_be_empty()
expect(added).not_to_have_text("—")
remove = row.locator(".git-source-remove")
expect(remove).to_have_count(1)
expect(remove).to_have_attribute("aria-label", f"Remove git source: {url}")
# DB rows exist → from_env is false: the env note is hidden and the
# empty state too.
expect(page.locator("#git-sources-env-note")).to_be_hidden()
expect(page.locator("#git-sources-empty")).to_be_hidden()
# ---------------------------------------------------------------------------
# 4. Admin: add → duplicate → invalid → remove (accept + cancel) — the
# full never-stale lifecycle
# ---------------------------------------------------------------------------
def test_admin_add_then_remove_lifecycle(
page: Page, app_url: str, db_ready: None
) -> None:
page.set_default_timeout(30_000)
_seed_rows([SEED_URL])
_admin_git_sources_page(page, app_url)
expect(page.locator("#git-sources-tbody tr")).to_have_count(1)
error = page.locator("#git-source-error")
add_btn = page.locator("#git-source-add")
# --- add: 201 → the row appears, the input clears, the button
# re-enables (never stale) ---------------------------------------
page.fill("#git-source-url", NEW_REPO_URL)
add_btn.click()
expect(page.locator("#git-sources-tbody tr")).to_have_count(2, timeout=30_000)
expect(_row(page, NEW_REPO_URL).locator("td.git-source-url-cell code")).to_have_text(
NEW_REPO_URL
)
expect(page.locator("#git-source-url")).to_have_value("")
expect(add_btn).to_be_enabled()
expect(add_btn).to_have_text("Add source")
# A stored row exists → the env note stays hidden.
expect(page.locator("#git-sources-env-note")).to_be_hidden()
# --- duplicate: inline role=alert, NO new row, button re-enabled,
# the (failed) input survives for one edit ------------------------
page.fill("#git-source-url", NEW_REPO_URL)
add_btn.click()
expect(error).to_be_visible(timeout=30_000)
assert error.get_attribute("role") == "alert"
expect(error).to_contain_text("already exists")
expect(page.locator("#git-sources-tbody tr")).to_have_count(2)
expect(add_btn).to_be_enabled()
expect(add_btn).to_have_text("Add source")
expect(page.locator("#git-source-url")).to_have_value(NEW_REPO_URL)
# --- invalid shape: inline 422 (the server detail, never the
# echoed input), NO new row, button re-enabled --------------------
page.fill("#git-source-url", "not a valid url")
add_btn.click()
expect(error).to_be_visible(timeout=30_000)
expect(error).to_contain_text("not a valid git URL")
expect(page.locator("#git-sources-tbody tr")).to_have_count(2)
expect(add_btn).to_be_enabled()
expect(add_btn).to_have_text("Add source")
# --- remove: accept the confirm → the row disappears --------------
dialog_action: dict[str, bool] = {"accept": True}
deletes: list[str] = []
def handle_dialog(dialog: Dialog) -> None:
if dialog_action["accept"]:
dialog.accept()
else:
dialog.dismiss()
def track_delete(r: Any) -> None:
if r.method == "DELETE" and "/api/git-sources/" in r.url:
deletes.append(r.url)
page.on("dialog", handle_dialog)
page.on("request", track_delete)
try:
_row(page, NEW_REPO_URL).locator(".git-source-remove").click()
expect(page.locator("#git-sources-tbody tr")).to_have_count(1, timeout=30_000)
expect(_row(page, NEW_REPO_URL)).to_have_count(0)
# The seed row survived — and exactly one DELETE went out.
expect(_row(page, SEED_URL)).to_have_count(1)
assert len(deletes) == 1, f"expected one DELETE, saw: {deletes}"
# --- remove: cancel the confirm → the row stays, NO DELETE ----
dialog_action["accept"] = False
_row(page, SEED_URL).locator(".git-source-remove").click()
# Wait for the dialog round-trip to settle (dismiss → the JS
# returns without fetching) so the "no second DELETE" claim is
# made on a settled page.
page.wait_for_timeout(500)
assert len(deletes) == 1, f"canceled removal still deleted: {deletes}"
expect(_row(page, SEED_URL)).to_have_count(1)
expect(page.locator("#git-sources-tbody tr")).to_have_count(1)
finally:
page.remove_listener("dialog", handle_dialog)
page.remove_listener("request", track_delete)
# ---------------------------------------------------------------------------
# 5. Admin: empty table → the env rows render + the env note is visible
# (BOR_GIT_SOURCES is the empty-table fallback)
# ---------------------------------------------------------------------------
def test_admin_env_fallback_rows_and_note(
page: Page, app_url: str, db_ready: None
) -> None:
page.set_default_timeout(30_000)
# The autouse fixture truncated git_sources — the table is EMPTY, so
# the module app's BOR_GIT_SOURCES CSV is the effective list.
_admin_git_sources_page(page, app_url)
# The two env rows render (CSV order), each tagged "from .env" with
# no Remove (nothing is stored to remove) and the "—" added date.
expect(page.locator("#git-sources-tbody tr")).to_have_count(2)
expect(page.locator("#git-sources-tbody tr").first).to_contain_text(ENV_SOURCE_A)
expect(page.locator("#git-sources-tbody tr").last).to_contain_text(ENV_SOURCE_B)
for row in page.locator("#git-sources-tbody tr").all():
expect(row.locator(".git-source-env-tag")).to_have_count(1)
expect(row.locator(".git-source-env-tag")).to_have_text("from .env")
expect(row.locator(".git-source-remove")).to_have_count(0)
expect(row.locator("td").nth(1)).to_have_text("—")
# The env-fallback note explains the active list's origin…
expect(page.locator("#git-sources-env-note")).to_be_visible()
# …and the API agrees: from_env true, null ids, the env URLs.
r = page.request.get(f"{app_url}/api/git-sources")
assert r.status == 200, r.text
body = r.json()
assert body["from_env"] is True
assert [s["url"] for s in body["sources"]] == [ENV_SOURCE_A, ENV_SOURCE_B]
assert all(s["id"] is None and s["added_at"] is None for s in body["sources"])
# ---------------------------------------------------------------------------
# 6. UI Structure Check (AGENTS.md rule 5) + no CDN (rule 6)
# ---------------------------------------------------------------------------
def test_admin_page_a11y_and_no_cdn(
page: Page, app_url: str, db_ready: None
) -> None:
page.set_default_timeout(30_000)
_seed_rows([SEED_URL])
_admin_git_sources_page(page, app_url)
expect(page.locator("#git-sources-tbody tr")).to_have_count(1)
# Standard app frame: landmarks + skip link (PLAN §7.2).
expect(page.locator("header.app-header")).to_have_count(1)
expect(page.locator('nav[aria-label="Primary"]')).to_have_count(1)
expect(page.locator("main#main")).to_have_count(1)
expect(page.locator("footer.app-footer")).to_have_count(1)
expect(page.locator(".skip-link")).to_have_count(1)
# No CDN (rule 6): no https:// asset tags; every script/link ref is
# same-origin or a data: URI.
html = page.content()
assert 'src="https://' not in html and 'href="https://' not in html
refs = page.evaluate(
"""() => [...document.querySelectorAll("script[src], link[href]")]
.map((el) => el.src || el.href)"""
)
assert refs, "expected local asset references"
for ref in refs:
assert ref.startswith(app_url) or ref.startswith("data:"), (
f"non-local asset reference: {ref}"
)
# The form input is labeled (visible <label for=…>).
url_input = page.get_by_label("Add a git source")
expect(url_input).to_have_count(1)
# Full-width table (PLAN §7.1 — no skinny list): the table fills the
# 72rem container (≥80%, the Sources-page assertion).
table = page.locator("#git-sources-table")
expect(table).to_be_visible()
table_box = table.bounding_box()
shell_box = page.locator(".git-sources-shell").bounding_box()
assert table_box is not None and shell_box is not None
assert table_box["width"] >= 0.80 * shell_box["width"], (
f"table is {table_box['width']:.0f}px in a {shell_box['width']:.0f}px container"
)
# Touch targets ≥44px (add button + a row's Remove).
for el in (page.locator("#git-source-add"), page.locator(".git-source-remove")):
box = el.bounding_box()
assert box is not None and box["height"] >= 44, f"target too small: {box}"
# :focus-visible draws the 3px outline (the theme contract).
page.focus("#git-source-url")
outline = page.evaluate(
"() => getComputedStyle(document.querySelector('#git-source-url')).outlineWidth"
)
assert outline == "3px", f"focus-visible outline missing: {outline!r}"
# The list updates are announced (aria-live) and the error line is a
# role=alert region; the env note + hint are role=note.
announcer = page.locator("#git-sources-announcer")
assert announcer.get_attribute("role") == "status"
assert announcer.get_attribute("aria-live") == "polite"
assert page.locator("#git-source-error").get_attribute("role") == "alert"
assert page.locator("#git-sources-env-note").get_attribute("role") == "note"
assert page.locator("#git-sources-hint").get_attribute("role") == "note"
+16 -9
View File
@@ -18,15 +18,17 @@ sources, document viewer, global tuning, login): one shared markup block
Per role, the VISIBLE inventory:
* admin: brand + nav [Chat, #nav-sources, #nav-tuning] + #steering-toggle
+ #sync-btn + #new-chat-btn + #sign-out-btn (with #sign-in-link
hidden) — on all five pages, same id+class inventory, same DOM order;
* anonymous: brand + nav [Chat] (#nav-sources / #nav-tuning hidden —
locked A10 UI revision) + #new-chat-btn + #sign-in-link (with
#sync-btn hidden, #sign-out-btn hidden) on all five pages — and the
steering toggle + panel are ABSENT from the DOM (phase 16 "absent,
not hidden" treatment, carried into phase 34 task 01; test_admin_auth
pins it).
* admin: brand + nav [Chat, #nav-sources, #nav-git-sources, #nav-tuning]
(four links, that order — the Git sources link joined in phase 35,
owner permission 2026-08-26) + #steering-toggle + #sync-btn +
#new-chat-btn + #sign-out-btn (with #sign-in-link hidden) — on all
five pages, same id+class inventory, same DOM order;
* anonymous: brand + nav [Chat] (#nav-sources / #nav-git-sources /
#nav-tuning hidden — locked A10 UI revision) + #new-chat-btn +
#sign-in-link (with #sync-btn hidden, #sign-out-btn hidden) on all
five pages — and the steering toggle + panel are ABSENT from the DOM
(phase 16 "absent, not hidden" treatment, carried into phase 34 task
01; test_admin_auth pins it).
Normalization for the inventory comparison: the current-page ``is-active``
nav marker and the sign-in ``?next=`` value legitimately differ per page,
@@ -214,6 +216,9 @@ def _visit(page: Page, app_url: str, name: str, url: str, admin: bool) -> list[s
expect(page.locator(".app-nav a[href='/']")).to_be_visible() # Chat
if admin:
expect(page.locator("#nav-sources")).to_be_visible()
# Phase 35: the fourth admin-only nav link (Git sources) is
# revealed on every page, between Sources and Tuning.
expect(page.locator("#nav-git-sources")).to_be_visible()
expect(page.locator("#nav-tuning")).to_be_visible()
expect(page.locator("#steering-toggle")).to_be_visible()
expect(page.locator("#sync-btn")).to_be_visible()
@@ -223,6 +228,7 @@ def _visit(page: Page, app_url: str, name: str, url: str, admin: bool) -> list[s
# Locked A10 UI revision: admin-only links ship hidden, never
# revealed for anonymous…
expect(page.locator("#nav-sources")).to_be_hidden()
expect(page.locator("#nav-git-sources")).to_be_hidden()
expect(page.locator("#nav-tuning")).to_be_hidden()
expect(page.locator("#sync-btn")).to_be_hidden()
expect(page.locator("#sign-out-btn")).to_be_hidden()
@@ -281,6 +287,7 @@ def _admin_login_page_inventory(page: Page, app_url: str) -> list[str]:
_wait_settled(page, admin=True)
expect(page.locator(".app-nav a[href='/']")).to_be_visible()
expect(page.locator("#nav-sources")).to_be_visible()
expect(page.locator("#nav-git-sources")).to_be_visible()
expect(page.locator("#nav-tuning")).to_be_visible()
expect(page.locator("#steering-toggle")).to_be_visible()
expect(page.locator("#sync-btn")).to_be_visible()
+5 -1
View File
@@ -58,6 +58,7 @@ def test_suggestions_honors_bor_suggestions_env_override(monkeypatch) -> None:
("/document.html", "Brain of Reese"), # phase 10: viewer page
("/login.html", "Sign in"), # phase 16: admin sign-in page
("/tuning.html", "Global Tuning"), # phase 27: global tuning page
("/git-sources.html", "Git sources"), # phase 35: admin git sources page
],
)
def test_html_pages_served_locally_no_cdn(client, path: str, marker: str) -> None:
@@ -94,7 +95,7 @@ def test_index_page_no_cache_with_versioned_asset_refs(client) -> None:
@pytest.mark.parametrize(
"path",
["/sources.html", "/document.html", "/login.html", "/tuning.html"],
["/sources.html", "/document.html", "/login.html", "/tuning.html", "/git-sources.html"],
)
def test_html_pages_no_cache_with_versioned_refs(client, path: str) -> None:
"""Each of the other four pages revalidates and carries at least one
@@ -146,6 +147,7 @@ def test_styles_and_js_served(client) -> None:
assert client.get("/assets/login.js").status_code == 200 # phase 16: login page
assert client.get("/assets/document-modal.js").status_code == 200 # phase 26: modal module
assert client.get("/assets/tuning.js").status_code == 200 # phase 27: tuning page
assert client.get("/assets/git-sources.js").status_code == 200 # phase 35: git sources page
# Emoji code points banned from UI chrome (phase 08): the pictograph
@@ -177,12 +179,14 @@ def _find_emoji(text: str) -> list[str]:
"/document.html",
"/login.html", # phase 16
"/tuning.html", # phase 27
"/git-sources.html", # phase 35
"/assets/app.js",
"/assets/sources.js",
"/assets/markdown.js",
"/assets/document.js",
"/assets/login.js", # phase 16
"/assets/document-modal.js", # phase 26: the document modal module
"/assets/git-sources.js", # phase 35: the git sources page module
"/assets/styles.css",
],
)
+302
View File
@@ -0,0 +1,302 @@
"""Integration: the admin git-sources CRUD API (phase 35, task 02).
Real Postgres (``podman compose up -d db``); the ``BOR_GIT_SOURCES``
fallback is exercised deterministically by monkeypatching the router's
``get_settings`` with a fresh ``Settings(_env_file=None, git_sources=…)``
(same pattern as ``test_sync_api.py`` — the dev ``.env`` never leaks in).
Contract under test:
* anonymous → 403 ``{"detail": "admin only"}`` on GET, POST, and DELETE
(phase 16 pattern, same as ``/api/sync``);
* GET — empty table + env set → the env rows with ``from_env: true`` and
null ``id``/``added_at``; empty table + empty env → ``sources: []``
with ``from_env: true``; any DB rows → ``from_env: false`` and the env
var is ignored (the phase's locked decision); DB rows ordered by
``(added_at, id)``;
* POST — 201 stored trimmed; duplicate (even with different surrounding
whitespace) → 409 with a generic detail that never echoes the URL
(credential safety), including when only the DB unique index catches
it; bad shape / blank / >500 chars → 422, also input-free;
* DELETE — 204 and gone; an emptied table falls back to the env list
again; unknown id → 404.
``git_sources`` is global state: truncated around every test.
"""
from __future__ import annotations
import uuid
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from typing import Any
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import select, text
from sqlalchemy.orm import Session
from app.api import git_sources as git_sources_api
from app.config import Settings
from app.models import GitSource
@pytest.fixture(autouse=True)
def clean_git_sources(db: Session) -> Iterator[None]:
"""The stored list is global state: reset around every test."""
db.execute(text("TRUNCATE git_sources"))
db.commit()
yield
db.execute(text("TRUNCATE git_sources"))
db.commit()
def _settings(git_sources: str = "") -> Settings:
"""Fresh settings with the ``.env`` file ignored; the explicit kwarg
beats any process env leaks (test_sync_api pattern)."""
return Settings(_env_file=None, git_sources=git_sources) # pyright: ignore[reportCallIssue]
# --- anonymous -------------------------------------------------------------
def test_anonymous_gets_403_on_all_routes(client: TestClient, db: Session) -> None:
r = client.get("/api/git-sources")
assert r.status_code == 403
assert r.json() == {"detail": "admin only"}
r = client.post("/api/git-sources", json={"url": "https://anon.example.com/x.git"})
assert r.status_code == 403
assert r.json() == {"detail": "admin only"}
r = client.delete(f"/api/git-sources/{uuid.uuid4()}")
assert r.status_code == 403
assert r.json() == {"detail": "admin only"}
# Nothing landed in the table.
assert db.execute(text("SELECT count(*) FROM git_sources")).scalar_one() == 0
# --- GET: env fallback -----------------------------------------------------
def test_get_empty_table_with_env_returns_env_rows(
admin_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
git_sources_api,
"get_settings",
lambda: _settings("https://a.example.com/one.git, git@b.example.com:two.git ,"),
)
r = admin_client.get("/api/git-sources")
assert r.status_code == 200
body = r.json()
assert body["from_env"] is True
# Whitespace-trimmed, empty entries dropped, order preserved; null ids.
assert body["sources"] == [
{"id": None, "url": "https://a.example.com/one.git", "added_at": None},
{"id": None, "url": "git@b.example.com:two.git", "added_at": None},
]
def test_get_empty_table_with_empty_env_returns_empty_list(
admin_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(git_sources_api, "get_settings", lambda: _settings())
r = admin_client.get("/api/git-sources")
assert r.status_code == 200
assert r.json() == {"sources": [], "from_env": True}
def test_get_orders_db_rows_by_added_at_then_id(admin_client: TestClient, db: Session) -> None:
base = datetime.now(UTC) - timedelta(hours=3)
db.add_all(
[
GitSource(url="https://example.com/oldest.git", added_at=base),
GitSource(url="https://example.com/second.git", added_at=base + timedelta(hours=1)),
GitSource(url="https://example.com/newest.git", added_at=base + timedelta(hours=2)),
# Same transaction → identical server-stamped added_at (≈ now,
# after the explicit rows): the id tie-break decides their order.
GitSource(url="https://example.com/tie-a.git"),
GitSource(url="https://example.com/tie-b.git"),
]
)
db.commit()
tie_rows = db.scalars(
select(GitSource).where(GitSource.url.like("https://example.com/tie-%"))
).all()
tie_ids = sorted(row.id for row in tie_rows)
body = admin_client.get("/api/git-sources").json()
assert body["from_env"] is False
assert [s["url"] for s in body["sources"][:3]] == [
"https://example.com/oldest.git",
"https://example.com/second.git",
"https://example.com/newest.git",
]
assert [s["id"] for s in body["sources"][3:]] == [str(i) for i in tie_ids]
# DB rows carry real ids + timestamps (the env shape has neither).
for s in body["sources"]:
assert s["id"] is not None
assert s["added_at"] is not None
# --- POST: create ----------------------------------------------------------
def test_post_creates_trimmed_and_list_stops_using_env(
admin_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
git_sources_api,
"get_settings",
lambda: _settings("https://env.example.com/env.git"),
)
r = admin_client.post("/api/git-sources", json={"url": " https://new.example.com/repo.git "})
assert r.status_code == 201
body = r.json()
assert body["url"] == "https://new.example.com/repo.git" # trimmed
uuid.UUID(body["id"])
assert body["added_at"] is not None
assert set(body) == {"id", "url", "added_at"}
# The DB row now wins: from_env False, the env URL is gone from the list.
body = admin_client.get("/api/git-sources").json()
assert body["from_env"] is False
assert [s["url"] for s in body["sources"]] == ["https://new.example.com/repo.git"]
def test_post_accepts_accepted_url_shapes(admin_client: TestClient) -> None:
for url in (
"https://github.com/owner/repo.git",
"http://git.local/repo.git",
"ssh://git@example.com/repo.git",
"git@github.com:owner/repo.git",
):
r = admin_client.post("/api/git-sources", json={"url": url})
assert r.status_code == 201, f"{url} must be accepted: {r.text}"
assert r.json()["url"] == url
def test_post_duplicate_url_returns_409_without_echoing_url(
admin_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""The 409 detail is a fixed generic string — URLs may embed
``user:pass@`` credentials (phase 32's masking discipline)."""
monkeypatch.setattr(git_sources_api, "get_settings", lambda: _settings())
url = "https://user:secret@example.com/creds.git"
assert admin_client.post("/api/git-sources", json={"url": url}).status_code == 201
# Same URL with different surrounding whitespace: the trim makes it a
# duplicate too.
r = admin_client.post("/api/git-sources", json={"url": f" {url}\t"})
assert r.status_code == 409
detail = r.json()["detail"]
assert detail == "a git source with this URL already exists"
assert url not in detail
assert "user:secret" not in detail
# Exactly one row stored.
assert len(admin_client.get("/api/git-sources").json()["sources"]) == 1
def test_post_concurrent_insert_backstop_still_409(
admin_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""If the duplicate pre-check misses (a concurrent insert lands
between the check and the commit), the DB unique index still yields
the generic 409 — never a 500."""
monkeypatch.setattr(git_sources_api, "get_settings", lambda: _settings())
url = "https://example.com/backstop.git"
assert admin_client.post("/api/git-sources", json={"url": url}).status_code == 201
real_select = git_sources_api.select
def blind_select(*args: Any, **kwargs: Any) -> Any:
if args and args[0] is GitSource: # the duplicate pre-check
# …now never matches — only the unique index can catch it.
return real_select(GitSource).where(GitSource.url == "zz-never-matches")
return real_select(*args, **kwargs)
monkeypatch.setattr(git_sources_api, "select", blind_select)
r = admin_client.post("/api/git-sources", json={"url": url})
assert r.status_code == 409
detail = r.json()["detail"]
assert detail == "a git source with this URL already exists"
assert url not in detail
def test_post_rejects_invalid_shapes_without_echoing_input(
admin_client: TestClient, db: Session
) -> None:
"""Bad shapes are 422 with a detail that never repeats the submitted
value."""
for bad in ("not a url", "host:repo", "ftp://example.com/x.git"):
r = admin_client.post("/api/git-sources", json={"url": bad})
assert r.status_code == 422, f"{bad!r} must be rejected"
assert bad not in r.text, f"422 detail must not echo the input ({bad!r})"
# Nothing stored.
assert db.execute(text("SELECT count(*) FROM git_sources")).scalar_one() == 0
def test_post_rejects_blank_and_oversized_urls(admin_client: TestClient, db: Session) -> None:
"""Whitespace-only (trim → empty) and >500-char URLs are 422; the
500-char boundary passes."""
assert admin_client.post("/api/git-sources", json={"url": " \t\n "}).status_code == 422
assert admin_client.post("/api/git-sources", json={"url": "x" * 501}).status_code == 422
boundary = "https://" + "x" * 492 # exactly 500 chars
assert len(boundary) == 500
assert admin_client.post("/api/git-sources", json={"url": boundary}).status_code == 201
assert db.execute(text("SELECT count(*) FROM git_sources")).scalar_one() == 1
# --- DB rows win over env ---------------------------------------------------
def test_db_rows_win_over_env(admin_client: TestClient, db: Session, monkeypatch) -> None:
"""Seed a row AND set the env: the GET returns only the DB rows and
``from_env: false`` — the env var is ignored once the table has rows."""
monkeypatch.setattr(
git_sources_api, "get_settings", lambda: _settings("https://env.example.com/env.git")
)
db.add(GitSource(url="https://db.example.com/db.git"))
db.commit()
body = admin_client.get("/api/git-sources").json()
assert body["from_env"] is False
assert [s["url"] for s in body["sources"]] == ["https://db.example.com/db.git"]
for s in body["sources"]:
assert s["id"] is not None
assert s["added_at"] is not None
# --- DELETE -----------------------------------------------------------------
def test_delete_removes_row_and_falls_back_to_env(
admin_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
git_sources_api,
"get_settings",
lambda: _settings("https://env.example.com/env.git"),
)
created = admin_client.post("/api/git-sources", json={"url": "https://new.example.com/x.git"})
assert created.status_code == 201
assert admin_client.delete(f"/api/git-sources/{created.json()['id']}").status_code == 204
# The table is empty again → the env fallback is live once more.
body = admin_client.get("/api/git-sources").json()
assert body["from_env"] is True
assert body["sources"] == [
{"id": None, "url": "https://env.example.com/env.git", "added_at": None}
]
def test_delete_unknown_id_returns_404(admin_client: TestClient) -> None:
r = admin_client.delete(f"/api/git-sources/{uuid.uuid4()}")
assert r.status_code == 404
assert r.json() == {"detail": "git source not found"}
def test_delete_invalid_id_returns_422(admin_client: TestClient) -> None:
assert admin_client.delete("/api/git-sources/not-a-uuid").status_code == 422
+51 -4
View File
@@ -4,9 +4,14 @@ Drives ``scripts.import_docs`` end to end with a fake ``clone_or_pull`` (no
real git, no network) and a recording fake ``import_sources`` (no real
DB), covering:
- ``BOR_GIT_SOURCES`` set → each URL is cloned/pulled into
``BOR_SOURCES_DIR/<repo-name>/`` and exactly those dirs are imported.
- ``--source`` still wins over ``BOR_GIT_SOURCES`` (no git at all).
- Effective git sources set (phase 35: the shared resolver — stubbed
here, keeping this file's no-real-DB style) → each URL is cloned/pulled
into ``BOR_SOURCES_DIR/<repo-name>/`` and exactly those dirs are
imported.
- DB rows win over ``BOR_GIT_SOURCES`` (the resolver's ``db`` origin —
the env list is ignored).
- ``--source`` still wins over git sources (no git at all, no resolver
call).
- No git sources + no ``--source`` → the legacy ``DEFAULT_SOURCES``.
- A failing git sync → exit code 1, an error naming the failing repo on
stderr, and **zero** import attempts.
@@ -91,6 +96,13 @@ def test_resolve_sources_git_urls_cloned_into_sources_dir(
) -> None:
calls, fake = _fake_clone_factory()
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
# Phase 35: resolution goes through the shared resolver (stubbed —
# this file keeps its no-real-DB style); the URLs are the env list.
monkeypatch.setattr(
import_docs,
"effective_git_sources",
lambda db: (["https://host/a/homelab.git", "git@host:user/deploy.git"], "env"),
)
settings = _settings(
git_sources="https://host/a/homelab.git, git@host:user/deploy.git ,",
sources_dir=str(tmp_path / "bor"),
@@ -119,7 +131,34 @@ def test_resolve_sources_cli_source_wins(
assert calls == [] # git is never touched when --source is given
def test_resolve_sources_defaults_when_nothing_configured() -> None:
def test_resolve_sources_db_rows_win_over_env(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""Phase 35: the resolver's ``db`` origin (table has rows) — the
``BOR_GIT_SOURCES`` list must be ignored; only the DB repo is cloned."""
calls, fake = _fake_clone_factory()
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
monkeypatch.setattr(
import_docs,
"effective_git_sources",
lambda db: (["https://db.example/only.git"], "db"),
)
settings = _settings(
git_sources="https://env.example/ignored.git",
sources_dir=str(tmp_path / "bor"),
)
sources = import_docs._resolve_sources(None, settings)
assert sources == [tmp_path / "bor" / "only"]
assert calls == [("https://db.example/only.git", tmp_path / "bor" / "only")]
def test_resolve_sources_defaults_when_nothing_configured(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Both origins empty (the resolver's ``([], "env")``) → legacy dirs.
monkeypatch.setattr(import_docs, "effective_git_sources", lambda db: ([], "env"))
sources = import_docs._resolve_sources(None, _settings())
assert sources == [p.expanduser() for p in import_docs.DEFAULT_SOURCES]
@@ -137,6 +176,11 @@ def test_main_git_sources_clone_then_import(
sources_dir=str(tmp_path / "bor"),
)
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
monkeypatch.setattr(
import_docs,
"effective_git_sources",
lambda db: (["https://host/a/homelab.git", "https://host/a/deploy.git"], "env"),
)
calls, fake = _fake_clone_factory()
monkeypatch.setattr(import_docs, "clone_or_pull", fake)
fake_import = FakeImportSources()
@@ -191,6 +235,9 @@ def test_main_git_failure_aborts_before_import(
sources_dir=str(tmp_path / "bor"),
)
monkeypatch.setattr(import_docs, "get_settings", lambda: settings)
monkeypatch.setattr(
import_docs, "effective_git_sources", lambda db: (["https://host/a/bad.git"], "env")
)
def failing_clone(url: str, dest: Path | str) -> Path:
raise GitSyncError(
+14 -9
View File
@@ -5,12 +5,15 @@ Drives the **real Alembic engine** against the live dev database
``test_migration_0004.py`` (information_schema assertions on the state the
migration must leave):
* upgrade to head → ``kb_overview`` exists with exactly the three columns
The tests target revision ``0005`` explicitly so later migrations
(0006, …) cannot break them.
* upgrade to 0005 → ``kb_overview`` exists with exactly the three columns
the phase locks in (``id INTEGER PK`` default 1, ``content TEXT NOT NULL``
default ``''``, ``updated_at TIMESTAMPTZ NOT NULL`` default ``now()``),
and a bare insert lands the single-row defaults (id=1, content='');
* downgrade to 0004 → the table is gone;
* upgrade to head again → it is back (round-trip).
* upgrade to 0005 again → it is back (round-trip).
The ``alembic`` fixture guarantees the DB ends at head even if a test
fails or the process is interrupted.
@@ -65,14 +68,14 @@ def _version(db: Session) -> str | None:
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
def test_upgrade_to_head_creates_kb_overview(db: Session, alembic: Config) -> None:
"""Upgrade to head: the single-row table exists with the locked
def test_upgrade_to_0005_creates_kb_overview(db: Session, alembic: Config) -> None:
"""Upgrade 0004 → 0005: the single-row table exists with the locked
column types, nullability, and server defaults."""
command.downgrade(alembic, "0004") # start from the pre-0005 state
assert _version(db) == "0004"
command.upgrade(alembic, "head")
assert _version(db) == "0005", "alembic_version must be at 0005 (head)"
command.upgrade(alembic, "0005")
assert _version(db) == "0005", "alembic_version must be at 0005"
pk = db.execute(
text(
@@ -139,9 +142,11 @@ def test_downgrade_to_0004_drops_table(db: Session, alembic: Config) -> None:
def test_upgrade_round_trip_restores_table(db: Session, alembic: Config) -> None:
"""Upgrade back to head after the downgrade: table + defaults are back."""
command.upgrade(alembic, "head")
assert _version(db) == "0005", "round-trip upgrade must land at 0005 (head)"
"""Downgrade to 0004, then upgrade back to 0005: table + defaults are
back (self-contained — does not rely on a prior test's downgrade)."""
command.downgrade(alembic, "0004")
command.upgrade(alembic, "0005")
assert _version(db) == "0005", "round-trip upgrade must land at 0005"
id_col = _column(db, "id")
assert id_col is not None and id_col[2] == "1", "kb_overview.id must be back with default 1"
+194
View File
@@ -0,0 +1,194 @@
"""Integration: migration 0006 (git_sources) schema contract.
Drives the **real Alembic engine** against the live dev database
(``podman compose up -d db``), mirroring the style of
``test_migration_0004.py`` / ``test_migration_0005.py`` (information_schema
assertions on the state the migration must leave). The tests target
revision ``0006`` explicitly so later migrations (0007, …) cannot break
them:
* upgrade 0005 → 0006 → ``git_sources`` exists with exactly the three
columns the phase locks in (``id UUID PK``, ``url TEXT NOT NULL``,
``added_at TIMESTAMPTZ NOT NULL`` default ``now()``) plus the unique
index ``uq_git_sources_url`` (duplicate URLs rejected with an
IntegrityError);
* an insert without ``added_at`` gets the server-stamped default;
* downgrade to 0005 → the table (and its index) is gone;
* upgrade back to 0006 → it is back (round-trip).
The ``alembic`` fixture guarantees the DB ends at head even if a test
fails or the process is interrupted.
"""
from __future__ import annotations
from collections.abc import Iterator
from typing import Any
import pytest
from alembic.config import Config
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from alembic import command
from app.db import db_available
@pytest.fixture()
def alembic(db: Session) -> Iterator[Config]:
"""Real Alembic config bound to the dev DB (URL from app settings).
Starts at head (repairs an interrupted earlier run); teardown upgrades
to head no matter what happened, so the dev DB is never left below
head.
"""
if not db_available():
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
cfg.set_main_option("script_location", "alembic")
command.upgrade(cfg, "head")
try:
yield cfg
finally:
command.upgrade(cfg, "head")
def _column(db: Session, column: str) -> tuple[Any, ...] | None:
"""(data_type, is_nullable, column_default) for one git_sources column."""
row = db.execute(
text(
"SELECT data_type, is_nullable, column_default"
" FROM information_schema.columns"
" WHERE table_name = 'git_sources' AND column_name = :c"
),
{"c": column},
).fetchone()
return tuple(row) if row is not None else None
def _version(db: Session) -> str | None:
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
def _unique_url_index(db: Session) -> int:
"""1 iff ``uq_git_sources_url`` exists as a UNIQUE index."""
count: Any = db.execute(
text(
"SELECT count(*) FROM pg_indexes"
" WHERE tablename = 'git_sources' AND indexname = 'uq_git_sources_url'"
" AND indexdef ILIKE 'CREATE UNIQUE%'"
)
).scalar()
assert count is not None, "pg_indexes count must be an int"
return int(count)
def _insert_url(db: Session, url: str) -> None:
db.execute(
text("INSERT INTO git_sources (id, url) VALUES (gen_random_uuid(), :u)"),
{"u": url},
)
db.commit()
def test_upgrade_to_0006_creates_git_sources(db: Session, alembic: Config) -> None:
"""Upgrade 0005 → 0006: the table exists with the locked column types,
nullability, server default, primary key, and unique URL index."""
command.downgrade(alembic, "0005") # start from the pre-0006 state
assert _version(db) == "0005"
command.upgrade(alembic, "0006")
assert _version(db) == "0006", "alembic_version must be at 0006"
pk = db.execute(
text(
"SELECT column_name FROM information_schema.table_constraints tc"
" JOIN information_schema.key_column_usage kcu"
" ON tc.constraint_name = kcu.constraint_name"
" WHERE tc.table_name = 'git_sources' AND tc.constraint_type = 'PRIMARY KEY'"
)
).scalar()
assert pk == "id", "git_sources primary key must be id"
id_col = _column(db, "id")
assert id_col is not None, "git_sources.id is missing"
assert id_col[0] == "uuid", "git_sources.id must be UUID"
assert id_col[1] == "NO", "git_sources.id must be NOT NULL"
url = _column(db, "url")
assert url is not None, "git_sources.url is missing"
assert url[0] == "text", "git_sources.url must be TEXT"
assert url[1] == "NO", "git_sources.url must be NOT NULL"
added = _column(db, "added_at")
assert added is not None, "git_sources.added_at is missing"
assert added[0] == "timestamp with time zone", "git_sources.added_at must be TIMESTAMPTZ"
assert added[1] == "NO", "git_sources.added_at must be NOT NULL"
assert added[2] is not None and "now()" in added[2], (
"git_sources.added_at must have server default now()"
)
assert _unique_url_index(db) == 1, "uq_git_sources_url unique index is missing"
def test_added_at_defaults_to_now(db: Session, alembic: Config) -> None:
"""An insert without ``added_at`` gets the server-stamped default —
the API layer never sets it itself (phase 35 task 02)."""
command.upgrade(alembic, "head")
try:
_insert_url(db, "https://git.example.com/mig-test/added-at.git")
stamped = db.execute(
text(
"SELECT added_at IS NOT NULL FROM git_sources"
" WHERE url = 'https://git.example.com/mig-test/added-at.git'"
)
).scalar()
assert stamped is True, "git_sources.added_at must be stamped by the server"
finally:
db.execute(
text(
"DELETE FROM git_sources"
" WHERE url = 'https://git.example.com/mig-test/added-at.git'"
)
)
db.commit()
def test_duplicate_url_rejected(db: Session, alembic: Config) -> None:
"""The unique index is what the API's 409 relies on: a duplicate URL
raises IntegrityError."""
command.upgrade(alembic, "head")
try:
_insert_url(db, "https://git.example.com/mig-test/dup.git")
with pytest.raises(IntegrityError):
_insert_url(db, "https://git.example.com/mig-test/dup.git")
finally:
db.rollback() # the IntegrityError aborts the open transaction
db.execute(
text("DELETE FROM git_sources WHERE url = 'https://git.example.com/mig-test/dup.git'")
)
db.commit()
def test_downgrade_to_0005_drops_table(db: Session, alembic: Config) -> None:
"""Downgrade to 0005: the table (and its index) is dropped (A13 —
reversible)."""
command.downgrade(alembic, "0005")
assert _version(db) == "0005"
exists = db.execute(text("SELECT to_regclass('public.git_sources') IS NOT NULL")).scalar()
assert exists is False, "git_sources must be dropped by the downgrade"
def test_upgrade_round_trip_restores_table(db: Session, alembic: Config) -> None:
"""Downgrade to 0005, then upgrade back to 0006: table, defaults, and
the unique index are back."""
command.downgrade(alembic, "0005")
command.upgrade(alembic, "0006")
assert _version(db) == "0006", "round-trip upgrade must land at 0006"
added = _column(db, "added_at")
assert added is not None and added[2] is not None and "now()" in added[2], (
"git_sources.added_at must keep its now() default after the round-trip"
)
assert _unique_url_index(db) == 1, "uq_git_sources_url must be back after the round-trip"
+149 -25
View File
@@ -1,17 +1,26 @@
"""Integration: the admin sources-sync API (phase 32, task 01).
"""Integration: the admin sources-sync API (phase 32, task 01; phase 35,
task 03 re-points the URL resolution at the shared resolver).
Covers the in-process sync runner end to end over HTTP: anonymous 403s
on both endpoints; admin idle → 202 → ``success`` with the full
ImportSummary detail; 409 on a double trigger while a run is in flight;
``GitSyncError`` → ``failed`` with the failing repo named and **zero**
import attempts; empty ``BOR_GIT_SOURCES`` → ``failed`` loudly; an
embedding failure → ``failed`` with any credentials masked; the import
always runs with ``prune=True``; and the phase-31 overview trigger is
change-gated (no ``lite`` call on an unchanged KB).
import attempts; empty on *both* origins (``git_sources`` table +
``BOR_GIT_SOURCES``) → ``failed`` loudly; an embedding failure →
``failed`` with any credentials masked; the import always runs with
``prune=True``; and the phase-31 overview trigger is change-gated (no
``lite`` call on an unchanged KB).
Phase 35: the runner resolves the repos through
:func:`app.rag.git_sources.effective_git_sources` — the **real**
resolver against the **real** ``git_sources`` table (truncated around
every test), so DB-over-env and the env fallback go through the actual
indirection; the env list is driven by a fresh ``Settings`` on the
resolver's module (the dev ``.env`` never leaks in).
The git / import / overview layers are monkeypatched in ``app.api.sync``
(same fake style as ``test_import_docs_git.py``) — no real git, no real
DB, no LLM: the runner's state machine and HTTP surface are under test.
(same fake style as ``test_import_docs_git.py``) — no real git, no LLM:
the runner's state machine and HTTP surface are under test.
The admin client is used **as a context manager** on purpose: the
background sync task lives on the app's event loop, so the loop must
@@ -22,6 +31,7 @@ request and would cancel the task on request exit.)
from __future__ import annotations
import asyncio
import logging
import time
from collections.abc import Iterator
from datetime import datetime
@@ -29,11 +39,14 @@ from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.api import sync as sync_api
from app.config import Settings
from app.main import app as fastapi_app
from app.models import GitSource
from app.rag import git_sources as git_sources_resolver
from app.rag.importer import ImportSummary
from app.rag.llm import EmbeddingError, LLMClient
from scripts.git_sync import GitSyncError
@@ -52,6 +65,18 @@ def _fresh_sync_state() -> Iterator[None]:
sync_api._task = None
@pytest.fixture(autouse=True)
def clean_git_sources(db: Session) -> Iterator[None]:
"""Phase 35: the runner resolves through the real ``git_sources``
table — global state, truncated around every test (the ``db``
fixture skips the file when Postgres is down)."""
db.execute(text("TRUNCATE git_sources"))
db.commit()
yield
db.execute(text("TRUNCATE git_sources"))
db.commit()
@pytest.fixture()
def sync_client() -> Iterator[TestClient]:
"""Context-managed TestClient — one app event loop across requests
@@ -60,9 +85,29 @@ def sync_client() -> Iterator[TestClient]:
yield client
def _settings(git_sources: str = "", sources_dir: str = "~/bor-sources") -> Settings:
"""Fresh settings (no .env file); explicit kwargs beat any env leaks."""
return Settings(_env_file=None, git_sources=git_sources, sources_dir=sources_dir) # pyright: ignore[reportCallIssue]
def _settings(sources_dir: str = "~/bor-sources") -> Settings:
"""Fresh settings (no .env file); explicit kwargs beat any env leaks.
(The ``git_sources`` kwarg is gone with phase 35 — the runner reads
the URLs from the resolver, not from its own settings; the env list
is stubbed on the resolver's module via :func:`_stub_env`.)
"""
return Settings(_env_file=None, sources_dir=sources_dir) # pyright: ignore[reportCallIssue]
def _stub_env(monkeypatch: pytest.MonkeyPatch, git_sources: str = "") -> None:
"""The resolver's env fallback, driven by a fresh ``Settings`` (the
dev ``.env`` never leaks in — the task-02 pattern)."""
monkeypatch.setattr(
git_sources_resolver,
"get_settings",
lambda: Settings(_env_file=None, git_sources=git_sources), # pyright: ignore[reportCallIssue]
)
def _seed(db: Session, url: str) -> None:
db.add(GitSource(url=url))
db.commit()
def _login(client: TestClient) -> None:
@@ -157,13 +202,15 @@ def test_anonymous_gets_403_on_both_endpoints(client: TestClient) -> None:
def test_admin_sync_success_reports_full_detail(
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
repo_url = f"file://{tmp_path / 'repo.git'}"
_seed(db, repo_url) # the phase-35 resolver picks the DB row up
_stub_env(monkeypatch) # env must not matter once the table has a row
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: _settings(git_sources=repo_url, sources_dir=str(tmp_path / "bor")),
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
clone_calls, fake_clone = _fake_clone()
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
@@ -214,14 +261,16 @@ def test_admin_sync_success_reports_full_detail(
def test_unchanged_kb_skips_overview_refresh(
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
"""Phase-31 trigger is change-gated: added + updated == 0 → no ``lite`` call."""
repo_url = f"file://{tmp_path / 'repo.git'}"
_seed(db, repo_url)
_stub_env(monkeypatch)
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: _settings(git_sources=repo_url, sources_dir=str(tmp_path / "bor")),
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
_, fake_clone = _fake_clone()
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
@@ -244,13 +293,15 @@ def test_unchanged_kb_skips_overview_refresh(
def test_double_trigger_while_running_returns_409(
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
repo_url = f"file://{tmp_path / 'repo.git'}"
_seed(db, repo_url)
_stub_env(monkeypatch)
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: _settings(git_sources=repo_url, sources_dir=str(tmp_path / "bor")),
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
_, fake_clone = _fake_clone()
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
@@ -282,13 +333,15 @@ def test_double_trigger_while_running_returns_409(
def test_git_failure_marks_failed_and_skips_import(
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
repo_url = f"file://{tmp_path / 'bad.git'}"
_seed(db, repo_url)
_stub_env(monkeypatch)
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: _settings(git_sources=repo_url, sources_dir=str(tmp_path / "bor")),
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
def failing_clone(url: str, dest: Path | str) -> Path:
@@ -322,11 +375,14 @@ def test_git_failure_marks_failed_and_skips_import(
def test_no_git_sources_configured_fails_loudly(
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""Both origins empty (the truncate fixture + a blank env) → the
fail-loud error names *both* (phase 35)."""
# Whitespace-only is just as unconfigured as empty.
_stub_env(monkeypatch, " , ")
monkeypatch.setattr(
sync_api,
"get_settings",
# Whitespace-only is just as unconfigured as empty.
lambda: _settings(git_sources=" , ", sources_dir=str(tmp_path / "bor")),
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
clone_calls, fake_clone = _fake_clone()
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
@@ -337,19 +393,87 @@ def test_no_git_sources_configured_fails_loudly(
assert sync_client.post("/api/sync").status_code == 202
body = _poll(sync_client, "failed")
assert body["error"] == "no git sources configured (BOR_GIT_SOURCES)"
assert body["error"] == (
"no git sources configured (git_sources table empty and BOR_GIT_SOURCES unset)"
)
assert clone_calls == [] # git is never touched
assert fake_import.sources == []
def test_import_error_is_reported_with_credentials_masked(
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
def test_db_rows_win_over_env(
sync_client: TestClient,
monkeypatch: pytest.MonkeyPatch,
db: Session,
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
) -> None:
repo_url = f"file://{tmp_path / 'repo.git'}"
"""Phase 35: a stored row beats ``BOR_GIT_SOURCES`` — only the DB URL
is cloned, and the started log names the origin."""
db_url = "https://db.example.com/managed.git"
_seed(db, db_url)
_stub_env(monkeypatch, "https://env.example.com/ignored.git")
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: _settings(git_sources=repo_url, sources_dir=str(tmp_path / "bor")),
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
clone_calls, fake_clone = _fake_clone()
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
fake_import = FakeImportSources(ImportSummary(files=1, added=1))
monkeypatch.setattr(sync_api, "import_sources", fake_import)
monkeypatch.setattr(sync_api, "regenerate_overview", FakeOverview(ok=True))
_login(sync_client)
with caplog.at_level(logging.INFO, logger="app.api.sync"):
assert sync_client.post("/api/sync").status_code == 202
body = _poll(sync_client, "success")
assert clone_calls == [(db_url, tmp_path / "bor" / "managed")]
assert fake_import.sources == [[tmp_path / "bor" / "managed"]]
assert "env.example.com" not in str(body) # the env URL never reaches the UI
assert any("sync: started repos=1 origin=db" in r.getMessage() for r in caplog.records)
def test_env_fallback_when_table_empty(
sync_client: TestClient,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
) -> None:
"""Phase 35: with the table empty (the truncate fixture), the
``BOR_GIT_SOURCES`` list is what gets cloned — origin ``env``."""
env_url = "https://env.example.com/fallback.git"
_stub_env(monkeypatch, env_url)
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
clone_calls, fake_clone = _fake_clone()
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
fake_import = FakeImportSources(ImportSummary(files=1, added=1))
monkeypatch.setattr(sync_api, "import_sources", fake_import)
monkeypatch.setattr(sync_api, "regenerate_overview", FakeOverview(ok=True))
_login(sync_client)
with caplog.at_level(logging.INFO, logger="app.api.sync"):
assert sync_client.post("/api/sync").status_code == 202
_poll(sync_client, "success")
assert clone_calls == [(env_url, tmp_path / "bor" / "fallback")]
assert any("sync: started repos=1 origin=env" in r.getMessage() for r in caplog.records)
def test_import_error_is_reported_with_credentials_masked(
sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
) -> None:
repo_url = f"file://{tmp_path / 'repo.git'}"
_seed(db, repo_url)
_stub_env(monkeypatch)
monkeypatch.setattr(
sync_api,
"get_settings",
lambda: _settings(sources_dir=str(tmp_path / "bor")),
)
_, fake_clone = _fake_clone()
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+110
View File
@@ -0,0 +1,110 @@
"""Unit: the shared git-source resolver (phase 35, task 03).
``effective_git_sources`` is driven with a stubbed session (no Postgres)
and a fresh ``Settings(_env_file=None)`` env (monkeypatched into the
resolver's module — the dev ``.env`` never leaks in, same pattern as the
integration suites): DB rows win in ``(added_at, id)`` order (env
ignored), the ``BOR_GIT_SOURCES`` list is a fallback only while the
table is empty (phase-28 CSV parse reused), and both-empty yields
``([], "env")`` so the callers keep their fail-loud behavior.
"""
from __future__ import annotations
import re
from typing import Any
import pytest
from app.config import Settings
from app.models import GitSource
from app.rag import git_sources as resolver
class _FakeScalars:
"""The ``.scalars(stmt).all()`` tail of the resolver's query."""
def __init__(self, rows: list[GitSource]) -> None:
self._rows = rows
def all(self) -> list[GitSource]:
return self._rows
class _FakeSession:
"""Just enough of a SQLAlchemy session for the resolver (test_steering
pattern). Records the statement so the ordering can be asserted."""
def __init__(self, rows: list[GitSource]) -> None:
self._rows = rows
self.statements: list[Any] = []
def scalars(self, stmt: Any) -> _FakeScalars:
self.statements.append(stmt)
return _FakeScalars(self._rows)
def _stub_env(monkeypatch: pytest.MonkeyPatch, git_sources: str) -> None:
"""Point the resolver's env fallback at a fresh Settings (no .env)."""
monkeypatch.setattr(
resolver,
"get_settings",
lambda: Settings(_env_file=None, git_sources=git_sources), # pyright: ignore[reportCallIssue]
)
# --- the three branches ----------------------------------------------------
def test_db_rows_win_over_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Seeded table + env set → the DB list, origin ``db`` (env ignored)."""
_stub_env(monkeypatch, "https://env.example/ignored.git")
session = _FakeSession([GitSource(url="https://db.example/a.git"), GitSource(url="https://db.example/b.git")])
urls, origin = resolver.effective_git_sources(session) # pyright: ignore[reportArgumentType]
assert urls == ["https://db.example/a.git", "https://db.example/b.git"]
assert origin == "db"
def test_env_fallback_while_table_empty(monkeypatch: pytest.MonkeyPatch) -> None:
"""Empty table + env set → the env list, origin ``env``.
The CSV parse is ``Settings.git_source_list`` itself (phase 28):
whitespace-trimmed, empty entries dropped, order preserved.
"""
_stub_env(monkeypatch, " https://env.example/one.git , ,git@env.example:two.git ")
session = _FakeSession([])
urls, origin = resolver.effective_git_sources(session) # pyright: ignore[reportArgumentType]
assert urls == ["https://env.example/one.git", "git@env.example:two.git"]
assert origin == "env"
def test_both_empty_returns_empty_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Empty table + unset env → ``([], "env")`` — the callers fail loudly."""
_stub_env(monkeypatch, " , ") # whitespace-only is just as unconfigured as empty
session = _FakeSession([])
urls, origin = resolver.effective_git_sources(session) # pyright: ignore[reportArgumentType]
assert urls == []
assert origin == "env"
# --- the ordering ----------------------------------------------------------
def test_db_rows_ordered_by_added_at_then_id(monkeypatch: pytest.MonkeyPatch) -> None:
"""The statement orders by ``(added_at, id)`` — oldest first, the id
tie-break deciding same-timestamp inserts (matches the API's GET)."""
_stub_env(monkeypatch, "")
session = _FakeSession([GitSource(url="https://db.example/a.git")])
resolver.effective_git_sources(session) # pyright: ignore[reportArgumentType]
assert len(session.statements) == 1
compiled = str(session.statements[0].compile(compile_kwargs={"literal_binds": True}))
assert re.search(
r"ORDER BY\s+git_sources\.added_at ASC,\s+git_sources\.id ASC", compiled
), compiled