From 1925bb66a86a526e652196de56a885fd0631e8b8 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Wed, 26 Aug 2026 18:34:32 -0400 Subject: [PATCH] feat(sources): admin page to add and remove git sources (TODO.md L4) --- .env.example | 15 +- Containerfile | 3 +- README.md | 72 ++- alembic/versions/0006_git_sources.py | 48 ++ app/api/git_sources.py | 132 +++++ app/api/sync.py | 33 +- app/core/caching.py | 6 +- app/main.py | 2 + app/models.py | 18 + app/rag/git_sources.py | 44 ++ app/schemas.py | 43 ++ frontend/assets/git-sources.js | 320 ++++++++++++ frontend/assets/header.js | 10 +- frontend/assets/styles.css | 285 ++++++++++- frontend/document.html | 5 + frontend/git-sources.html | 229 +++++++++ frontend/index.html | 5 + frontend/login.html | 7 + frontend/sources.html | 5 + frontend/tuning.html | 5 + scripts/import_docs.py | 35 +- tests/e2e/test_git_sources_admin.py | 572 ++++++++++++++++++++++ tests/e2e/test_nav_consistency.py | 25 +- tests/integration/test_api.py | 6 +- tests/integration/test_git_sources_api.py | 302 ++++++++++++ tests/integration/test_import_docs_git.py | 55 ++- tests/integration/test_migration_0005.py | 23 +- tests/integration/test_migration_0006.py | 194 ++++++++ tests/integration/test_sync_api.py | 174 ++++++- tests/unit/test_git_sources.py | 110 +++++ 30 files changed, 2673 insertions(+), 110 deletions(-) create mode 100644 alembic/versions/0006_git_sources.py create mode 100644 app/api/git_sources.py create mode 100644 app/rag/git_sources.py create mode 100644 frontend/assets/git-sources.js create mode 100644 frontend/git-sources.html create mode 100644 tests/e2e/test_git_sources_admin.py create mode 100644 tests/integration/test_git_sources_api.py create mode 100644 tests/integration/test_migration_0006.py create mode 100644 tests/unit/test_git_sources.py diff --git a/.env.example b/.env.example index d5bd7f1..930c0c1 100644 --- a/.env.example +++ b/.env.example @@ -40,12 +40,19 @@ BOR_RRF_K=60 # Reciprocal Rank Fusion damping constant # BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py # BOR_SUGGESTIONS=["How is my Kubernetes cluster set up?"] # JSON list of onboarding chips -# --- Import sources (git; phase 28) --- +# --- Import sources (git; phase 28, admin-managed since phase 35) --- # Comma-separated git repo URLs; import_docs clones each (first run) or # pulls it (subsequent runs) into BOR_SOURCES_DIR// and indexes -# the result. Empty = no git sources (import_docs falls back to --source / -# the old ~/Homelab + ~/Deployments defaults). Auth via URL (e.g. an -# https token) or SSH keys. +# the result. Auth via URL (e.g. an https token) or SSH keys. +# +# Phase 35 (owner permission 2026-08-26): the PRIMARY way to manage the +# list is the admin Git sources page (http://localhost:8000/git-sources.html) +# — rows stored in Postgres (git_sources table, migration 0006). This +# variable is the EMPTY-TABLE FALLBACK: it only applies while the admin +# list is empty; once the page has stored any source, this variable is +# ignored (the page is the source of truth). Empty table + empty variable +# = no git sources (import_docs falls back to --source / the old +# ~/Homelab + ~/Deployments defaults; the UI Sync button fails loudly). # BOR_GIT_SOURCES=https://github.com/user/homelab.git,https://github.com/user/deployments.git # BOR_SOURCES_DIR=~/bor-sources diff --git a/Containerfile b/Containerfile index a672e7d..543223e 100644 --- a/Containerfile +++ b/Containerfile @@ -19,9 +19,10 @@ RUN mkdir -p /out/assets \ && esbuild ./assets/document.js --bundle --minify --format=esm --target=es2022 --outfile=/out/assets/document.js \ && esbuild ./assets/login.js --bundle --minify --format=esm --target=es2022 --outfile=/out/assets/login.js \ && esbuild ./assets/tuning.js --bundle --minify --format=esm --target=es2022 --outfile=/out/assets/tuning.js \ + && esbuild ./assets/git-sources.js --bundle --minify --format=esm --target=es2022 --outfile=/out/assets/git-sources.js \ && esbuild ./assets/markdown.js --minify --outfile=/out/assets/markdown.js \ && esbuild ./assets/styles.css --minify --outfile=/out/assets/styles.css \ - && cp ./index.html ./sources.html ./document.html ./login.html ./tuning.html /out/ + && cp ./index.html ./sources.html ./document.html ./login.html ./tuning.html ./git-sources.html /out/ # ---------- Stage 2: python dependencies ---------- FROM docker.io/python:3.12-slim AS python diff --git a/README.md b/README.md index fed4447..ce8d79a 100644 --- a/README.md +++ b/README.md @@ -65,16 +65,21 @@ uv run python -m scripts.import_docs # import the configured sources (below) Two source modes: -- **Git sources (recommended)** — set `BOR_GIT_SOURCES` in `.env` to a - comma-separated list of git repo URLs. `import_docs` clones each repo +- **Git sources (recommended)** — the list is managed on the **admin + Git sources page** (`/git-sources.html`) and stored in Postgres (see + [Git-based sources](#git-based-sources)). `import_docs` clones each repo (first run) or pulls it (subsequent runs) into `BOR_SOURCES_DIR//` (default `~/bor-sources`) and indexes the - checkouts — see [Git-based sources](#git-based-sources). + checkouts. While the stored list is empty, the `BOR_GIT_SOURCES` variable + in `.env` is the fallback — the moment the page stores a source, the + variable is ignored. - **Manual directories** — `--source ` (repeatable) imports local - directories directly and *always wins* over `BOR_GIT_SOURCES`. -- If neither is set, the import falls back to the **previous** default, + directories directly and *always wins* over the git sources (stored list + or env). +- If neither is set (stored list, `--source`, and `BOR_GIT_SOURCES` all + empty), the import falls back to the **previous** default, `~/Homelab` + `~/Deployments` — kept only for backwards compatibility, - now replaced by `BOR_GIT_SOURCES`. + now replaced by the git sources list. ### 6. Run the app ```bash @@ -102,6 +107,12 @@ uv run uvicorn app.main:app --reload new tab). **Admin-only** — anonymous visitors see a sign-in gate instead (the catalog is what the login locks; the document viewer itself stays open to everyone). +- **Git sources** (`/git-sources.html`) — the list of git repositories the + **Sync sources** button clones and indexes; **admin-only** (the same + sign-in gate as Sources). Add or remove repositories here — no `.env` + editing, no restart. Adding/removing does not clone or prune on its + own: the Sync button performs that, and a removed repository's documents + leave the index on the next sync. ## Thinking @@ -262,26 +273,38 @@ embedded. Rather than pointing the import at local folders, point it at **git repositories** — the notes live in the repos and `import_docs` keeps local -checkouts of them up to date for you: +checkouts of them up to date for you. + +**Where the list lives (phase 35):** the primary management surface is +the **admin Git sources page** (`/git-sources.html`) — add or remove +repositories there and the list is stored in Postgres (the `git_sources` +table). `BOR_GIT_SOURCES` in `.env` is the **empty-table fallback**: it +only applies while the stored list is empty, and is ignored once the page +has any row (the page becomes the source of truth — no `.env` editing, no +restart needed afterwards). ```env -# .env +# .env — the fallback list (fresh setups, or until the admin page +# stores a source; phase 35 demotes this variable, it does not remove it) BOR_GIT_SOURCES=https://git.reeseapps.com/reese/homelab.git,git@github.com:reese/deployments.git BOR_SOURCES_DIR=~/bor-sources # default; each repo lands in // ``` -- `BOR_GIT_SOURCES` is a **comma-separated list** of URLs. Auth is whatever - the machine supplies — `https://…` via the OS credential helper, or - `git@host:repo.git` via your SSH key; no credentials are stored in the - app or `.env`. +- The effective list (stored rows, else `BOR_GIT_SOURCES` while the stored + list is empty) is a set of git repo URLs. Auth is whatever the machine + supplies — `https://…` via the OS credential helper, or `git@host:repo.git` + via your SSH key; no credentials are stored in the app or `.env`. + Stored URLs are shape-validated on the page (`https://`, `ssh://`, + `git@…` — scp-style `host:repo` is rejected). - Every run **clones** each repo (first time, shallow `--depth 1`) or **pulls** it (`git pull --ff-only` — fast-forward only, so a diverged or broken checkout fails loudly instead of merging) into `BOR_SOURCES_DIR//`, then indexes the checkouts exactly like any local directory (A9 format filter, hidden-dir skip, sha256 delta). `documents.source` is the repo directory name (e.g. `homelab`). -- **`--source ` overrides**: when the flag is given, - `BOR_GIT_SOURCES` is ignored and the manual directory(ies) are imported. +- **`--source ` overrides**: when the flag is given, the git + sources (stored list *and* `BOR_GIT_SOURCES`) are ignored and the + manual directory(ies) are imported. - **A failed sync aborts the run**: if any repo cannot be cloned/pulled, `import_docs` exits non-zero naming the failing repo and imports **nothing** (no partial junk). Fix the URL/connectivity and re-run — the @@ -293,9 +316,10 @@ The **Sync sources** button on the **Sources** page — visible to the **admin only** (anonymous visitors never see it) — runs the whole git-source refresh in one click, in-process: -1. **clone/pull** every `BOR_GIT_SOURCES` repo (the same - `clone_or_pull` the CLI uses — shallow clone on first run, - `git pull --ff-only` afterwards); +1. **clone/pull** every configured git source — the admin-managed list + (the `git_sources` table; `BOR_GIT_SOURCES` only while that list is + empty), through the same `clone_or_pull` the CLI uses (shallow clone + on first run, `git pull --ff-only` afterwards); 2. **re-import with prune** — the `--prune` equivalent, so files deleted upstream leave the index (the button is the canonical "mirror the repos" action); the sha256 delta still skips unchanged files, so an @@ -304,10 +328,12 @@ git-source refresh in one click, in-process: chat turn injects) — but only when the import actually changed the knowledge base. -- **Prerequisites:** `BOR_GIT_SOURCES` must be set — an unset/empty list - fails the sync loudly ("no git sources configured"), because the button - targets the git repos only (manual `--source` directories have no repo - to clone) — and `git` must be on the app's `PATH`. +- **Prerequisites:** at least one git source must be configured — a row + on the admin Git sources page, or `BOR_GIT_SOURCES` in `.env` while the + stored list is empty; **both** empty fails the sync loudly ("no git + sources configured"), because the button targets the git repos only + (manual `--source` directories have no repo to clone) — and `git` must + be on the app's `PATH`. - **States:** clicking starts the run (`202`) and the button goes disabled with **Syncing…** (spinning icon) while the page polls `GET /api/sync/status` every 2 s. There is deliberately **no @@ -534,8 +560,8 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`. | `BOR_HYBRID_LEXICAL_CANDIDATES` | `30` | FTS list width for the RRF fusion | | `BOR_RRF_K` | `60` | RRF damping constant (`1/(k + rank)`) | | `BOR_IMPORT_EXTENSIONS` | `md,markdown,txt,yaml,yml,json,py` | csv of importable formats (may only narrow the A9 set) | -| `BOR_GIT_SOURCES` | — (empty) | csv of git repo URLs; `import_docs` clones/pulls them into `BOR_SOURCES_DIR` and indexes the checkouts (see *Git-based sources*) | -| `BOR_SOURCES_DIR` | `~/bor-sources` | where the `BOR_GIT_SOURCES` repos are cloned/pulled (one subdirectory per repo) | +| `BOR_GIT_SOURCES` | — (empty) | csv of git repo URLs — **fallback while the admin Git sources page's list (Postgres `git_sources`) is empty**; the page is the primary management surface (see *Git-based sources*) | +| `BOR_SOURCES_DIR` | `~/bor-sources` | where the git source repos are cloned/pulled (one subdirectory per repo) | | `BOR_STEERING_MAX_CHARS` | `8000` | char budget for the `` (steering notes) prompt section | | `BOR_SUMMARY_MAX_CHARS` | `12000` | cap on document content sent to the `lite` summary model at import (see *Document summaries*) | | `BOR_KB_OVERVIEW_MAX_CHARS` | `4000` | char budget for the `` (KB overview) prompt section | diff --git a/alembic/versions/0006_git_sources.py b/alembic/versions/0006_git_sources.py new file mode 100644 index 0000000..4ea0e37 --- /dev/null +++ b/alembic/versions/0006_git_sources.py @@ -0,0 +1,48 @@ +"""git sources: admin-managed list of git repo URLs + +Revision ID: 0006 +Revises: 0005 +Create Date: 2026-08-26 + +Phase 35 (git-sources-admin story, A13 — one additive, reversible table, +no other schema change): + +* ``git_sources`` — one row per admin-managed git source URL. The admin + page (``/git-sources.html``) maintains the list that the Sync button + (phase 32) and ``import_docs`` (phase 28) clone/pull. DB rows win over + the ``BOR_GIT_SOURCES`` env var, which stays a fallback while the table + is empty (once the table has rows it is ignored — the UI is the source + of truth). ``url`` is unique (``uq_git_sources_url``); ``added_at`` is + stamped server-side and orders the list oldest-first. +""" +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision = "0006" +down_revision = "0005" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "git_sources", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column( + "added_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + op.create_index("uq_git_sources_url", "git_sources", ["url"], unique=True) + + +def downgrade() -> None: + op.drop_index("uq_git_sources_url", table_name="git_sources") + op.drop_table("git_sources") diff --git a/app/api/git_sources.py b/app/api/git_sources.py new file mode 100644 index 0000000..3fb9c94 --- /dev/null +++ b/app/api/git_sources.py @@ -0,0 +1,132 @@ +"""Admin-managed git sources API (phase 35, task 02). + +Admin-only CRUD under ``/api/git-sources`` (phase 16 pattern, A10 +extended — the public API surface stays stateless and the signed cookie +remains the only session state, same as ``/api/steering`` and +``/api/sync``): the ``git_sources`` table holds the repo URLs the Sync +button (phase 32) and ``import_docs`` (phase 28) clone/pull. DB rows win +over ``BOR_GIT_SOURCES``, which is a fallback while the table is empty +(the phase's locked decision — ``from_env`` tells the UI which list it is +looking at, so the page can show the env note only while the fallback is +active). + +Routes: ``GET`` (DB rows oldest-first, or the env list with +``from_env: true`` while the table is empty), ``POST`` (201, validated +create), ``DELETE /{source_id}`` (204). The whole router sits behind +:func:`app.core.auth.require_admin` — anonymous callers get 403 on every +route. + +No credential-echo path: URLs may embed ``user:pass@`` (phase 32's +masking discipline), so the 409/422 details are fixed generic strings +that never repeat the submitted URL. + +Scope boundary (phase locked decisions): adding or removing a repo does +NOT clone, import, or prune anything — the existing Sync button performs +that (a removal prunes on the next sync, ``prune=True``). +""" +from __future__ import annotations + +import re +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Response +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.config import get_settings +from app.core.auth import require_admin +from app.db import get_db +from app.models import GitSource +from app.schemas import GitSourceIn, GitSourceList, GitSourceOut + +router = APIRouter( + prefix="/git-sources", + tags=["git-sources"], + dependencies=[Depends(require_admin)], # phase 16 pattern: admin-only surface +) + +#: Accepted git URL shapes — the trimmed URL must *start* with one of them. +#: Covers the phase-28 real URLs (HTTPS + ``git@`` SSH); scp-style +#: ``host:repo`` is deliberately rejected (422). ASSUMPTION (task 02): the +#: accepted shapes are exactly these four prefixes. +URL_RE = re.compile(r"^(https?://|ssh://|git@)") + + +@router.get("", response_model=GitSourceList) +def list_git_sources( + db: Session = Depends(get_db), # noqa: B008 +) -> GitSourceList: + """The effective git source list. + + DB rows ordered by ``(added_at, id)`` (oldest first, id tie-break for + same-timestamp inserts) with ``from_env: false``; while the table is + empty, the ``BOR_GIT_SOURCES`` env URLs as rows with null + ``id``/``added_at`` and ``from_env: true``. + """ + rows = db.scalars( + select(GitSource).order_by(GitSource.added_at.asc(), GitSource.id.asc()) + ).all() + if rows: + return GitSourceList( + sources=[GitSourceOut(id=row.id, url=row.url, added_at=row.added_at) for row in rows], + from_env=False, + ) + return GitSourceList( + sources=[ + GitSourceOut(id=None, url=url, added_at=None) + for url in get_settings().git_source_list + ], + from_env=True, + ) + + +@router.post("", response_model=GitSourceOut, status_code=201) +def create_git_source( + payload: GitSourceIn, + db: Session = Depends(get_db), # noqa: B008 +) -> GitSourceOut: + """Store one repo URL (already trimmed by the schema). + + 422 when the shape is not one of the accepted prefixes (generic + detail — the input is never echoed); 409 when the trimmed URL is + already stored (same; the unique index is the backstop against a + concurrent insert the pre-check missed); 201 + the created row + otherwise. + """ + url = payload.url + if not URL_RE.match(url): + raise HTTPException( + status_code=422, detail="not a valid git URL (expected https://, ssh:// or git@…)" + ) + if db.scalar(select(GitSource).where(GitSource.url == url)) is not None: + raise HTTPException(status_code=409, detail="a git source with this URL already exists") + row = GitSource(url=url) + db.add(row) + try: + db.commit() + except IntegrityError: + db.rollback() + raise HTTPException( + status_code=409, detail="a git source with this URL already exists" + ) from None + db.refresh(row) + return GitSourceOut(id=row.id, url=row.url, added_at=row.added_at) + + +@router.delete("/{source_id}", status_code=204) +def delete_git_source( + source_id: uuid.UUID, + db: Session = Depends(get_db), # noqa: B008 +) -> Response: + """Remove a stored row; 404 when the id is unknown. + + Removing does not touch the clones or the index — the next Sync + (``prune=True``) prunes the dropped repo (phase scope boundary). + """ + row = db.get(GitSource, source_id) + if row is None: + raise HTTPException(status_code=404, detail="git source not found") + db.delete(row) + db.commit() + return Response(status_code=204) diff --git a/app/api/sync.py b/app/api/sync.py index ad9706e..cb7cdbe 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -15,9 +15,12 @@ stale button state (§7.4 adaptation, phase locked decisions). Pipeline (the canonical "mirror the repos" action — phase locked decisions): -1. resolve the ``BOR_GIT_SOURCES`` URLs — empty/missing fails loudly - (``no git sources configured``) instead of silently importing the - legacy local directories; +1. resolve the effective git sources — the ``git_sources`` DB rows, + else the ``BOR_GIT_SOURCES`` fallback + (:func:`app.rag.git_sources.effective_git_sources`, shared with the + CLI) — empty on both origins fails loudly (``no git sources + configured``) instead of silently importing the legacy local + directories; 2. :func:`scripts.git_sync.clone_or_pull` each repo into ``BOR_SOURCES_DIR//`` (phase 28 — reused, not re-implemented; a failing repo aborts before any import); @@ -44,6 +47,8 @@ from fastapi import APIRouter, Depends, HTTPException from app.config import get_settings from app.core.auth import require_admin +from app.db import SessionLocal +from app.rag.git_sources import effective_git_sources from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.overview import regenerate_overview @@ -141,13 +146,23 @@ async def _run_sync() -> None: _status.error = None try: settings = get_settings() - git_urls = settings.git_source_list + # The background task has no request session: open a short-lived + # one around the shared phase-35 resolver (DB rows win, the + # BOR_GIT_SOURCES list is a fallback while the table is empty). + db = SessionLocal() + try: + git_urls, origin = effective_git_sources(db) + finally: + db.close() if not git_urls: - # The button targets BOR_GIT_SOURCES only (manual --source - # dirs have no repo to clone) — an empty config fails loudly - # instead of silently importing the legacy directories. - raise GitSyncError("no git sources configured (BOR_GIT_SOURCES)") - logger.info("sync: started repos=%d", len(git_urls)) + # The button targets git sources only (manual --source dirs + # have no repo to clone) — an empty config on *both* origins + # fails loudly instead of silently importing the legacy + # directories. + raise GitSyncError( + "no git sources configured (git_sources table empty and BOR_GIT_SOURCES unset)" + ) + logger.info("sync: started repos=%d origin=%s", len(git_urls), origin) sources_root = Path(settings.sources_dir).expanduser() sources = [clone_or_pull(url, sources_root / repo_name(url)) for url in git_urls] llm = LLMClient() diff --git a/app/core/caching.py b/app/core/caching.py index 879aa15..d76c055 100644 --- a/app/core/caching.py +++ b/app/core/caching.py @@ -111,7 +111,10 @@ def asset_version(static_dir: str | None = None) -> str: # Response-layer cache busting (task 02) # --------------------------------------------------------------------------- -#: The five known HTML pages — the ONLY paths whose body is rewritten. +#: The known HTML pages — the ONLY paths whose body is rewritten. +#: Phase 35 adds the admin git sources page: without this entry it +#: would serve unversioned asset refs, which the immutable-for-a-year +#: asset caching would pin to stale CSS after a deploy. HTML_PAGES: tuple[str, ...] = ( "/", "/index.html", @@ -119,6 +122,7 @@ HTML_PAGES: tuple[str, ...] = ( "/document.html", "/login.html", "/tuning.html", + "/git-sources.html", # phase 35: the admin git sources page ) #: Prefix of the versioned static assets (header-only caching; the body is diff --git a/app/main.py b/app/main.py index f7fef01..39e541c 100644 --- a/app/main.py +++ b/app/main.py @@ -22,6 +22,7 @@ from starlette.middleware.sessions import SessionMiddleware from app.api.auth import router as auth_router from app.api.chat import router as chat_router from app.api.docs import router as docs_router +from app.api.git_sources import router as git_sources_router from app.api.health import router as health_router from app.api.steering import router as steering_router from app.api.suggestions import router as suggestions_router @@ -64,6 +65,7 @@ def create_app() -> FastAPI: app.include_router(auth_router, prefix="/api") app.include_router(suggestions_router, prefix="/api") app.include_router(docs_router, prefix="/api") + app.include_router(git_sources_router, prefix="/api") app.include_router(chat_router, prefix="/api") app.include_router(steering_router, prefix="/api") app.include_router(sync_router, prefix="/api") diff --git a/app/models.py b/app/models.py index 1e27adc..618d647 100644 --- a/app/models.py +++ b/app/models.py @@ -13,6 +13,8 @@ Data model — see ``.agent/PLAN.md`` §Data Model: * ``kb_overview`` — single-row lite-generated outline of the KB's basic categories, injected as the ```` section of every chat turn (phase 31). +* ``git_sources`` — admin-managed git source URLs the Sync button and + import_docs clone/pull (phase 35). """ from __future__ import annotations @@ -133,3 +135,19 @@ class KbOverview(Base): updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now() ) + + +class GitSource(Base): + """One admin-managed git source (phase 35). + + The UI-maintained list of repo URLs the Sync button (phase 32) and + import_docs (phase 28) clone/pull. DB rows win over the + BOR_GIT_SOURCES env var, which is a fallback while this table is + empty (see app.rag.git_sources.effective_git_sources). + """ + + __tablename__ = "git_sources" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + url: Mapped[str] = mapped_column(Text, unique=True, nullable=False) + added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) diff --git a/app/rag/git_sources.py b/app/rag/git_sources.py new file mode 100644 index 0000000..e2f7ea8 --- /dev/null +++ b/app/rag/git_sources.py @@ -0,0 +1,44 @@ +"""The shared git-source resolver (phase 35, task 03). + +One function, two callers: the in-app Sync pipeline +(``app/api/sync.py::_run_sync``) and the CLI +(``scripts/import_docs.py::_resolve_sources``) — both resolve the repo +URLs to clone/pull through :func:`effective_git_sources`, so the +admin-managed ``git_sources`` table is what actually gets cloned and +indexed from either entry point (the API's ``GET /api/git-sources`` +fallback list is the only other place that reads the env list — task 02). + +Precedence (the phase's locked decision — the env var is demoted, not +removed): ``git_sources`` DB rows win, in ``(added_at, id)`` order; +``BOR_GIT_SOURCES`` is a fallback only while the table is empty; both +empty → ``([], "env")`` and each caller keeps its existing fail-loud +behavior (sync: ``GitSyncError``; CLI: the legacy ``DEFAULT_SOURCES`` +fallback). +""" +from __future__ import annotations + +from typing import Literal + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.config import get_settings +from app.models import GitSource + + +def effective_git_sources(db: Session) -> tuple[list[str], Literal["db", "env"]]: + """``(urls, origin)`` — the repo URLs to clone, and where they came from. + + ``"db"``: the ``git_sources`` rows in ``(added_at, id)`` order (oldest + first, id tie-break for same-timestamp inserts) — once the table has + any row, ``BOR_GIT_SOURCES`` is ignored entirely. + ``"env"``: the ``BOR_GIT_SOURCES`` list — ``Settings.git_source_list``, + the phase-28 CSV parse, reused not re-implemented — used only while + the table is empty; both empty → ``([], "env")``. + """ + rows = db.scalars( + select(GitSource).order_by(GitSource.added_at.asc(), GitSource.id.asc()) + ).all() + if rows: + return [row.url for row in rows], "db" + return list(get_settings().git_source_list), "env" diff --git a/app/schemas.py b/app/schemas.py index 88d074f..9ef272c 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -154,3 +154,46 @@ class SteeringNoteList(BaseModel): """``GET /api/steering`` response: all notes, newest first.""" notes: list[SteeringNote] + + +class GitSourceIn(BaseModel): + """``POST /api/git-sources`` body: one repo URL (phase 35, task 02). + + Mirrors :class:`SteeringNoteIn` — the URL is trimmed *before* the + length constraints run, so a whitespace-only body is a 422 and a URL + with surrounding spaces is stored clean. Shape validation + (``https://``, ``ssh://``, ``git@``) happens in the API layer so the + 422 detail can be one generic string that never echoes the input. + """ + + url: str = Field(min_length=1, max_length=500) + + @field_validator("url", mode="before") + @classmethod + def _trim_url(cls, v: object) -> object: + return v.strip() if isinstance(v, str) else v + + +class GitSourceOut(BaseModel): + """One git source as returned by the API (phase 35, task 02). + + ``id`` / ``added_at`` are nullable: env-fallback rows (table empty → + the list comes from ``BOR_GIT_SOURCES``) carry neither, only a URL. + """ + + id: uuid.UUID | None + url: str + added_at: datetime | None + + +class GitSourceList(BaseModel): + """``GET /api/git-sources`` response (phase 35, task 02). + + ``from_env`` is True only when the ``git_sources`` table is empty and + the list comes from ``BOR_GIT_SOURCES`` (the phase's locked fallback); + once the table has rows the env var is ignored and ``from_env`` is + False — the UI is the source of truth. + """ + + sources: list[GitSourceOut] + from_env: bool diff --git a/frontend/assets/git-sources.js b/frontend/assets/git-sources.js new file mode 100644 index 0000000..4f770cc --- /dev/null +++ b/frontend/assets/git-sources.js @@ -0,0 +1,320 @@ +/* Brain of Reese — Git sources admin page (phase 35, task 04). + * + * The page module for /git-sources.html: the admin-only manager for the + * stored git source list (git-sources table, phase 35 tasks 01/02). + * This module is the single owner of the page's behaviour: + * + * • boot — initSharedHeader() (one cached whoami, shared with the + * header toggling): anonymous → the sign-in gate shows and the + * manager stays hidden (the exact Sources page gate pattern, and + * NO /api/git-sources call is made); admin → gate hidden, + * #git-sources-content revealed, loadSources(). + * • loadSources() — GET /api/git-sources → the table rows + * (#git-sources-tbody), the env-fallback note's visibility + * (from_env), and the empty state. URLs are ALWAYS rendered with + * textContent — never innerHTML (they may embed user:pass@ + * credentials; phase 32's masking discipline). Non-2xx or a + * network failure renders the role="alert" load error with a + * retry button — never a stuck page. + * • add — #git-source-form submit → POST /api/git-sources {url}. + * §7.4 never-stale: the button disables + relabels "Adding…" + * while the request is out, re-enables ("Add source") on success + * AND failure. 201 clears the input, reloads the list, and + * focuses the new row's Remove button (a11y); a failure (409 + * duplicate, 422 shape) shows the server detail inline under the + * form (role="alert", 422 shape-aware like the tuning forms) and + * keeps the input — the instruction survives. + * • remove — a row's Remove button asks window.confirm first + * (removal prunes the documents only on the NEXT sync — the + * confirm says so). Cancel → nothing; ok → the row button + * disables, DELETE /api/git-sources/{id}, loadSources(). A + * failure shows a per-row role="alert" error and re-enables the + * button. Env-fallback rows (id null — the list comes from + * BOR_GIT_SOURCES, not the table) carry no Remove: nothing is + * stored to remove — they show a "from .env" tag instead. + * • announce(msg) — #git-sources-announcer (role=status, + * aria-live=polite): the screen-reader confirmation for loads, + * adds, and removals. + * + * Scope boundary (phase locked decisions): adding or removing a repo + * does NOT clone, import, or prune — the header's Sync sources button + * (module-owned in assets/header.js) performs that; the page's hint + * box says so. + * + * The shared header module loads through this script's own relative + * import ("./header.js") — a hoisted import evaluated before this body + * runs (single-evaluation design: no direct + + diff --git a/frontend/index.html b/frontend/index.html index 1f49081..8a10b24 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -25,6 +25,11 @@ reveals it once whoami says admin. The soft-gated page itself is unchanged. --> + + + + + + + +