diff --git a/.agent/phases/complete/28_git_based_sources/01_env_var_and_settings.md b/.agent/phases/complete/28_git_based_sources/01_env_var_and_settings.md new file mode 100644 index 0000000..6d01fed --- /dev/null +++ b/.agent/phases/complete/28_git_based_sources/01_env_var_and_settings.md @@ -0,0 +1,37 @@ +# Task 01 — Settings: `BOR_GIT_SOURCES` + `BOR_SOURCES_DIR` + +**Phase:** `28_git_based_sources` · **Source:** `TODO.md:5 — "We shouldn't be hard-coding Homelab and Deployments. Instead, a list of git links should be specified."` +**Story:** `.agent/user_stories/git-sources.md` + +## Objective +Add the two new settings that drive the git-based source flow: the comma-separated list of git URLs and the dedicated local clone location. + +## Work +1. `app/config.py` — in the "Import scope" section (near `import_extensions`) add: + - `git_sources: str = ""` — comma-separated git repository URLs (no default; when empty, `import_docs` falls back to `--source` / the old `DEFAULT_SOURCES`). Document: "list of git repo URLs to clone/pull into `sources_dir` before indexing (phase 28); empty means no git sources". + - `sources_dir: str = "~/bor-sources"` — the dedicated local directory the repos are cloned/pulled into (phase 28). Document: "where `import_docs` clones/pulls `git_sources` repos (expanded ~ via `Path.expanduser`)". + - Add a property `git_source_list` returning the non-empty, stripped URLs (list[str]) — used by the import script. +2. `tests/unit/test_config.py` — add tests: + - `git_sources` default is `""`; `git_source_list` returns `[]` when empty. + - `git_sources` env override parses a comma-separated list (whitespace trimmed, empties dropped). + - `sources_dir` default is `~/bor-sources` (raw string, not expanded in the setting — expansion happens in the script). +3. `.env.example` — document both under an "Import scope / git sources" comment block: + ``` + # --- Import sources (git; phase 28) --- + # BOR_GIT_SOURCES=https://github.com/user/homelab.git,https://github.com/user/deployments.git + # BOR_SOURCES_DIR=~/bor-sources + ``` + +## ASSUMPTIONS +- `git_sources` is empty by default (backwards-compatible: the `import_docs` script keeps its `--source` default until the owner sets `BOR_GIT_SOURCES`). +- URLs are stored raw (no parsing of scheme/auth in the setting) — parsing happens in `git_sync.py`. +- `sources_dir` is stored as a raw string; `Path.expanduser()` is applied in the script (so the setting stays env-agnostic and testable). + +## Testing & Quality +- Unit: `tests/unit/test_config.py` additions. +- Coverage: the new setting is exercised by the unit test; the >90% `app/` gate is maintained. + +## Completion Criteria +- [ ] `Settings().git_source_list` returns `[]` by default and the parsed list when `BOR_GIT_SOURCES` is set. +- [ ] `sources_dir` defaults to `~/bor-sources`. +- [ ] `.env.example` documents both vars. diff --git a/.agent/phases/complete/28_git_based_sources/02_git_clone_pull_utility.md b/.agent/phases/complete/28_git_based_sources/02_git_clone_pull_utility.md new file mode 100644 index 0000000..2c210a1 --- /dev/null +++ b/.agent/phases/complete/28_git_based_sources/02_git_clone_pull_utility.md @@ -0,0 +1,67 @@ +# Task 02 — Git clone/pull utility + +**Phase:** `28_git_based_sources` · **Source:** `TODO.md:5 — "import_docs should clone or pull to a dedicated repository location and then index all the specified repository code."` +**Story:** `.agent/user_stories/git-sources.md` + +## Objective +Create `scripts/git_sync.py` with a single `clone_or_pull(url, dest)` function that clones a repo (shallow, first run) or fast-forwards it (subsequent runs), returning the destination path. This is the only place `git` is invoked. + +## Work +1. `scripts/git_sync.py` (new) — stdlib `subprocess` only (A11: no new packages): + ```python + """Git source sync for import_docs (phase 28). + + clone_or_pull(url, dest) clones ``url`` into ``dest`` (shallow, depth 1) + the first time, or fast-forwards an existing checkout with ``git pull + --ff-only`` on subsequent runs. Auth is whatever the URL/SSH config + supplies — no credentials are stored here. + """ + from __future__ import annotations + import subprocess + from pathlib import Path + + class GitSyncError(RuntimeError): ... + + def clone_or_pull(url: str, dest: Path | str) -> Path: + dest = Path(dest) + if not dest.exists() or not (dest / ".git").exists(): + dest.parent.mkdir(parents=True, exist_ok=True) + _run(["git", "clone", "--depth", "1", url, str(dest)], cwd=dest.parent) + else: + _run(["git", "pull", "--ff-only"], cwd=dest) + return dest + + def _run(argv, cwd): + try: + proc = subprocess.run(argv, cwd=cwd, capture_output=True, text=True) + except FileNotFoundError: + raise GitSyncError("git was not found on PATH — install git and retry") + if proc.returncode != 0: + raise GitSyncError(f"git {' '.join(argv[1:])} failed (exit {proc.returncode}): {proc.stderr.strip()}") + return proc.stdout + ``` + - `--depth 1` clone (fast, and the KB is re-imported incrementally anyway). + - `--ff-only` pull (refuses to merge unrelated histories — a broken checkout fails loudly rather than producing a dirty index). + - `GitSyncError` carries the `git` stderr so the caller can name the failing repo + reason. + - Auth: nothing special — a `https://…` URL uses the OS credential helper / prompts; an `git@host:repo.git` URL uses the machine's SSH key. Document this in the module docstring. +2. `tests/unit/test_git_sync.py` (new): + - `clone_or_pull` **clones** when the dest has no `.git` — verify by monkeypatching `subprocess.run` to a fake that records the argv and returns `returncode=0`; assert `["git","clone","--depth","1",url,str(dest)]` was called and `dest` returned. + - `clone_or_pull` **pulls** when `.git` exists — monkeypatch; assert `["git","pull","--ff-only"]` called. + - `clone_or_pull` raises `GitSyncError` on non-zero exit, carrying the stderr text. + - `clone_or_pull` raises `GitSyncError("git was not found…")` when `subprocess.run` raises `FileNotFoundError`. + - `dest.parent` is created before clone (assert the fake saw a parent that `mkdir` would create — or test the `mkdir` call path directly). + +## ASSUMPTIONS +- `--depth 1` shallow clone is sufficient (the KB is re-imported incrementally; no need for full history). +- `--ff-only` pull is the right policy (refuse merges — a dirty/broken checkout fails loudly). +- Auth is delegated to the machine (SSH key / credential helper); no secrets are stored in code or `.env`. +- `git` CLI is assumed present (standard on homelab machines; a clear error is raised otherwise). + +## Testing & Quality +- Unit: `tests/unit/test_git_sync.py` (clone vs pull dispatch, error propagation, missing-git error). +- Coverage: the new `scripts/git_sync.py` is fully covered. + +## Completion Criteria +- [ ] `clone_or_pull` clones a fresh repo and pulls an existing one (verified via monkeypatched `subprocess`). +- [ ] A failing `git` call raises `GitSyncError` with the stderr; a missing `git` raises `GitSyncError` naming git. +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/complete/28_git_based_sources/03_import_docs_script_refactor.md b/.agent/phases/complete/28_git_based_sources/03_import_docs_script_refactor.md new file mode 100644 index 0000000..f61040c --- /dev/null +++ b/.agent/phases/complete/28_git_based_sources/03_import_docs_script_refactor.md @@ -0,0 +1,49 @@ +# Task 03 — `import_docs` resolves git sources → local dirs + +**Phase:** `28_git_based_sources` · **Source:** `TODO.md:5 — "import_docs should clone or pull to a dedicated repository location and then index all the specified repository code."` +**Story:** `.agent/user_stories/git-sources.md` + +## Objective +Rewire `scripts/import_docs.py` so that, when `BOR_GIT_SOURCES` is set, it clones/pulls each repo into `BOR_SOURCES_DIR//` and indexes the resulting directories — while keeping `--source ` overriding for manual local directories. + +## Work +1. `scripts/import_docs.py`: + - Import `clone_or_pull` from the sibling module: `from scripts.git_sync import clone_or_pull` (or, since `main()` runs as `python -m scripts.import_docs`, a top-level `from git_sync import clone_or_pull` works — `scripts/` is on `sys.path`). + - Keep `DEFAULT_SOURCES` as the fallback for the no-`--source` + no-`BOR_GIT_SOURCES` case (backwards-compatible). + - New resolution logic in `main()`: + ```python + settings = get_settings() + sources = _resolve_sources(args.source, settings) + ``` + where `_resolve_sources`: + - If `args.source` is given → expanduser each and return (unchanged manual behaviour; `--source` wins). + - Else if `settings.git_source_list` is non-empty → for each URL, `clone_or_pull(url, Path(settings.sources_dir).expanduser() / repo_name(url))`; collect the dest dirs; return them. Any `GitSyncError` propagates (the script exits non-zero naming the failing repo — see below). + - Else → return the old `DEFAULT_SOURCES`. + - `repo_name(url)` — derive a directory name from the URL: strip a trailing `.git`, take the basename after the last `/` (or `:` for scp-style `git@host:repo.git`). Fall back to a slug of the URL if no basename. + - Before importing, log which dirs are being imported (so the operator sees the cloned paths). The existing "source dir not found" warning still applies if a clone left an empty dir. +2. Keep the existing `--prune` / `--limit` flags and the summary print unchanged. +3. Exit code: if any `GitSyncError` is raised, let it propagate to a top-level `except` that prints `import_docs: git sync failed: ` to stderr and returns 1 **before** importing anything (so a bad repo doesn't silently import partial junk). Structure: + ```python + try: + sources = _resolve_sources(...) + except GitSyncError as e: + print(f"import_docs: git sync failed: {e}", file=sys.stderr) + return 1 + ``` + +## ASSUMPTIONS +- `--source` always wins over `BOR_GIT_SOURCES` (explicit CLI flag beats env). +- When `BOR_GIT_SOURCES` is set, `--source` is ignored (only one source mode at a time) — document this. +- `BOR_SOURCES_DIR` defaults to `~/bor-sources`; each repo is a subdirectory named after the repo. +- A repo that fails to clone/pull aborts the whole run (no partial import) — the operator fixes the URL and re-runs; the already-cloned repos are left on disk and will be pulled on the next run. +- The `documents.source` column will be the repo directory name (e.g. `homelab`), matching the current `source=root.name` behaviour in the importer. + +## Testing & Quality +- Integration: `tests/integration/test_import_docs_git.py` (new) — monkeypatch `clone_or_pull` to a fake that creates a temp dir with a fixture `.md` file and returns it; assert `import_docs.main(["--source"])`-equivalent resolution picks the cloned dir and that `import_sources` is called with it. Also assert a `GitSyncError` from the fake → exit code 1 and no import attempt. +- Coverage: the new `main()` resolution branch is covered. + +## Completion Criteria +- [ ] `BOR_GIT_SOURCES` set → `import_docs` clones/pulls each repo into `BOR_SOURCES_DIR//` and imports them. +- [ ] `--source ` still imports that manual directory (unchanged). +- [ ] A failing git sync → non-zero exit, message naming the repo, no partial import. +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/complete/28_git_based_sources/04_integration_test_and_docs.md b/.agent/phases/complete/28_git_based_sources/04_integration_test_and_docs.md new file mode 100644 index 0000000..48baec2 --- /dev/null +++ b/.agent/phases/complete/28_git_based_sources/04_integration_test_and_docs.md @@ -0,0 +1,37 @@ +# Task 04 — Integration test + docs + +**Phase:** `28_git_based_sources` · **Source:** `TODO.md:5 — "import_docs should clone or pull to a dedicated repository location and then index all the specified repository code."` +**Story:** `.agent/user_stories/git-sources.md` + +## Objective +Add the integration test that exercises the full `import_docs` git-resolution path with a mocked `clone_or_pull`, and finalise the docs. + +## Work +1. `tests/integration/test_import_docs_git.py` (new) — drive `scripts.import_docs.main(argv)` end to end with the git flow mocked: + - Monkeypatch `scripts.import_docs.clone_or_pull` (import the name into the module's namespace after importing it) so it **creates** a temp directory containing a fixture `.md` file and returns that path (simulating a clone/pull landing real content). + - Set `settings.sources_dir` to a `tmp_path`-based dir (via `monkeypatch.setattr` on the settings or by patching `get_settings`). + - Call `main([])` with `BOR_GIT_SOURCES` patched to a single URL. + - Assert: `clone_or_pull` was called once with the URL; the returned dir was passed to `import_sources`; the summary printed includes the fixture file (added > 0). + - Negative case: monkeypatch `clone_or_pull` to raise `GitSyncError`; assert `main([])` returns `1` and prints a message naming the repo, and that `import_sources` was **not** called. + - Manual override case: `main(["--source", str(tmp_path)])` imports the manual dir and does **not** call `clone_or_pull`. + - Use the same `FakeEmbedder`-style approach the importer tests use so no real LLM is needed (the importer's two-phase embed is duck-typed; pass a fake `llm` if `main` allows injection, or let `import_docs` build the real `LLMClient` but patch `import_sources` to capture its `sources` arg and short-circuit). Simpler: monkeypatch `scripts.import_docs.import_sources` to a fake that records the `sources` list and returns a trivial `ImportSummary` — this isolates the git-resolution logic from the whole embed pipeline. +2. `README.md`: + - Section 5 (Import your knowledge base): document the two source modes — (a) git sources via `BOR_GIT_SOURCES` (clone/pull into `BOR_SOURCES_DIR`), (b) `--source ` for manual directories. Keep the `~/Homelab + ~/Deployments` note as the *previous* default, now replaced by `BOR_GIT_SOURCES`. + - Add a short "Git-based sources" subsection: set `BOR_GIT_SOURCES` (comma-separated URLs) + `BOR_SOURCES_DIR`; `import_docs` clones (first run) or pulls (subsequent runs) each repo and indexes them; `--source` overrides; a failed sync aborts the run. + - The "Clicking a chip opens that document in a new tab" bullet (line ~69) is now stale — update to "opens in an almost-fullscreen modal" (phase 26). +3. `.env.example` — already updated in task 01; double-check the comment block is present and correct. +4. `.agent/user_stories/git-sources.md` — write the story file. + +## ASSUMPTIONS +- The integration test mocks `clone_or_pull` + `import_sources` so it runs without network, without `git`, and without a real embed endpoint — it isolates the resolution logic. +- The README keeps a migration note for owners currently relying on the hardcoded `~/Homelab`/`~/Deployments` default (set `BOR_GIT_SOURCES` to the same two repos). + +## Testing & Quality +- Integration: `tests/integration/test_import_docs_git.py`. +- Coverage: the new `main()` branch covered. + +## Completion Criteria +- [ ] `uv run pytest tests/integration/test_import_docs_git.py -v --no-cov` green. +- [ ] `uv run pytest` green; `uv run pytest --cov=app --cov-report=term-missing` TOTAL ≥ pre-change number. +- [ ] `uv run ruff check . && uv run pyright` clean. +- [ ] README documents both source modes; the "opens in a new tab" bullet updated to "modal". diff --git a/.env.example b/.env.example index 26af007..8517575 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,15 @@ BOR_RRF_K=60 # Reciprocal Rank Fusion damping constant # BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py # BOR_SUGGESTIONS=["How is my Kubernetes cluster set up?"] # JSON list of onboarding chips +# --- Import sources (git; phase 28) --- +# Comma-separated git repo URLs; import_docs clones each (first run) or +# pulls it (subsequent runs) into BOR_SOURCES_DIR// and indexes +# the result. Empty = no git sources (import_docs falls back to --source / +# the old ~/Homelab + ~/Deployments defaults). Auth via URL (e.g. an +# https token) or SSH keys. +# BOR_GIT_SOURCES=https://github.com/user/homelab.git,https://github.com/user/deployments.git +# BOR_SOURCES_DIR=~/bor-sources + # --- Admin & sign-in (single-admin password login; BOTH required) --- # The app refuses to start while either is empty (names the missing # variable(s) — README "Admin & sign-in"). Generate the secret with: diff --git a/README.md b/README.md index 48afd52..093a2fa 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,22 @@ uv run alembic upgrade head ### 5. Import your knowledge base ```bash uv run python -m scripts.llm_probe # sanity: models + 768-dim check -uv run python -m scripts.import_docs # defaults: ~/Homelab + ~/Deployments +uv run python -m scripts.import_docs # import the configured sources (below) ``` +Two source modes: + +- **Git sources (recommended)** — set `BOR_GIT_SOURCES` in `.env` to a + comma-separated list of git repo URLs. `import_docs` clones each repo + (first run) or pulls it (subsequent runs) into + `BOR_SOURCES_DIR//` (default `~/bor-sources`) and indexes the + checkouts — see [Git-based sources](#git-based-sources). +- **Manual directories** — `--source ` (repeatable) imports local + directories directly and *always wins* over `BOR_GIT_SOURCES`. +- If neither is set, the import falls back to the **previous** default, + `~/Homelab` + `~/Deployments` — kept only for backwards compatibility, + now replaced by `BOR_GIT_SOURCES`. + ### 6. Run the app ```bash uv run uvicorn app.main:app --reload @@ -76,16 +89,19 @@ uv run uvicorn app.main:app --reload - **Chat** (`/`) — ask questions; answers stream in with **source chips** that cite the exact documents used. Clicking a chip opens that document - **in a new tab**. -- **Document viewer** (`/document.html?source=…&path=…`) — the full text of - any indexed document, served from the database (no filesystem access): + in an **almost-fullscreen modal** on the same page (no new tab). +- **Document viewer** — the modal above *is* the viewer; the full text of + any indexed document is served from the database (no filesystem access): markdown is rendered, every other format (`yaml`, `json`, `py`, `txt`, …) - is shown as escaped monospace text. Unknown documents get a designed - not-found state with a link back to the index. + is shown as escaped monospace text. `/document.html?source=…&path=…` + stays as the **full-page / direct-link** form (the modal's “Full page” + button and the URL to share — it works without JS). Unknown documents + get a designed not-found state with a link back to the index. - **Sources** (`/sources.html`) — the indexed document list; the *Path* - column links each document to the viewer in a new tab. **Admin-only** — - anonymous visitors see a sign-in gate instead (the catalog is what the - login locks; the document viewer itself stays open to everyone). + column opens each document in the same **almost-fullscreen modal** (no + new tab). **Admin-only** — anonymous visitors see a sign-in gate instead + (the catalog is what the login locks; the document viewer itself stays + open to everyone). ## Thinking @@ -205,6 +221,10 @@ uv run python -m scripts.import_docs --prune # also drop deleted/out-of- uv run python -m scripts.import_docs --source ~/SomeOtherDocs ``` +With **git-based sources** (below) each run first pulls the latest commits +of your repos, so this same command is the whole update loop: commit in the +repo → re-run the import. + Then check the **Sources** page (`http://localhost:8000/sources.html`): the *documents* / *chunks* counters and *last indexed* timestamp should reflect the new files, and each document row shows when it was last @@ -230,6 +250,35 @@ embedded. - To sanity-check the LLM backend (models + embedding dimension) after any aipi change: `uv run python -m scripts.llm_probe`. +### Git-based sources + +Rather than pointing the import at local folders, point it at **git +repositories** — the notes live in the repos and `import_docs` keeps local +checkouts of them up to date for you: + +```env +# .env +BOR_GIT_SOURCES=https://git.reeseapps.com/reese/homelab.git,git@github.com:reese/deployments.git +BOR_SOURCES_DIR=~/bor-sources # default; each repo lands in // +``` + +- `BOR_GIT_SOURCES` is a **comma-separated list** of URLs. Auth is whatever + the machine supplies — `https://…` via the OS credential helper, or + `git@host:repo.git` via your SSH key; no credentials are stored in the + app or `.env`. +- Every run **clones** each repo (first time, shallow `--depth 1`) or + **pulls** it (`git pull --ff-only` — fast-forward only, so a diverged or + broken checkout fails loudly instead of merging) into + `BOR_SOURCES_DIR//`, then indexes the checkouts exactly like + any local directory (A9 format filter, hidden-dir skip, sha256 delta). + `documents.source` is the repo directory name (e.g. `homelab`). +- **`--source ` overrides**: when the flag is given, + `BOR_GIT_SOURCES` is ignored and the manual directory(ies) are imported. +- **A failed sync aborts the run**: if any repo cannot be cloned/pulled, + `import_docs` exits non-zero naming the failing repo and imports + **nothing** (no partial junk). Fix the URL/connectivity and re-run — the + other checkouts stay on disk and are pulled as usual. + ## Checking retrieval quality Ask the *real* pipeline (live aipi embeddings + the current KB) whether a @@ -374,6 +423,8 @@ served locally (no CDN), `BOR_ENVIRONMENT=production`. | `BOR_HYBRID_LEXICAL_CANDIDATES` | `30` | FTS list width for the RRF fusion | | `BOR_RRF_K` | `60` | RRF damping constant (`1/(k + rank)`) | | `BOR_IMPORT_EXTENSIONS` | `md,markdown,txt,yaml,yml,json,py` | csv of importable formats (may only narrow the A9 set) | +| `BOR_GIT_SOURCES` | — (empty) | csv of git repo URLs; `import_docs` clones/pulls them into `BOR_SOURCES_DIR` and indexes the checkouts (see *Git-based sources*) | +| `BOR_SOURCES_DIR` | `~/bor-sources` | where the `BOR_GIT_SOURCES` repos are cloned/pulled (one subdirectory per repo) | | `BOR_STEERING_MAX_CHARS` | `8000` | char budget for the `` (steering notes) prompt section | | `BOR_SUGGESTIONS` | built-in list | JSON list of onboarding chips | | `BOR_ADMIN_PASSWORD` | *(required)* | the single admin's password (plaintext, `.env`); app refuses to start when empty | diff --git a/app/config.py b/app/config.py index 3c8e602..bff3a00 100644 --- a/app/config.py +++ b/app/config.py @@ -105,6 +105,15 @@ class Settings(BaseSettings): # via :py:meth:`import_extension_set`. ``mode="after"`` validation runs # against the raw string so a typo fails loudly at startup. import_extensions: str = "md,markdown,txt,yaml,yml,json,py" + #: List of git repo URLs to clone/pull into ``sources_dir`` before + #: indexing (phase 28); comma-separated, stored raw. Empty means no git + #: sources — ``import_docs`` then falls back to ``--source`` / the old + #: ``DEFAULT_SOURCES``. + git_sources: str = "" + #: Where ``import_docs`` clones/pulls the ``git_sources`` repos (phase + #: 28). Stored as a raw string — ``Path.expanduser()`` is applied in + #: the import script, not here. + sources_dir: str = "~/bor-sources" @field_validator("import_extensions") @classmethod @@ -139,6 +148,16 @@ class Settings(BaseSettings): if part.strip() ) + @property + def git_source_list(self) -> list[str]: + """Non-empty, stripped git URLs from :py:attr:`git_sources` (phase 28). + + Whitespace around each entry is trimmed and empty entries dropped; + an unset/empty value yields ``[]`` (the import script then uses its + legacy local-directory defaults). + """ + return [part.strip() for part in self.git_sources.split(",") if part.strip()] + @property def effective_api_key(self) -> str: """API key for aipi: explicit setting, then $AIPI_KEY, then a placeholder.""" diff --git a/scripts/git_sync.py b/scripts/git_sync.py new file mode 100644 index 0000000..1c5afc2 --- /dev/null +++ b/scripts/git_sync.py @@ -0,0 +1,60 @@ +"""Git source sync for import_docs (phase 28). + +clone_or_pull(url, dest) clones ``url`` into ``dest`` (shallow, depth 1) +the first time, or fast-forwards an existing checkout with ``git pull +--ff-only`` on subsequent runs. + +Auth: nothing special — an ``https://…`` URL uses the OS credential +helper / prompts; a ``git@host:repo.git`` URL uses the machine's SSH key. +No credentials are stored here; whatever the URL/SSH config supplies is +used. + +This module is the only place the ``git`` CLI is invoked (A11: stdlib +``subprocess`` only, no new packages). +""" +from __future__ import annotations + +import subprocess +from pathlib import Path + +__all__ = ["GitSyncError", "clone_or_pull"] + + +class GitSyncError(RuntimeError): + """A git clone/pull failed (or git is missing); carries git's stderr.""" + + +def clone_or_pull(url: str, dest: Path | str) -> Path: + """Clone ``url`` into ``dest`` (shallow, first run) or fast-forward it. + + - dest without a ``.git`` (or absent) → ``git clone --depth 1 url dest`` + (shallow: the KB is re-imported incrementally anyway). + - dest with a ``.git`` → ``git pull --ff-only`` (refuses to merge + unrelated histories — a broken checkout fails loudly rather than + producing a dirty index). + + Returns the destination path. Raises :class:`GitSyncError` when git is + missing or a git invocation exits non-zero (with git's stderr in the + message, so the caller can name the failing repo + reason). + """ + dest = Path(dest) + if not dest.exists() or not (dest / ".git").exists(): + dest.parent.mkdir(parents=True, exist_ok=True) + _run(["git", "clone", "--depth", "1", url, str(dest)], cwd=dest.parent) + else: + _run(["git", "pull", "--ff-only"], cwd=dest) + return dest + + +def _run(argv: list[str], cwd: Path) -> str: + """Run a git command, capturing output; raise GitSyncError on failure.""" + try: + proc = subprocess.run(argv, cwd=cwd, capture_output=True, text=True) + except FileNotFoundError: + raise GitSyncError("git was not found on PATH — install git and retry") from None + if proc.returncode != 0: + raise GitSyncError( + f"git {' '.join(argv[1:])} failed (exit {proc.returncode}): " + f"{proc.stderr.strip()}" + ) + return proc.stdout diff --git a/scripts/import_docs.py b/scripts/import_docs.py index 7f076c8..66e4e2a 100644 --- a/scripts/import_docs.py +++ b/scripts/import_docs.py @@ -2,16 +2,28 @@ Examples:: - uv run python -m scripts.import_docs # ~/Homelab + ~/Deployments - uv run python -m scripts.import_docs --source ~/OtherDocs # extra dir (repeatable) + uv run python -m scripts.import_docs # BOR_GIT_SOURCES, else ~/Homelab + ~/Deployments + uv run python -m scripts.import_docs --source ~/OtherDocs # explicit dir(s); always wins uv run python -m scripts.import_docs --prune # also drop deleted/out-of-scope files uv run python -m scripts.import_docs --limit 5 # debug: first 5 files only +Source resolution (phase 28), in precedence order: + +1. ``--source PATH`` — explicit manual directories (repeatable) always win; + ``BOR_GIT_SOURCES`` is ignored when this flag is used. +2. ``BOR_GIT_SOURCES`` (comma-separated git URLs) — each repo is cloned + (first run, shallow ``--depth 1``) or fast-forwarded (``git pull + --ff-only``) into ``BOR_SOURCES_DIR//`` (default + ``~/bor-sources``) and the resulting checkouts are imported. A failing + clone/pull aborts the whole run *before* anything is imported. +3. Fallback — the legacy ``DEFAULT_SOURCES`` (``~/Homelab`` + + ``~/Deployments``), kept for backwards compatibility. + Imported formats (PLAN anchor A9, revised): ``md, markdown, txt, yaml, yml, json, py`` (case-insensitive; narrow with ``BOR_IMPORT_EXTENSIONS``). Any path with a dot-prefixed component (hidden files/dirs — vendored caches) is skipped, along with non-content dirs (``.venv``, -``node_modules``, ``.git``, ``__pycache__``, ``.pytest_cache``, ``dist``, +``node_modules``, ``.git``, ``__pycache``, ``.pytest_cache``, ``dist``, ``build``). Re-runs are cheap: files are diffed by sha256 and unchanged ones are not re-embedded; ``--prune`` also drops documents whose files no longer match the format filter. @@ -20,14 +32,19 @@ from __future__ import annotations import argparse import asyncio +import logging +import re import sys from pathlib import Path -from app.config import get_settings +from app.config import Settings, get_settings from app.core.debugging import configure_debugging from app.core.logging import configure_logging from app.rag.importer import import_sources from app.rag.llm import LLMClient +from scripts.git_sync import GitSyncError, clone_or_pull + +logger = logging.getLogger("scripts.import_docs") DEFAULT_SOURCES: list[Path] = [Path("~/Homelab"), Path("~/Deployments")] @@ -42,7 +59,10 @@ def build_parser() -> argparse.ArgumentParser: action="append", type=Path, metavar="PATH", - help="directory to import (repeatable; default: ~/Homelab ~/Deployments)", + help=( + "directory to import (repeatable; always wins over BOR_GIT_SOURCES; " + "default when neither is given: ~/Homelab ~/Deployments)" + ), ) p.add_argument( "--prune", @@ -59,12 +79,63 @@ def build_parser() -> argparse.ArgumentParser: return p +def repo_name(url: str) -> str: + """Local directory name for a git URL (phase 28). + + Strips a trailing ``.git`` and takes the basename after the last ``/`` + (``:`` for scp-style ``git@host:repo.git`` URLs); falls back to a slug + of the whole URL when no usable basename remains. + """ + name = url.strip() + if name.endswith(".git"): + name = name[: -len(".git")] + base = name.rsplit("/", 1)[-1].rsplit(":", 1)[-1].strip() + if base: + return base + slug = re.sub(r"[^A-Za-z0-9]+", "-", name).strip("-") + return slug or "repo" + + +def _resolve_sources(cli_sources: list[Path] | None, settings: Settings) -> list[Path]: + """Resolve the directories to import (phase 28). + + Precedence: ``--source`` (explicit manual paths — always wins) > + ``BOR_GIT_SOURCES`` (each URL cloned/pulled via + :func:`scripts.git_sync.clone_or_pull` into + ``BOR_SOURCES_DIR//``) > the legacy ``DEFAULT_SOURCES``. + + A :class:`GitSyncError` from a failing clone/pull propagates to + :func:`main`, which aborts the run before importing anything. + """ + if cli_sources: + return [path.expanduser() for path in cli_sources] + git_urls = settings.git_source_list + if git_urls: + sources_root = Path(settings.sources_dir).expanduser() + return [clone_or_pull(url, sources_root / repo_name(url)) for url in git_urls] + return [path.expanduser() for path in DEFAULT_SOURCES] + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) - configure_logging(get_settings().log_level) + settings = get_settings() + configure_logging(settings.log_level) configure_debugging() - sources = [path.expanduser() for path in (args.source or DEFAULT_SOURCES)] + # Git sources resolve (and clone/pull) *before* any import: a failing + # repo aborts the run with a non-zero exit, naming the failure — a bad + # URL must never silently import partial junk. + try: + sources = _resolve_sources(args.source, settings) + except GitSyncError as e: + print(f"import_docs: git sync failed: {e}", file=sys.stderr) + return 1 + + logger.info( + "import_docs: importing %d source dir(s): %s", + len(sources), + ", ".join(str(s) for s in sources), + ) missing = [s for s in sources if not s.is_dir()] for s in missing: print(f"import_docs: source dir not found: {s}", file=sys.stderr) diff --git a/tests/integration/test_import_docs_git.py b/tests/integration/test_import_docs_git.py new file mode 100644 index 0000000..b30cb8f --- /dev/null +++ b/tests/integration/test_import_docs_git.py @@ -0,0 +1,212 @@ +"""Integration test: ``import_docs`` git-source resolution (phase 28, task 03). + +Drives ``scripts.import_docs`` end to end with a fake ``clone_or_pull`` (no +real git, no network) and a recording fake ``import_sources`` (no real +DB), covering: + +- ``BOR_GIT_SOURCES`` set → each URL is cloned/pulled into + ``BOR_SOURCES_DIR//`` and exactly those dirs are imported. +- ``--source`` still wins over ``BOR_GIT_SOURCES`` (no git at all). +- No git sources + no ``--source`` → the legacy ``DEFAULT_SOURCES``. +- A failing git sync → exit code 1, an error naming the failing repo on + stderr, and **zero** import attempts. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from app.config import Settings +from app.rag.importer import ImportSummary +from scripts import import_docs +from scripts.git_sync import GitSyncError + + +def _settings(git_sources: str = "", sources_dir: str = "~/bor-sources") -> Settings: + """Fresh settings (no .env file); explicit kwargs beat any env leaks.""" + return Settings(_env_file=None, git_sources=git_sources, sources_dir=sources_dir) # pyright: ignore[reportCallIssue] + + +class FakeImportSources: + """Records every ``import_sources`` call instead of touching a DB.""" + + def __init__(self) -> None: + self.calls: list[dict] = [] + + async def __call__( + self, + sources: list[Path], + llm: object, + *, + prune: bool = False, + limit: int | None = None, + ) -> ImportSummary: + self.calls.append({"sources": list(sources), "prune": prune, "limit": limit}) + return ImportSummary(files=1, added=1) + + +def _fake_clone_factory() -> tuple[list[tuple[str, Path]], object]: + """A ``clone_or_pull`` that materialises a checkout with one .md file.""" + calls: list[tuple[str, Path]] = [] + + def fake_clone_or_pull(url: str, dest: Path | str) -> Path: + dest = Path(dest) + dest.mkdir(parents=True, exist_ok=True) + (dest / "notes.md").write_text(f"# {dest.name}\ncontent for the KB\n", encoding="utf-8") + calls.append((url, dest)) + return dest + + return calls, fake_clone_or_pull + + +# --- repo_name ------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("url", "name"), + [ + ("https://github.com/user/homelab.git", "homelab"), + ("https://github.com/user/homelab", "homelab"), + ("git@github.com:user/homelab.git", "homelab"), + ("git@github.com:homelab.git", "homelab"), + ("ssh://git@host:2222/group/deployments.git", "deployments"), + ("https://git.reeseapps.com:8443/proj/notes", "notes"), + ], +) +def test_repo_name(url: str, name: str) -> None: + assert import_docs.repo_name(url) == name + + +def test_repo_name_slug_fallback() -> None: + # No usable basename (path ends in the .git suffix itself) → slug. + assert import_docs.repo_name("https://host/.git") == "https-host" + + +# --- _resolve_sources ------------------------------------------------------ + + +def test_resolve_sources_git_urls_cloned_into_sources_dir( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + calls, fake = _fake_clone_factory() + monkeypatch.setattr(import_docs, "clone_or_pull", fake) + settings = _settings( + git_sources="https://host/a/homelab.git, git@host:user/deploy.git ,", + sources_dir=str(tmp_path / "bor"), + ) + + sources = import_docs._resolve_sources(None, settings) + + assert sources == [tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"] + assert calls == [ + ("https://host/a/homelab.git", tmp_path / "bor" / "homelab"), + ("git@host:user/deploy.git", tmp_path / "bor" / "deploy"), + ] + + +def test_resolve_sources_cli_source_wins( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + calls, fake = _fake_clone_factory() + monkeypatch.setattr(import_docs, "clone_or_pull", fake) + settings = _settings(git_sources="https://host/a/repo.git") + manual = tmp_path / "Manual" + + sources = import_docs._resolve_sources([manual], settings) + + assert sources == [manual] + assert calls == [] # git is never touched when --source is given + + +def test_resolve_sources_defaults_when_nothing_configured() -> None: + sources = import_docs._resolve_sources(None, _settings()) + assert sources == [p.expanduser() for p in import_docs.DEFAULT_SOURCES] + + +# --- main() ---------------------------------------------------------------- + + +def test_main_git_sources_clone_then_import( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + settings = _settings( + git_sources="https://host/a/homelab.git,https://host/a/deploy.git", + sources_dir=str(tmp_path / "bor"), + ) + monkeypatch.setattr(import_docs, "get_settings", lambda: settings) + calls, fake = _fake_clone_factory() + monkeypatch.setattr(import_docs, "clone_or_pull", fake) + fake_import = FakeImportSources() + monkeypatch.setattr(import_docs, "import_sources", fake_import) + + rc = import_docs.main([]) + + assert rc == 0 + assert [(url, dest) for url, dest in calls] == [ + ("https://host/a/homelab.git", tmp_path / "bor" / "homelab"), + ("https://host/a/deploy.git", tmp_path / "bor" / "deploy"), + ] + assert len(fake_import.calls) == 1 + # The cloned checkouts are exactly what gets imported. + assert fake_import.calls[0]["sources"] == [ + tmp_path / "bor" / "homelab", + tmp_path / "bor" / "deploy", + ] + for dest in (tmp_path / "bor" / "homelab", tmp_path / "bor" / "deploy"): + assert (dest / "notes.md").is_file() + # The final summary print reflects the import (added > 0). + assert "added=1" in capsys.readouterr().out + + +def test_main_cli_source_still_imports_manual_dir( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + manual = tmp_path / "manual" + manual.mkdir() + (manual / "a.md").write_text("# A\nhi\n", encoding="utf-8") + # BOR_GIT_SOURCES is set but must be ignored — --source always wins. + settings = _settings(git_sources="https://host/a/repo.git") + monkeypatch.setattr(import_docs, "get_settings", lambda: settings) + calls, fake = _fake_clone_factory() + monkeypatch.setattr(import_docs, "clone_or_pull", fake) + fake_import = FakeImportSources() + monkeypatch.setattr(import_docs, "import_sources", fake_import) + + rc = import_docs.main(["--source", str(manual)]) + + assert rc == 0 + assert calls == [] + assert fake_import.calls[0]["sources"] == [manual] + assert fake_import.calls[0]["prune"] is False + + +def test_main_git_failure_aborts_before_import( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + settings = _settings( + git_sources="https://host/a/bad.git", + sources_dir=str(tmp_path / "bor"), + ) + monkeypatch.setattr(import_docs, "get_settings", lambda: settings) + + def failing_clone(url: str, dest: Path | str) -> Path: + raise GitSyncError( + f"git clone --depth 1 {url} failed (exit 128): " + "fatal: repository not found" + ) + + monkeypatch.setattr(import_docs, "clone_or_pull", failing_clone) + fake_import = FakeImportSources() + monkeypatch.setattr(import_docs, "import_sources", fake_import) + + rc = import_docs.main([]) + + assert rc == 1 + err = capsys.readouterr().err + assert "import_docs: git sync failed" in err + assert "bad.git" in err # the failing repo is named + assert fake_import.calls == [] # no partial import + assert not (tmp_path / "bor").exists() diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index afa10e9..5fae9c6 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -92,6 +92,53 @@ def test_import_extensions_rejects_empty(monkeypatch) -> None: _settings() +def test_git_sources_default_empty_and_sources_dir_default() -> None: + """Phase 28: no git sources by default (backwards-compatible with the + ``--source`` / ``DEFAULT_SOURCES`` fallback); the clone location stays + a raw string (``~`` is expanded by the import script, not the setting).""" + s = _settings() + assert s.git_sources == "" + assert s.git_source_list == [] + assert s.sources_dir == "~/bor-sources" + + +def test_git_sources_env_override_parses_comma_separated_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``BOR_GIT_SOURCES`` is a raw CSV: entries are trimmed and empty + entries dropped; URLs are stored untouched (no scheme parsing here).""" + monkeypatch.setenv( + "BOR_GIT_SOURCES", + "https://github.com/user/homelab.git, " + " git@github.com:user/deployments.git ,, https://git.reeseapps.com/x/y.git ", + ) + s = _settings() + # The raw CSV string is preserved untouched (no parsing in the setting). + assert s.git_sources == ( + "https://github.com/user/homelab.git, " + " git@github.com:user/deployments.git ,, https://git.reeseapps.com/x/y.git " + ) + assert s.git_source_list == [ + "https://github.com/user/homelab.git", + "git@github.com:user/deployments.git", + "https://git.reeseapps.com/x/y.git", + ] + + +def test_git_sources_whitespace_only_yields_empty_list(monkeypatch) -> None: + """A configured-but-blank value behaves the same as unset: no git + sources, so the script falls back to its legacy local defaults.""" + monkeypatch.setenv("BOR_GIT_SOURCES", " , , ") + s = _settings() + assert s.git_source_list == [] + + +def test_sources_dir_env_override_is_raw_string(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BOR_SOURCES_DIR", "/data/bor/sources") + s = _settings() + assert s.sources_dir == "/data/bor/sources" + + def test_suggestions_default_is_three_plus_real_questions() -> None: s = _settings() assert len(s.suggestions) >= 3 diff --git a/tests/unit/test_git_sync.py b/tests/unit/test_git_sync.py new file mode 100644 index 0000000..0e9d578 --- /dev/null +++ b/tests/unit/test_git_sync.py @@ -0,0 +1,153 @@ +"""Unit tests: git clone/pull utility (phase 28). + +``clone_or_pull`` dispatches to ``git clone --depth 1`` (fresh dest) or +``git pull --ff-only`` (existing checkout) with ``subprocess`` fully +mocked — the real git CLI is never invoked, so the tests run anywhere. +""" +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +import scripts.git_sync as git_sync +from scripts.git_sync import GitSyncError, clone_or_pull + + +class _FakeProc: + def __init__(self, returncode: int = 0, stdout: str = "", stderr: str = "") -> None: + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +def _fake_run( + monkeypatch: pytest.MonkeyPatch, + returncode: int = 0, + stdout: str = "", + stderr: str = "", + missing_git: bool = False, +) -> list[dict[str, object]]: + """Monkeypatch scripts.git_sync.subprocess.run; record each call.""" + calls: list[dict[str, object]] = [] + + def fake_run(argv: list[str], cwd: Path | None = None, **kwargs: object) -> _FakeProc: + calls.append({"argv": list(argv), "cwd": cwd, **kwargs}) + if missing_git: + raise FileNotFoundError("git") + return _FakeProc(returncode, stdout, stderr) + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + +def test_clone_or_pull_clones_when_dest_has_no_git_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """First run: dest absent → ``git clone --depth 1`` from the parent dir.""" + url = "https://github.com/user/homelab.git" + dest = tmp_path / "homelab" + calls = _fake_run(monkeypatch) + + result = clone_or_pull(url, dest) + + assert result == dest + assert len(calls) == 1 + call = calls[0] + assert call["argv"] == ["git", "clone", "--depth", "1", url, str(dest)] + assert call["cwd"] == dest.parent + assert call["capture_output"] is True + assert call["text"] is True + + +def test_clone_or_pull_pulls_when_git_dir_exists( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Subsequent run: dest/.git present → ``git pull --ff-only`` in-place.""" + dest = tmp_path / "homelab" + (dest / ".git").mkdir(parents=True) + calls = _fake_run(monkeypatch) + + result = clone_or_pull("https://github.com/user/homelab.git", dest) + + assert result == dest + assert calls[0]["argv"] == ["git", "pull", "--ff-only"] + assert calls[0]["cwd"] == dest + + +def test_clone_or_pull_creates_missing_parent_before_clone( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The (nested) parent of the destination is mkdir'd before git runs.""" + dest = tmp_path / "nested" / "deeper" / "homelab" + assert not dest.parent.exists() + _fake_run(monkeypatch) + + clone_or_pull("https://github.com/user/homelab.git", dest) + + assert dest.parent.is_dir() + + +def test_existing_dir_without_git_dir_is_treated_as_fresh( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A leftover non-repo dir at the dest still takes the clone path — + git fails loudly (non-zero exit) if the dir is not empty.""" + dest = tmp_path / "homelab" + dest.mkdir() + calls = _fake_run(monkeypatch) + + clone_or_pull("https://github.com/user/homelab.git", dest) + + assert calls[0]["argv"] == [ + "git", + "clone", + "--depth", + "1", + "https://github.com/user/homelab.git", + str(dest), + ] + + +def test_failing_git_raises_error_carrying_stderr( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Non-zero exit → GitSyncError naming the subcommand, exit code, stderr.""" + dest = tmp_path / "homelab" + _fake_run(monkeypatch, returncode=128, stderr="fatal: repository not found\n") + + with pytest.raises( + GitSyncError, + match=r"git clone --depth 1 .* failed \(exit 128\): fatal: repository not found", + ): + clone_or_pull("https://example.com/nope.git", dest) + + # Same for the pull path (broken checkout, e.g. diverged history). + (dest / ".git").mkdir(parents=True) + _fake_run(monkeypatch, returncode=1, stderr="error: cannot pull with rebase") + + with pytest.raises( + GitSyncError, match=r"git pull --ff-only failed \(exit 1\): error: cannot pull with rebase" + ): + clone_or_pull("https://example.com/homelab.git", dest) + + +def test_missing_git_raises_named_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """No git binary on PATH → GitSyncError telling the user to install it.""" + _fake_run(monkeypatch, missing_git=True) + + with pytest.raises(GitSyncError, match="git was not found on PATH"): + clone_or_pull("https://example.com/homelab.git", tmp_path / "homelab") + + +def test_run_captures_and_returns_stdout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """_run returns the captured stdout on success (git output is not lost).""" + calls = _fake_run(monkeypatch, stdout="From example.com\n + abc..def main") + + assert git_sync._run(["git", "status"], cwd=tmp_path) == "From example.com\n + abc..def main" + assert len(calls) == 1