brain-of-reese/app/rag/git_sources.py

"""The shared source resolver (phase 35, task 03; local kind, phase 38).

One function, two callers: the in-app Sync pipeline
(``app/api/sync.py::_run_sync``) and the CLI
(``scripts/import_docs.py::_resolve_sources``) — both resolve the
``git_sources`` rows to import through :func:`effective_sources`, so
the admin-managed ``git_sources`` table is what actually gets cloned
(git rows) or walked directly (local rows) from either entry point
(the API's ``GET /api/git-sources`` fallback list is the only other
place that reads the env list — task 02).

Precedence (the phase's locked decision — the env var is demoted, not
removed): ``git_sources`` DB rows of **both kinds** win, in
``(added_at, id)`` order; ``BOR_GIT_SOURCES`` is a fallback only while
the table is empty (git URLs surface as synthetic ``kind='git'`` rows —
the env fallback is git-only); both empty → ``([], "env")`` and each
caller keeps its existing fail-loud behavior (sync: ``GitSyncError``;
CLI: the legacy ``DEFAULT_SOURCES`` fallback).

:func:`effective_git_sources` is kept as the phase-35 back-compat alias
(repo URLs of the effective git rows) so existing importers of the old
name keep working; new code calls :func:`effective_sources` and
branches on ``row.kind``.

Phase 121 (private git sources) adds the token mechanics next to the
resolver — three pure helpers, no DB of their own:

* :func:`sanitize_url` — the OUTPUT mask: strips the ``user:pass@``
  userinfo of ``https?://`` URLs so no API/UI surface ever shows an
  embedded credential (legacy rows included; the stored value is
  untouched — LOCKED A2);
* :func:`clone_url_for` — the CLONE-time credential: a row's
  ``token`` column is injected into the URL handed to git, and only
  there (NULL token → the bare stored URL verbatim);
* :func:`normalize_credential` — the WRITE-path normalizer: an
  old-style ``https://user:pass@host/repo.git`` URL pasted into the
  API is stored bare and the embedded credential is moved into the
  ``token`` column (an explicit ``token`` field wins — LOCKED A6).
"""
from __future__ import annotations

import logging
import re
from typing import Literal

from sqlalchemy import select
from sqlalchemy.orm import Session

from app.config import get_settings
from app.models import GitSource

logger = logging.getLogger(__name__)

#: Phase 121 — the userinfo component of an ``https?://`` URL: the
#: scheme, a run of one-or-more characters that are neither ``@`` nor
#: ``/`` (the ``user`` or ``user:pass`` part), and the terminating
#: ``@``. Deliberately a small anchored regex — never a URL parser
#: re-serialization: for a credential-free URL there is no match and
#: the input is returned byte-identical (the phase-50/35 contract that
#: stored URLs surface verbatim when they carry no credential).
_USERINFO_RE = re.compile(r"^(https?://)([^/@]+)@")


def sanitize_url(url: str) -> str:
    """Phase 121, LOCKED A2 — the token-free form of a source URL,
    for API/UI output only.

    Strips the userinfo component of ``https?://`` URLs
    (``https://user:pass@host/path`` → ``https://host/path``);
    ``ssh://``, ``git@`` (scp-style), and local paths are left
    untouched. Idempotent — and byte-identical for URLs that carry no
    userinfo (no match → the input unchanged, including a ``@`` inside
    the *path*, which is not userinfo). The stored row value is NOT
    modified: a legacy row whose credential is still embedded in
    ``url`` keeps cloning with its original stored URL; this is the
    output mask that keeps that credential out of every response and
    the UI (the env-fallback rows get the same treatment — the env
    *value* itself is untouched, only the response is masked).
    """
    return _USERINFO_RE.sub(r"\1", url, count=1)


def clone_url_for(row: GitSource) -> str:
    """Phase 121, LOCKED A2 — the URL git actually clones, with the
    row's credential injected ONLY here.

    * ``token`` NULL/falsy → ``row.url`` verbatim: public repos and
      local rows behave byte-identically to pre-phase-121, and a
      legacy embedded-token row (``token`` NULL, credential in the
      stored URL) keeps cloning with its ORIGINAL stored URL — the
      credential keeps working;
    * an ``https?://`` row with a token →
      ``https://x-access-token:<token>@<host>/<path>`` — any existing
      userinfo in the stored URL is replaced by the column credential
      (``x-access-token`` as the username: GitHub-agnostic, any host
      that accepts ``https://user:token@`` treats the first component
      opaquely — the task-02 assumption, task 02 step 4);
    * a non-https row with a token (``ssh://``/``git@``/local path)
      → ``row.url`` unchanged + a WARNING log (a token cannot
      authenticate ssh — the owner must use a deploy key/agent there;
      the log names the repo via its sanitized URL, never the token).

    ``repo_name`` (and every other checkout-path derivation) keeps
    operating on the bare ``row.url`` — the checkout directory name is
    credential-free.
    """
    token = row.token
    if not token:
        return row.url
    if not row.url.startswith(("https://", "http://")):
        logger.warning(
            "git source %s has a stored token but a non-https? URL — "
            "a token cannot authenticate ssh/git@ clones; the stored "
            "URL is used as-is (configure a deploy key or SSH agent "
            "for private ssh repos)",
            sanitize_url(row.url),
        )
        return row.url
    bare = sanitize_url(row.url)
    return bare.replace("://", f"://x-access-token:{token}@", 1)


def normalize_credential(url: str, token: str | None) -> tuple[str, str | None]:
    """Phase 121, LOCKED A6 — the write-path credential normalizer.

    If the (``https?://``-only) URL carries userinfo, it is stripped
    for storage and the EMBEDDED CREDENTIAL becomes the effective
    token — UNLESS the caller also sent an explicit ``token``
    (non-None), which WINS (explicit beats embedded — a blank masked
    field, i.e. an explicit "", is a deliberate "no credential").
    Pasting the old-style ``https://user:ghp_…@host/repo.git`` URL
    still works and lands token-column-clean; the caller stores the
    bare URL + ``effective_token or None`` (an empty explicit token
    stores NULL) and runs its duplicate check on the BARE URL, so the
    same repo with a different token is still the same source
    (409, not a second row).

    The embedded credential is the *password* part of a
    ``user:pass`` userinfo (after the first colon — the password may
    contain further colons), or the whole userinfo run for the
    username-as-token form (``https://<token>@host/…``, the documented
    GitHub shape, no colon). Clean URLs and ``ssh://``/``git@``/local
    paths return ``(url, token)`` untouched — byte-identical
    pre-phase behavior.
    """
    match = _USERINFO_RE.match(url)
    if match is None:
        return url, token
    userinfo = match.group(2)
    user, sep, password = userinfo.partition(":")
    embedded = password if sep else userinfo
    effective = token if token is not None else embedded
    return sanitize_url(url), effective


def effective_sources(db: Session) -> tuple[list[GitSource], Literal["db", "env"]]:
    """``(rows, origin)`` — the effective source rows (both kinds) and
    where they came from.

    ``"db"``: the ``git_sources`` rows — ``kind='git'`` (repo URL to
    clone/pull) and ``kind='local'`` (existing directory to walk
    directly) — in ``(added_at, id)`` order (oldest first, id tie-break
    for same-timestamp inserts). Once the table has any row,
    ``BOR_GIT_SOURCES`` is ignored entirely.
    ``"env"``: the ``BOR_GIT_SOURCES`` list — ``Settings.git_source_list``,
    the phase-28 CSV parse, reused not re-implemented — surfaced as
    synthetic ``kind='git'`` rows (``path`` NULL; the env fallback is
    git-only), used only while the table is empty; both empty →
    ``([], "env")``.
    """
    rows = db.scalars(
        select(GitSource).order_by(GitSource.added_at.asc(), GitSource.id.asc())
    ).all()
    if rows:
        return list(rows), "db"
    return [GitSource(url=url, kind="git") for url in get_settings().git_source_list], "env"


def effective_git_sources(db: Session) -> tuple[list[str], Literal["db", "env"]]:
    """Back-compat alias for the phase-35 name (existing importers).

    Returns the repo URLs of the effective ``kind='git'`` rows only, with
    the same origin label — the env fallback is git-only, so its
    behavior is unchanged. Local rows carry no URL (they are walked
    directly); new code should call :func:`effective_sources` and branch
    on ``row.kind``.
    """
    rows, origin = effective_sources(db)
    return [row.url for row in rows if row.kind == "git"], origin