brain-of-reese/app/rag/git_sources.py

"""The shared source resolver (phase 35, task 03; local kind, phase 38).

One function, two callers: the in-app Sync pipeline
(``app/api/sync.py::_run_sync``) and the CLI
(``scripts/import_docs.py::_resolve_sources``) — both resolve the
``git_sources`` rows to import through :func:`effective_sources`, so
the admin-managed ``git_sources`` table is what actually gets cloned
(git rows) or walked directly (local rows) from either entry point
(the API's ``GET /api/git-sources`` fallback list is the only other
place that reads the env list — task 02).

Precedence (the phase's locked decision — the env var is demoted, not
removed): ``git_sources`` DB rows of **both kinds** win, in
``(added_at, id)`` order; ``BOR_GIT_SOURCES`` is a fallback only while
the table is empty (git URLs surface as synthetic ``kind='git'`` rows —
the env fallback is git-only); both empty → ``([], "env")`` and each
caller keeps its existing fail-loud behavior (sync: ``GitSyncError``;
CLI: the legacy ``DEFAULT_SOURCES`` fallback).

:func:`effective_git_sources` is kept as the phase-35 back-compat alias
(repo URLs of the effective git rows) so existing importers of the old
name keep working; new code calls :func:`effective_sources` and
branches on ``row.kind``.

Phase 121 (private git sources) adds the token mechanics next to the
resolver — three pure helpers, no DB of their own:

* :func:`sanitize_url` — the OUTPUT mask: strips the ``user:pass@``
  userinfo of ``https?://`` URLs so no API/UI surface ever shows an
  embedded credential (legacy rows included; the stored value is
  untouched — LOCKED A2);
* :func:`clone_url_for` — the CLONE-time credential: a row's
  ``token`` column is injected into the URL handed to git, and only
  there (NULL token → the bare stored URL verbatim);
* :func:`normalize_credential` — the WRITE-path normalizer: an
  old-style ``https://user:pass@host/repo.git`` URL pasted into the
  API is stored bare and the embedded credential is moved into the
  ``token`` column (an explicit ``token`` field wins — LOCKED A6).

Phase 135 (per-source branch pinning) adds the write-path branch
validator next to them — :func:`validate_branch_name`, the single
choke point: the API maps its ``ValueError`` to the fixed-detail 422
(the raw input is never echoed, the house credential-safety
discipline, applied for consistency).
"""
from __future__ import annotations

import logging
import re
from typing import Literal

from sqlalchemy import select
from sqlalchemy.orm import Session

from app.config import get_settings
from app.models import GitSource

logger = logging.getLogger(__name__)

#: Phase 121 — the userinfo component of an ``https?://`` URL: the
#: scheme, a run of one-or-more characters that are neither ``@`` nor
#: ``/`` (the ``user`` or ``user:pass`` part), and the terminating
#: ``@``. Deliberately a small anchored regex — never a URL parser
#: re-serialization: for a credential-free URL there is no match and
#: the input is returned byte-identical (the phase-50/35 contract that
#: stored URLs surface verbatim when they carry no credential).
_USERINFO_RE = re.compile(r"^(https?://)([^/@]+)@")


def sanitize_url(url: str) -> str:
    """Phase 121, LOCKED A2 — the token-free form of a source URL,
    for API/UI output only.

    Strips the userinfo component of ``https?://`` URLs
    (``https://user:pass@host/path`` → ``https://host/path``);
    ``ssh://``, ``git@`` (scp-style), and local paths are left
    untouched. Idempotent — and byte-identical for URLs that carry no
    userinfo (no match → the input unchanged, including a ``@`` inside
    the *path*, which is not userinfo). The stored row value is NOT
    modified: a legacy row whose credential is still embedded in
    ``url`` keeps cloning with its original stored URL; this is the
    output mask that keeps that credential out of every response and
    the UI (the env-fallback rows get the same treatment — the env
    *value* itself is untouched, only the response is masked).
    """
    return _USERINFO_RE.sub(r"\1", url, count=1)


def _url_basename(url: str) -> str:
    """Phase 145 (task 01) — the repository directory a git row's URL
    imports into.

    The URL's path component — after the ``://``, or after the ``:``
    for scp-style ``git@host:owner/repo`` (the port-carrying
    ``ssh://user@host:22/…`` form already fell into the ``://``
    branch) — with trailing ``/`` stripped, a single trailing ``.git``
    suffix stripped, and the last segment taken. The importer clones
    into (and names the source after) exactly this basename
    (``source = root.name``), so it is the join key between a
    document's ``source`` name and the ``git_sources`` row whose clone
    produced it. An ``@`` inside the *path* is not userinfo and is
    never treated as one (no split, no strip).
    """
    rest = url
    if "://" in rest:
        rest = rest.partition("://")[2]
    elif ":" in rest:
        # scp-style ``git@host:owner/repo`` — the path component starts
        # after the colon (``host:port`` has no other form here: the
        # only colon-carrying scheme without ``//`` is the scp one).
        rest = rest.partition(":")[2]
    rest = rest.rstrip("/")
    if rest.endswith(".git"):
        rest = rest[: -len(".git")]
    return rest.rsplit("/", 1)[-1]


def origin_for_source(db: Session, source_name: str) -> tuple[str, str | None] | None:
    """Phase 145 (task 01) — the origin a read result of *source_name*
    documents carries: the sanitized repository URL (and the pinned
    branch, when one exists), or ``None``.

    The candidate rows are the ``kind='git'`` rows (local rows are
    never origins — nothing to link to from the user's browser); a row
    matches when its URL basename (:func:`_url_basename` — the clone
    directory the importer names the source after) equals
    ``source_name``.

    * EXACTLY ONE match → ``(sanitize_url(row.url), row.branch)`` —
      the phase-121 output mask keeps a legacy embedded-credential
      row's secret out of the prompt (the stored value is never
      modified — the :func:`clone_url_for` lesson), and ``branch`` is
      the pinned branch or ``None`` (NULL = the default branch);
    * zero matches, or TWO-OR-MORE (two repos sharing a basename) →
      ``None`` + a WARNING log (the ambiguity names the sanitized
      candidates) — never guess: the read result would otherwise
      carry a wrong origin into every answer about the document, and
      a ``None`` result is byte-identical to pre-phase.

    Pure-DB helper — one query, no git subprocess
    (``scripts/git_sync.py`` stays the only git-invocation site, PLAN
    §13), no caching (the table is a handful of rows; a read's extra
    query is trivial).
    """
    rows = db.scalars(select(GitSource).where(GitSource.kind == "git")).all()
    candidates = [row for row in rows if _url_basename(row.url) == source_name]
    if len(candidates) == 1:
        row = candidates[0]
        return sanitize_url(row.url), row.branch
    if not candidates:
        logger.warning(
            "no git origin for source %r: no kind='git' row's URL basename matches",
            source_name,
        )
        return None
    logger.warning(
        "ambiguous git origin for source %r: %d rows share the URL basename (%s) — "
        "no origin line is added to its read results",
        source_name,
        len(candidates),
        ", ".join(sanitize_url(row.url) for row in candidates),
    )
    return None


def clone_url_for(row: GitSource) -> str:
    """Phase 121, LOCKED A2 — the URL git actually clones, with the
    row's credential injected ONLY here.

    * ``token`` NULL/falsy → ``row.url`` verbatim: public repos and
      local rows behave byte-identically to pre-phase-121, and a
      legacy embedded-token row (``token`` NULL, credential in the
      stored URL) keeps cloning with its ORIGINAL stored URL — the
      credential keeps working;
    * an ``https?://`` row with a token →
      ``https://x-access-token:<token>@<host>/<path>`` — any existing
      userinfo in the stored URL is replaced by the column credential
      (``x-access-token`` as the username: GitHub-agnostic, any host
      that accepts ``https://user:token@`` treats the first component
      opaquely — the task-02 assumption, task 02 step 4);
    * a non-https row with a token (``ssh://``/``git@``/local path)
      → ``row.url`` unchanged + a WARNING log (a token cannot
      authenticate ssh — the owner must use a deploy key/agent there;
      the log names the repo via its sanitized URL, never the token).

    ``repo_name`` (and every other checkout-path derivation) keeps
    operating on the bare ``row.url`` — the checkout directory name is
    credential-free.
    """
    token = row.token
    if not token:
        return row.url
    if not row.url.startswith(("https://", "http://")):
        logger.warning(
            "git source %s has a stored token but a non-https? URL — "
            "a token cannot authenticate ssh/git@ clones; the stored "
            "URL is used as-is (configure a deploy key or SSH agent "
            "for private ssh repos)",
            sanitize_url(row.url),
        )
        return row.url
    bare = sanitize_url(row.url)
    return bare.replace("://", f"://x-access-token:{token}@", 1)


def normalize_credential(url: str, token: str | None) -> tuple[str, str | None]:
    """Phase 121, LOCKED A6 — the write-path credential normalizer.

    If the (``https?://``-only) URL carries userinfo, it is stripped
    for storage and the EMBEDDED CREDENTIAL becomes the effective
    token — UNLESS the caller also sent an explicit ``token``
    (non-None), which WINS (explicit beats embedded — a blank masked
    field, i.e. an explicit "", is a deliberate "no credential").
    Pasting the old-style ``https://user:ghp_…@host/repo.git`` URL
    still works and lands token-column-clean; the caller stores the
    bare URL + ``effective_token or None`` (an empty explicit token
    stores NULL) and runs its duplicate check on the BARE URL, so the
    same repo with a different token is still the same source
    (409, not a second row).

    The embedded credential is the *password* part of a
    ``user:pass`` userinfo (after the first colon — the password may
    contain further colons), or the whole userinfo run for the
    username-as-token form (``https://<token>@host/…``, the documented
    GitHub shape, no colon). Clean URLs and ``ssh://``/``git@``/local
    paths return ``(url, token)`` untouched — byte-identical
    pre-phase behavior.
    """
    match = _USERINFO_RE.match(url)
    if match is None:
        return url, token
    userinfo = match.group(2)
    user, sep, password = userinfo.partition(":")
    embedded = password if sep else userinfo
    effective = token if token is not None else embedded
    return sanitize_url(url), effective


#: Phase 135 (ASSUMPTION B2) — the safe branch-name charset:
#: alphanumerics + dot, underscore, hyphen, and slash (a slash allows
#: ``release/1.2``-style names — git branch names are paths). Anything
#: else (whitespace, ``@``, ``~``, ``^``, ``:``, ``?``, ``*``, ``[``,
#: backslash, …) is rejected by the character-set check.
_BRANCH_RE = re.compile(r"^[A-Za-z0-9._/-]+$")


def validate_branch_name(raw: str) -> str:
    """Trim + validate a branch name (phase 135, ASSUMPTION B2).

    Returns the trimmed name; raises ``ValueError`` (fixed reason
    strings, for logs only — the API maps to the fixed-detail 422 and
    the raw input is never echoed) when the name is empty, > 200
    chars, outside the safe charset (``[A-Za-z0-9._/-]+``), contains
    ``..`` or ``//``, starts with ``-``/``.``/``/``, ends with ``/``
    or ``.``, or is ``HEAD`` (case-insensitive — git's refname; a
    branch literally named ``head`` is a documented cost of the simple
    rule, the plan's design note).

    Pure — no DB, no logging of the raw value (the reason strings are
    fixed, input-free: the same discipline that keeps credentials out
    of the 422 details, applied here for consistency).
    """
    name = raw.strip()
    if not name:
        raise ValueError("branch name must not be empty")
    if len(name) > 200:
        raise ValueError("branch name exceeds 200 characters")
    if not _BRANCH_RE.match(name):
        raise ValueError("branch name contains invalid characters")
    if ".." in name or "//" in name:
        raise ValueError("branch name contains a disallowed path sequence")
    if name[0] in ("-", ".", "/"):
        raise ValueError("branch name has a disallowed leading character")
    if name[-1] in ("/", "."):
        raise ValueError("branch name has a disallowed trailing character")
    if name.upper() == "HEAD":
        # Fixed reason, no echo: the input here is the reserved name
        # itself, and the reason must stay input-free (the log-line
        # discipline — a typed name never surfaces, reserved or not).
        raise ValueError("branch name is reserved")
    return name


def effective_sources(db: Session) -> tuple[list[GitSource], Literal["db", "env"]]:
    """``(rows, origin)`` — the effective source rows (both kinds) and
    where they came from.

    ``"db"``: the ``git_sources`` rows — ``kind='git'`` (repo URL to
    clone/pull) and ``kind='local'`` (existing directory to walk
    directly) — in ``(added_at, id)`` order (oldest first, id tie-break
    for same-timestamp inserts). Once the table has any row,
    ``BOR_GIT_SOURCES`` is ignored entirely.
    ``"env"``: the ``BOR_GIT_SOURCES`` list — ``Settings.git_source_list``,
    the phase-28 CSV parse, reused not re-implemented — surfaced as
    synthetic ``kind='git'`` rows (``path`` NULL; the env fallback is
    git-only), used only while the table is empty; both empty →
    ``([], "env")``.
    """
    rows = db.scalars(
        select(GitSource).order_by(GitSource.added_at.asc(), GitSource.id.asc())
    ).all()
    if rows:
        return list(rows), "db"
    return [GitSource(url=url, kind="git") for url in get_settings().git_source_list], "env"


def effective_git_sources(db: Session) -> tuple[list[str], Literal["db", "env"]]:
    """Back-compat alias for the phase-35 name (existing importers).

    Returns the repo URLs of the effective ``kind='git'`` rows only, with
    the same origin label — the env fallback is git-only, so its
    behavior is unchanged. Local rows carry no URL (they are walked
    directly); new code should call :func:`effective_sources` and branch
    on ``row.kind``.
    """
    rows, origin = effective_sources(db)
    return [row.url for row in rows if row.kind == "git"], origin