phase: 121_git_source_tokens
**Phase 121 final verification pass — all green** (all 4 tasks already in `complete/`; verified, no defects found, no changes needed) - Verified implementation vs phase design: migration `0021` (reversible, round-tripped via `alembic downgrade base` + `upgrade head` → head `0021`), `GitSource.token` column, `normalize_credential`/`clone_url_for`/`sanitize_url`, clone callers switched (`sync.py`, `import_docs.py`), masked token fields in add form + editor, `extra="forbid"` output shapes - Tests: `uv run pytest` → 2662 passed, 0 failed (exit 0); `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (≥90% gate) - Lint/types: `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors, 0 warnings - E2E in isolation: `uv run pytest tests/e2e/test_git_source_tokens.py -v --no-cov` → **4 passed** Completion criteria: 1. Private repo (UI add or pasted embedded-token URL) clones with injected token; token absent from every API response, page text, title attr, and full HTML — **PASS** (integration raw-JSON assertions + E2E `_assert_token_nowhere`) 2. Legacy embedded-token rows still clone from stored URL; output sanitized — **PASS** (`test_sync_legacy_row_clones_with_original_stored_url`, `test_get_masks_legacy_embedded_token_row`, env-fallback masking) 3. Public/local sources byte-identical — **PASS** (verbatim-URL + no-userinfo-unchanged tests) 4. pytest / coverage / ruff / pyright — **PASS** (see above) 5. Commit + phase move — harness responsibility; task files already in `complete/`, changes left in working tree (no commit made, per protocol) Notable: no deviations; DB left at head, functional. Next pending phase: **122_image_documents** (then 123_chat_image_questions).
This commit is contained in:
@@ -45,9 +45,23 @@ Contract under test:
|
||||
unchanged.
|
||||
|
||||
``git_sources`` is global state: truncated around every test.
|
||||
|
||||
Phase 121 (task 02 — clone-time credential + output sanitization):
|
||||
POST normalizes an old-style embedded ``user:pass@`` URL into the bare
|
||||
URL + ``token`` column (explicit ``token`` wins — LOCKED A6), the
|
||||
duplicate check runs on the bare URL, the PATCH token is tri-state
|
||||
(absent/None = no change, non-empty = replace, "" = clear) and
|
||||
re-normalizes a legacy row's URL, and every response (list DB rows,
|
||||
list env rows, POST 201, PATCH 200) is token-free — asserted on the
|
||||
RAW response text — while a token row's SYNC clone receives the
|
||||
injected ``https://x-access-token:<token>@…`` URL with a
|
||||
credential-free checkout path (mock ``clone_or_pull``) and a legacy
|
||||
row's clone still receives its ORIGINAL stored URL (the credential
|
||||
keeps working).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -60,8 +74,12 @@ from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api import git_sources as git_sources_api
|
||||
from app.api import sync as sync_api
|
||||
from app.config import Settings
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import GitSource
|
||||
from app.rag import git_sources as git_sources_resolver
|
||||
from app.rag.importer import ImportSummary
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -949,3 +967,457 @@ def test_patch_422_does_not_half_apply_flag(admin_client: TestClient) -> None:
|
||||
row = admin_client.get("/api/git-sources").json()["sources"][0]
|
||||
assert row["ignore_paths"] == ["keep"]
|
||||
assert row["include_hidden"] is False
|
||||
|
||||
|
||||
# --- token: write-path normalization (phase 121, task 02) ------------------
|
||||
|
||||
TOKEN = "ghp_phase121secrettoken"
|
||||
|
||||
|
||||
def test_post_with_token_stores_column_and_never_echoes(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""Bare URL + masked token → the token is stored in the dedicated
|
||||
column and appears in NO API response — asserted on the RAW text of
|
||||
both the 201 and the GET list (the response shapes have no token
|
||||
field by contract; this pins that nothing else smuggles it out)."""
|
||||
r = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"url": "https://github.com/owner/private.git", "token": TOKEN},
|
||||
)
|
||||
assert r.status_code == 201, r.text
|
||||
body = r.json()
|
||||
assert body["url"] == "https://github.com/owner/private.git"
|
||||
assert set(body) == {"id", "url", "added_at", "ignore_paths", "include_hidden"}
|
||||
assert TOKEN not in r.text
|
||||
|
||||
row = db.scalars(select(GitSource)).one()
|
||||
assert row.url == "https://github.com/owner/private.git" # bare
|
||||
assert row.token == TOKEN # the column, not the URL
|
||||
|
||||
r = admin_client.get("/api/git-sources")
|
||||
assert r.status_code == 200
|
||||
assert TOKEN not in r.text
|
||||
assert r.json()["sources"][0]["url"] == "https://github.com/owner/private.git"
|
||||
|
||||
|
||||
def test_post_embedded_token_url_normalizes_to_bare_plus_column(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""The TODO L5 paste — old-style ``user:token@`` URL with no token
|
||||
field: stored BARE, the embedded PASSWORD part moved to the token
|
||||
column, and every response is token-free (raw text)."""
|
||||
r = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"url": f"https://myuser:{TOKEN}@github.com/owner/private-repo.git"},
|
||||
)
|
||||
assert r.status_code == 201, r.text
|
||||
assert r.json()["url"] == "https://github.com/owner/private-repo.git"
|
||||
assert TOKEN not in r.text
|
||||
assert "myuser" not in r.text
|
||||
|
||||
row = db.scalars(select(GitSource)).one()
|
||||
assert row.url == "https://github.com/owner/private-repo.git"
|
||||
assert row.token == TOKEN # the password part, not the whole userinfo
|
||||
|
||||
r = admin_client.get("/api/git-sources")
|
||||
assert TOKEN not in r.text
|
||||
assert r.json()["sources"][0]["url"] == "https://github.com/owner/private-repo.git"
|
||||
|
||||
|
||||
def test_post_explicit_token_wins_over_embedded(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""LOCKED A6 — the masked field and the pasted URL disagree: the
|
||||
explicit field is the intent and beats the embedded credential."""
|
||||
r = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={
|
||||
"url": f"https://myuser:{TOKEN}@github.com/owner/private.git",
|
||||
"token": "ghp_explicitwins",
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201, r.text
|
||||
assert r.json()["url"] == "https://github.com/owner/private.git"
|
||||
|
||||
row = db.scalars(select(GitSource)).one()
|
||||
assert row.url == "https://github.com/owner/private.git"
|
||||
assert row.token == "ghp_explicitwins"
|
||||
|
||||
|
||||
def test_post_username_as_token_form_moves_whole_run(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""The documented GitHub shape (no colon) — the whole userinfo run
|
||||
is the credential."""
|
||||
r = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"url": f"https://{TOKEN}@github.com/owner/private.git"},
|
||||
)
|
||||
assert r.status_code == 201, r.text
|
||||
row = db.scalars(select(GitSource)).one()
|
||||
assert row.url == "https://github.com/owner/private.git"
|
||||
assert row.token == TOKEN
|
||||
|
||||
|
||||
def test_post_same_repo_different_token_is_409_on_bare_url(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""The duplicate check runs on the NORMALIZED bare URL: the same
|
||||
repo pasted with a different credential is the same source — 409,
|
||||
not a second row (both via embedded and via explicit token)."""
|
||||
assert (
|
||||
admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"url": f"https://user1:{TOKEN}@github.com/owner/dup.git"},
|
||||
).status_code
|
||||
== 201
|
||||
)
|
||||
r = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"url": "https://user2:other-cred@github.com/owner/dup.git"},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == "a git source with this URL already exists"
|
||||
assert TOKEN not in r.text
|
||||
# And the bare form of the same repo 409s too.
|
||||
r = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"url": "https://github.com/owner/dup.git", "token": "another"},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert db.execute(text("SELECT count(*) FROM git_sources")).scalar_one() == 1
|
||||
|
||||
|
||||
def test_post_local_kind_token_is_inert_and_never_echoed(
|
||||
admin_client: TestClient, tmp_path: Path, db: Session
|
||||
) -> None:
|
||||
"""Local-kind sources have no URL credential (the design does not
|
||||
touch them): a token on a local row is stored inert (never used —
|
||||
local rows are walked, not cloned) and, like on git rows, never
|
||||
echoed."""
|
||||
real_dir = tmp_path / "local-tok"
|
||||
real_dir.mkdir()
|
||||
r = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"kind": "local", "path": str(real_dir), "token": TOKEN},
|
||||
)
|
||||
assert r.status_code == 201, r.text
|
||||
assert TOKEN not in r.text
|
||||
assert r.json()["url"] == str(real_dir)
|
||||
|
||||
row = db.scalars(select(GitSource)).one()
|
||||
assert row.token == TOKEN # inert — clone_url_for never sees it
|
||||
assert TOKEN not in admin_client.get("/api/git-sources").text
|
||||
|
||||
|
||||
# --- token: PATCH tri-state (phase 121, task 02) ---------------------------
|
||||
|
||||
|
||||
def test_patch_token_replace_clear_and_noop(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""The tri-state on a clean-URL row: absent/None = no change,
|
||||
non-empty = replace, "" = clear (stored NULL) — the token is never
|
||||
in any response (raw text)."""
|
||||
created = admin_client.post(
|
||||
"/api/git-sources",
|
||||
json={"url": "https://github.com/owner/patch.git", "token": TOKEN},
|
||||
)
|
||||
assert created.status_code == 201
|
||||
sid = created.json()["id"]
|
||||
|
||||
# Absent (and explicit None) = no change.
|
||||
# The endpoint commits in its own session — expire this one's
|
||||
# identity map before reading the row back (the house pattern for
|
||||
# cross-session reads).
|
||||
def stored() -> GitSource:
|
||||
db.expire_all()
|
||||
row = db.get(GitSource, uuid.UUID(sid))
|
||||
assert row is not None
|
||||
return row
|
||||
|
||||
r = admin_client.patch(f"/api/git-sources/{sid}", json={"ignore_paths": ["a"]})
|
||||
assert r.status_code == 200, r.text
|
||||
assert TOKEN not in r.text
|
||||
assert r.json()["url"] == "https://github.com/owner/patch.git"
|
||||
assert stored().token == TOKEN
|
||||
|
||||
r = admin_client.patch(
|
||||
f"/api/git-sources/{sid}", json={"ignore_paths": None, "token": None}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert stored().token == TOKEN
|
||||
|
||||
# Non-empty = replace (the URL is untouched — a clean URL comes
|
||||
# back byte-identical).
|
||||
r = admin_client.patch(f"/api/git-sources/{sid}", json={"token": "ghp_replaced"})
|
||||
assert r.status_code == 200, r.text
|
||||
assert TOKEN not in r.text
|
||||
assert "ghp_replaced" not in r.text
|
||||
assert r.json()["url"] == "https://github.com/owner/patch.git"
|
||||
row = stored()
|
||||
assert row.url == "https://github.com/owner/patch.git"
|
||||
assert row.token == "ghp_replaced"
|
||||
|
||||
# "" = clear — stored NULL.
|
||||
r = admin_client.patch(f"/api/git-sources/{sid}", json={"token": ""})
|
||||
assert r.status_code == 200, r.text
|
||||
row = stored()
|
||||
assert row.token is None
|
||||
assert r.json()["url"] == "https://github.com/owner/patch.git"
|
||||
|
||||
|
||||
def test_patch_token_on_legacy_row_renormalizes_url(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""A pre-phase row (credential embedded in the stored URL, token
|
||||
NULL) gets its userinfo stripped (moved to the column) the first
|
||||
time an explicit credential is written — and a "" clear also moves
|
||||
it (a cleared credential is gone from both the column and the URL;
|
||||
the owner explicitly asked for no credential)."""
|
||||
legacy = f"https://user:{TOKEN}@github.com/owner/legacy.git"
|
||||
db.add(GitSource(url=legacy))
|
||||
db.commit()
|
||||
row = db.scalars(select(GitSource)).one()
|
||||
sid = row.id
|
||||
|
||||
r = admin_client.patch(f"/api/git-sources/{sid}", json={"token": "ghp_new"})
|
||||
assert r.status_code == 200, r.text
|
||||
assert TOKEN not in r.text
|
||||
assert "ghp_new" not in r.text
|
||||
assert r.json()["url"] == "https://github.com/owner/legacy.git"
|
||||
db.expire_all() # the endpoint committed in its own session
|
||||
row = db.get(GitSource, sid)
|
||||
assert row is not None
|
||||
assert row.url == "https://github.com/owner/legacy.git" # normalized bare
|
||||
assert row.token == "ghp_new"
|
||||
|
||||
# "" clears the (now column) credential — the URL stays bare.
|
||||
r = admin_client.patch(f"/api/git-sources/{sid}", json={"token": ""})
|
||||
assert r.status_code == 200, r.text
|
||||
db.expire_all()
|
||||
row = db.get(GitSource, sid)
|
||||
assert row is not None
|
||||
assert row.url == "https://github.com/owner/legacy.git"
|
||||
assert row.token is None
|
||||
|
||||
|
||||
def test_patch_token_409_backstop_on_renormalized_collision(
|
||||
admin_client: TestClient, db: Session
|
||||
) -> None:
|
||||
"""A legacy embedded row + a bare row for the same repo can only
|
||||
coexist pre-phase; re-normalizing the legacy row's URL on a token
|
||||
PATCH makes the stored URLs collide — the unique index yields the
|
||||
generic 409 (never a 500) and the failed PATCH leaves the row
|
||||
untouched."""
|
||||
legacy = f"https://user:{TOKEN}@github.com/owner/collide.git"
|
||||
bare = "https://github.com/owner/collide.git"
|
||||
db.add_all([GitSource(url=legacy), GitSource(url=bare)])
|
||||
db.commit()
|
||||
legacy_row = db.scalars(select(GitSource).where(GitSource.url == legacy)).one()
|
||||
|
||||
r = admin_client.patch(f"/api/git-sources/{legacy_row.id}", json={"token": "ghp_x"})
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == "a git source with this URL already exists"
|
||||
assert TOKEN not in r.text
|
||||
# The rollback left the legacy row exactly as it was.
|
||||
db.expire_all()
|
||||
row = db.get(GitSource, legacy_row.id)
|
||||
assert row is not None
|
||||
assert row.url == legacy
|
||||
assert row.token is None
|
||||
|
||||
|
||||
# --- token: output sanitization (phase 121, task 02) ------------------------
|
||||
|
||||
|
||||
def test_get_masks_legacy_embedded_token_row(admin_client: TestClient, db: Session) -> None:
|
||||
"""Completion criterion: a legacy row (token embedded in the
|
||||
stored URL, token column NULL) clones with its original URL but
|
||||
its API output is token-free — the stored DB value is untouched,
|
||||
the response is bare."""
|
||||
legacy = f"https://user:{TOKEN}@github.com/owner/legacy.git"
|
||||
db.add(GitSource(url=legacy))
|
||||
db.commit()
|
||||
|
||||
r = admin_client.get("/api/git-sources")
|
||||
assert r.status_code == 200
|
||||
assert TOKEN not in r.text
|
||||
assert "user:" not in r.text
|
||||
assert r.json()["sources"][0]["url"] == "https://github.com/owner/legacy.git"
|
||||
# The stored value is untouched (the clone still authenticates).
|
||||
assert db.scalars(select(GitSource)).one().url == legacy
|
||||
|
||||
|
||||
def test_get_masks_env_fallback_rows_with_embedded_token(
|
||||
admin_client: TestClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An env URL can embed a token too — the ENV VALUE itself is
|
||||
untouched (the config is the operator's), only the response is
|
||||
masked."""
|
||||
monkeypatch.setattr(
|
||||
git_sources_api,
|
||||
"get_settings",
|
||||
lambda: _settings(f"https://user:{TOKEN}@env.example.com/env.git"),
|
||||
)
|
||||
r = admin_client.get("/api/git-sources")
|
||||
assert r.status_code == 200
|
||||
assert TOKEN not in r.text
|
||||
assert r.json()["from_env"] is True
|
||||
assert r.json()["sources"][0]["url"] == "https://env.example.com/env.git"
|
||||
|
||||
|
||||
def test_get_clean_urls_are_byte_identical(admin_client: TestClient, db: Session) -> None:
|
||||
"""The phase-50/35 contract through the mask: credential-free
|
||||
stored URLs (including ``git@`` and local paths) surface verbatim."""
|
||||
urls = [
|
||||
"https://github.com/owner/clean.git",
|
||||
"git@github.com:owner/scp.git",
|
||||
"ssh://git@example.com/repo.git",
|
||||
]
|
||||
base = datetime.now(UTC) - timedelta(hours=1)
|
||||
# Distinct added_at: same-timestamp rows order by random uuid4 id.
|
||||
db.add_all(
|
||||
GitSource(url=u, added_at=base + timedelta(minutes=i)) for i, u in enumerate(urls)
|
||||
)
|
||||
db.commit()
|
||||
body = admin_client.get("/api/git-sources").json()
|
||||
assert [s["url"] for s in body["sources"]] == urls
|
||||
|
||||
|
||||
# --- token: the sync clone URL (phase 121, task 02) -------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def sync_admin_client() -> Iterator[TestClient]:
|
||||
"""A context-managed, logged-in client — one app event loop across
|
||||
requests (the sync background task must survive between the POST
|
||||
and the polls; the test_sync_api.py pattern)."""
|
||||
with TestClient(fastapi_app) as client:
|
||||
r = client.post("/api/login", json={"password": "test-admin-password"})
|
||||
assert r.status_code == 204
|
||||
yield client
|
||||
|
||||
|
||||
def _stub_sync_stack(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> list[tuple[str, Path]]:
|
||||
"""The sync pipeline minus the real git/import/LLM layers (the
|
||||
test_sync_api.py pattern): a fresh settings dir, an empty env list
|
||||
(the DB row wins), a no-op model probe, a zero-count import
|
||||
(change-gated steps skipped), and a recording clone.
|
||||
Returns the clone call list."""
|
||||
monkeypatch.setattr(
|
||||
sync_api,
|
||||
"get_settings",
|
||||
lambda: Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
sources_dir=str(tmp_path / "sources"),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
git_sources_resolver,
|
||||
"get_settings",
|
||||
lambda: Settings(_env_file=None, git_sources=""), # pyright: ignore[reportCallIssue]
|
||||
)
|
||||
|
||||
async def fake_check_models(llm: object) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(sync_api, "check_models", fake_check_models)
|
||||
|
||||
clone_calls: list[tuple[str, Path]] = []
|
||||
|
||||
def fake_clone_or_pull(url: str, dest: Path | str) -> Path:
|
||||
dest = Path(dest)
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
clone_calls.append((url, dest))
|
||||
return dest
|
||||
|
||||
monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone_or_pull)
|
||||
|
||||
async def fake_import_sources(*args: object, **kwargs: object) -> ImportSummary:
|
||||
return ImportSummary() # all-zero: the change-gated steps skip
|
||||
|
||||
monkeypatch.setattr(sync_api, "import_sources", fake_import_sources)
|
||||
# The unchanged-walk gap probe: no candidates → no generator call
|
||||
# (hermetic — the real probe reads the global KB state, and the
|
||||
# generator would burn real lite calls).
|
||||
monkeypatch.setattr(sync_api, "missing_folder_summaries", lambda db: [])
|
||||
return clone_calls
|
||||
|
||||
|
||||
def _poll_sync(client: TestClient, want: str, timeout: float = 5.0) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
body = client.get("/api/sync/status").json()
|
||||
if body["state"] == want:
|
||||
return body
|
||||
assert body["state"] == "running", body
|
||||
time.sleep(0.02)
|
||||
raise AssertionError(f"sync did not reach {want!r} in {timeout}s")
|
||||
|
||||
|
||||
def test_sync_token_row_clones_with_injected_url_and_bare_checkout(
|
||||
sync_admin_client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A token row's sync: ``clone_or_pull`` receives the injected
|
||||
``https://x-access-token:<token>@…`` URL, the checkout directory is
|
||||
derived from the BARE URL (credential-free), and no token leaks
|
||||
into the status surface (raw text)."""
|
||||
db.add(GitSource(url="https://github.com/owner/priv.git", token=TOKEN))
|
||||
db.commit()
|
||||
clone_calls = _stub_sync_stack(monkeypatch, tmp_path)
|
||||
|
||||
assert sync_admin_client.post("/api/sync").status_code == 202
|
||||
body = _poll_sync(sync_admin_client, "success")
|
||||
assert body["error"] is None
|
||||
assert TOKEN not in str(body)
|
||||
|
||||
assert len(clone_calls) == 1
|
||||
clone_url, dest = clone_calls[0]
|
||||
assert clone_url == f"https://x-access-token:{TOKEN}@github.com/owner/priv.git"
|
||||
# The checkout name is the bare repo name — no userinfo, no token.
|
||||
assert dest.name == "priv"
|
||||
assert dest == tmp_path / "sources" / "priv"
|
||||
assert TOKEN not in str(dest)
|
||||
|
||||
|
||||
def test_sync_legacy_row_clones_with_original_stored_url(
|
||||
sync_admin_client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Completion criterion: a pre-phase row (credential embedded in
|
||||
the stored URL, token NULL) produces the ORIGINAL stored URL at
|
||||
clone time — the credential keeps working — and the checkout name
|
||||
is still the credential-free repo name (repo_name on the bare-URL
|
||||
semantics: the basename after the last / of the stored URL)."""
|
||||
stored = f"https://user:{TOKEN}@github.com/owner/priv.git"
|
||||
db.add(GitSource(url=stored))
|
||||
db.commit()
|
||||
clone_calls = _stub_sync_stack(monkeypatch, tmp_path)
|
||||
|
||||
assert sync_admin_client.post("/api/sync").status_code == 202
|
||||
body = _poll_sync(sync_admin_client, "success")
|
||||
assert body["error"] is None
|
||||
|
||||
assert len(clone_calls) == 1
|
||||
clone_url, dest = clone_calls[0]
|
||||
assert clone_url == stored # the original stored URL, verbatim
|
||||
assert dest.name == "priv"
|
||||
|
||||
|
||||
def test_sync_public_row_clones_with_verbatim_url(
|
||||
sync_admin_client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Public repos behave byte-identically to pre-phase: a NULL-token
|
||||
row's clone URL is the stored URL itself (no injection)."""
|
||||
url = "https://github.com/owner/public.git"
|
||||
db.add(GitSource(url=url))
|
||||
db.commit()
|
||||
clone_calls = _stub_sync_stack(monkeypatch, tmp_path)
|
||||
|
||||
assert sync_admin_client.post("/api/sync").status_code == 202
|
||||
assert _poll_sync(sync_admin_client, "success")["error"] is None
|
||||
|
||||
assert clone_calls == [(url, tmp_path / "sources" / "public")]
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
"""Integration: migration 0021 (git_sources.token) schema contract
|
||||
(phase 121, task 01).
|
||||
|
||||
Drives the **real Alembic engine** against the live dev database
|
||||
(``podman compose up -d db``), mirroring the house pattern of
|
||||
``test_migration_0020.py`` (information_schema assertions on the state
|
||||
the migration must leave). The tests target the 0020 → 0021 step
|
||||
explicitly so later migrations cannot break the pins:
|
||||
|
||||
* upgrade 0020 → 0021 → ``token`` exists with the full contract —
|
||||
TEXT, NULLABLE, no server default — while the 0020 ``git_sources``
|
||||
schema (``url`` NOT NULL + the unique index, ``kind``, ``path``,
|
||||
``ignore_paths``, ``include_hidden``, ``added_at``) survives;
|
||||
* a ``git_sources`` row inserted while the DB is at 0020 backfills
|
||||
``token`` to NULL (a pre-phase-121 row is a public repo — or a
|
||||
legacy embedded-token row whose credential lives in ``url``);
|
||||
* the ORM contract agrees: a freshly inserted ``GitSource`` with an
|
||||
explicit ``token`` round-trips it through a fresh session, and a row
|
||||
without one reads ``token is None``;
|
||||
* downgrade to 0020 → the column is GONE (A13) while the row + its
|
||||
``url`` survive; upgrade back to 0021 → the column is back
|
||||
(round-trip).
|
||||
|
||||
The ``alembic`` fixture guarantees the DB ends at head even if a test
|
||||
fails or the process is interrupted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from alembic import command
|
||||
from app.db import SessionLocal, db_available
|
||||
from app.models import GitSource
|
||||
|
||||
URL_BARE = "https://github.com/mig0021/bare.git"
|
||||
URL_TOKENED = "https://github.com/mig0021/tokened.git"
|
||||
TOKEN = "ghp_mig0021secret"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def alembic(db: Session) -> Iterator[Config]:
|
||||
"""Real Alembic config bound to the dev DB (URL from app settings).
|
||||
|
||||
Starts at head (repairs an interrupted earlier run); teardown
|
||||
upgrades to head no matter what happened, so the dev DB is never
|
||||
left below head.
|
||||
"""
|
||||
if not db_available():
|
||||
pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
|
||||
cfg = Config() # no alembic.ini file — env.py gets the URL from app config
|
||||
cfg.set_main_option("script_location", "alembic")
|
||||
command.upgrade(cfg, "head")
|
||||
try:
|
||||
yield cfg
|
||||
finally:
|
||||
# Release the test session's open transaction BEFORE the repair
|
||||
# DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock
|
||||
# on ``git_sources``, which would deadlock the repair's
|
||||
# ``ALTER TABLE`` (0021) forever.
|
||||
db.rollback()
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
|
||||
def _version(db: Session) -> str | None:
|
||||
return db.execute(text("SELECT version_num FROM alembic_version")).scalar()
|
||||
|
||||
|
||||
def _column(db: Session, column: str) -> tuple[Any, ...] | None:
|
||||
"""(data_type, is_nullable, column_default) for one git_sources
|
||||
column."""
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT data_type, is_nullable, column_default"
|
||||
" FROM information_schema.columns"
|
||||
" WHERE table_name = 'git_sources' AND column_name = :c"
|
||||
),
|
||||
{"c": column},
|
||||
).fetchone()
|
||||
return tuple(row) if row is not None else None
|
||||
|
||||
|
||||
def _insert_sql(db: Session, url: str) -> uuid.UUID:
|
||||
"""Insert one git_sources row with the PRE-0021 column set (the
|
||||
0020 shape — the token column, when present, is omitted so a NULL
|
||||
backfill is what the row reads)."""
|
||||
row_id = uuid.uuid4()
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO git_sources (id, url, kind, path, ignore_paths,"
|
||||
" include_hidden)"
|
||||
" VALUES (:id, :u, 'git', NULL, '[]', false)"
|
||||
),
|
||||
{"id": row_id, "u": url},
|
||||
)
|
||||
db.commit()
|
||||
return row_id
|
||||
|
||||
|
||||
def _delete(db: Session, row_id: uuid.UUID) -> None:
|
||||
db.execute(text("DELETE FROM git_sources WHERE id = :id"), {"id": row_id})
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_upgrade_to_0021_adds_token(db: Session, alembic: Config) -> None:
|
||||
"""Upgrade 0020 → 0021: ``token`` exists with the full contract
|
||||
(TEXT, NULLABLE, no server default — NULL = public/legacy row), is
|
||||
ABSENT at 0020, a pre-0021 row backfills ``token`` to NULL, and an
|
||||
omitted token on a new row stays NULL — while the 0020 table
|
||||
contract (``url`` + unique index, ``kind``, ``ignore_paths``,
|
||||
``include_hidden``, ``added_at``) survives."""
|
||||
command.downgrade(alembic, "0020") # start from the pre-0021 state
|
||||
assert _version(db) == "0020"
|
||||
assert _column(db, "token") is None, "token must be absent at 0020"
|
||||
|
||||
pre_id = _insert_sql(db, URL_BARE) # the 0020 column set
|
||||
try:
|
||||
command.upgrade(alembic, "0021")
|
||||
assert _version(db) == "0021", "alembic_version must be at 0021"
|
||||
|
||||
token = _column(db, "token")
|
||||
assert token is not None, "git_sources.token is missing"
|
||||
assert token[0] == "text", "token must be TEXT"
|
||||
assert token[1] == "YES", "token must be NULLABLE"
|
||||
assert token[2] is None, (
|
||||
"token must carry NO server default — NULL is the"
|
||||
" public/legacy value"
|
||||
)
|
||||
|
||||
# The pre-0021 row backfilled to NULL (a public repo, or a
|
||||
# legacy embedded-token row whose credential lives in url).
|
||||
row = db.execute(
|
||||
text("SELECT url, token FROM git_sources WHERE id = :id"),
|
||||
{"id": pre_id},
|
||||
).fetchone()
|
||||
assert row is not None and row[0] == URL_BARE, (
|
||||
"the pre-0021 row must survive the upgrade"
|
||||
)
|
||||
assert row[1] is None, "the backfilled token must be NULL"
|
||||
|
||||
# A row written without the column stays NULL (no default to
|
||||
# fill it in — the Python-side default is None, same value).
|
||||
new_id = _insert_sql(db, URL_TOKENED)
|
||||
try:
|
||||
tokened = db.execute(
|
||||
text("SELECT token FROM git_sources WHERE id = :id"),
|
||||
{"id": new_id},
|
||||
).scalar()
|
||||
assert tokened is None, "an omitted token must stay NULL"
|
||||
finally:
|
||||
_delete(db, new_id)
|
||||
|
||||
# The 0020 schema survives the additive upgrade.
|
||||
url = _column(db, "url")
|
||||
assert url is not None and url[0] == "text" and url[1] == "NO", (
|
||||
"git_sources.url (0006) must keep its 0020 contract"
|
||||
)
|
||||
kind = _column(db, "kind")
|
||||
assert kind is not None and kind[0] == "text" and kind[1] == "NO", (
|
||||
"git_sources.kind (0007) must survive the upgrade"
|
||||
)
|
||||
ignore = _column(db, "ignore_paths")
|
||||
assert ignore is not None and ignore[0] == "jsonb" and ignore[1] == "NO", (
|
||||
"git_sources.ignore_paths (0013) must survive the upgrade"
|
||||
)
|
||||
hidden = _column(db, "include_hidden")
|
||||
assert hidden is not None and hidden[0] == "boolean" and hidden[1] == "NO", (
|
||||
"git_sources.include_hidden (0019) must survive the upgrade"
|
||||
)
|
||||
added = _column(db, "added_at")
|
||||
assert added is not None and added[0] == "timestamp with time zone", (
|
||||
"git_sources.added_at (0006) must survive the upgrade"
|
||||
)
|
||||
assert added[1] == "NO" and "now()" in str(added[2]), (
|
||||
"git_sources.added_at must keep its `now()` server default"
|
||||
)
|
||||
index = db.execute(
|
||||
text(
|
||||
"SELECT indexname FROM pg_indexes"
|
||||
" WHERE tablename = 'git_sources'"
|
||||
" AND indexname = 'uq_git_sources_url'"
|
||||
)
|
||||
).scalar()
|
||||
assert index is not None, (
|
||||
"the uq_git_sources_url unique index must survive the upgrade"
|
||||
)
|
||||
finally:
|
||||
_delete(db, pre_id)
|
||||
|
||||
|
||||
def test_orm_token_round_trips(db: Session, alembic: Config) -> None:
|
||||
"""The ORM contract agrees with the column contract: a freshly
|
||||
inserted ``GitSource`` with an explicit ``token`` round-trips the
|
||||
credential through a FRESH session, and a row without one reads
|
||||
``token is None`` (the NULL public/legacy state)."""
|
||||
command.upgrade(alembic, "head")
|
||||
row_tokened = GitSource(url=URL_TOKENED, kind="git", token=TOKEN)
|
||||
row_bare = GitSource(url=URL_BARE, kind="git")
|
||||
db.add(row_tokened)
|
||||
db.add(row_bare)
|
||||
db.commit()
|
||||
try:
|
||||
with SessionLocal() as fresh:
|
||||
reloaded_tokened = fresh.get(GitSource, row_tokened.id)
|
||||
assert reloaded_tokened is not None, "the tokened row must be readable"
|
||||
assert reloaded_tokened.token == TOKEN, (
|
||||
"the explicit token must round-trip through the DB"
|
||||
)
|
||||
reloaded_bare = fresh.get(GitSource, row_bare.id)
|
||||
assert reloaded_bare is not None, "the bare row must be readable"
|
||||
assert reloaded_bare.token is None, (
|
||||
"a row without a token must read token is None"
|
||||
)
|
||||
finally:
|
||||
_delete(db, row_tokened.id)
|
||||
_delete(db, row_bare.id)
|
||||
|
||||
|
||||
def test_downgrade_to_0020_drops_the_column(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade 0021 → 0020: the token column is gone (A13 — fully
|
||||
reversible) while the row + its ``url`` survive, and the rest of
|
||||
the 0020 table contract (``url`` unique index, ``kind``,
|
||||
``added_at``) is intact."""
|
||||
command.upgrade(alembic, "head")
|
||||
row = GitSource(url=URL_TOKENED, kind="git", token=TOKEN)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
try:
|
||||
command.downgrade(alembic, "0020")
|
||||
assert _version(db) == "0020"
|
||||
assert _column(db, "token") is None, "token must be dropped"
|
||||
|
||||
surviving = db.execute(
|
||||
text(
|
||||
"SELECT url, kind, ignore_paths, include_hidden, added_at"
|
||||
" FROM git_sources WHERE id = :id"
|
||||
),
|
||||
{"id": row.id},
|
||||
).fetchone()
|
||||
assert surviving is not None and surviving[0] == URL_TOKENED, (
|
||||
"the row must survive the column drop"
|
||||
)
|
||||
assert surviving[1] == "git" and surviving[2] == [] and surviving[3] is False, (
|
||||
"kind + ignore_paths + include_hidden must survive the drop"
|
||||
)
|
||||
assert surviving[4] is not None, "added_at must survive the drop"
|
||||
|
||||
index = db.execute(
|
||||
text(
|
||||
"SELECT indexname FROM pg_indexes"
|
||||
" WHERE tablename = 'git_sources'"
|
||||
" AND indexname = 'uq_git_sources_url'"
|
||||
)
|
||||
).scalar()
|
||||
assert index is not None, "the unique index must survive the downgrade"
|
||||
finally:
|
||||
_delete(db, row.id)
|
||||
# Repair: the fixture teardown re-upgrades to head.
|
||||
|
||||
|
||||
def test_upgrade_round_trip_restores_the_column(db: Session, alembic: Config) -> None:
|
||||
"""Downgrade to 0020, then upgrade back to 0021: ``token`` is back
|
||||
with the full contract (TEXT, NULLABLE, no server default)."""
|
||||
command.downgrade(alembic, "0020")
|
||||
command.upgrade(alembic, "0021")
|
||||
assert _version(db) == "0021", "round-trip upgrade must land at 0021"
|
||||
|
||||
token = _column(db, "token")
|
||||
assert token is not None, "git_sources.token must be back"
|
||||
assert token[0] == "text", "token must be TEXT after the round-trip"
|
||||
assert token[1] == "YES", "token must be NULLABLE after the round-trip"
|
||||
assert token[2] is None, (
|
||||
"token must still carry NO server default after the round-trip"
|
||||
)
|
||||
Reference in New Issue
Block a user