ducoterra
bc70ce36e0
GFM pipe tables in the shared renderer (TODO.md L6): a table-protection
pass in frontend/assets/markdown.js (fences -> tables -> escape order)
pulls each header+separator+body block out as a placeholder, renders
cells escape-first with the same inline transforms, and reinserts a
semantic <table class="md-table"> inside a horizontal-overflow
.md-table-wrap — so a pipe table in a chat answer, the document
viewer/modal, and the thinking block all render the same semantic
table. Fences win over tables; lone pipes stay text.
- styles.css: .md-table palette rules (PLAN §7.2 tokens, no motion);
min-width: max-content so a WIDE table keeps its natural width and
the wrapper is the real scroller (width:100% alone wrapped the wide
table's cells — proven by the new E2E).
- mock_llm.py: TABLE_TRIGGER ("show me a table") -> byte-stable
TABLE_ANSWER (3-column table, <img onerror> XSS probe line, wide
5-column table), checked before DEFLECT_MODE like SUMMARY_MODE.
- tests/fixtures/docs/homelab/tables.md: 3x3 pipe table + pipe-heavy
fenced block (viewer/fence subject); the shared fixture set grows
8 -> 9 docs, so every suite pinning the count (added/formats/
stat-docs/EXPECTED_ROWS) is updated accordingly.
- tests/e2e/test_markdown_tables.py (new, story suite): chat table
shape + non-deflection, wide-table wrapper scroll (no page
overflow), XSS probe inert, viewer modal table, fence-not-a-table,
lone pipe stays text.
- tests/e2e/test_agent_document_tools.py: fix a pre-existing flake —
the "Calling tool…" label window is ~0.4 s at the mock's 0.1 s
tool-frame pacing, and a polling expect could stride over it
(failed 3 of 5 runs on the committed baseline). The pre-submit
MutationObserver record is the deterministic source of truth; the
racy to_have_text gate is gone.
uv run pytest: 738 passed, app/ coverage 99% (TOTAL unchanged);
ruff + pyright clean; story E2E 6/6 in isolation; regression E2E
suites (chat_rag, document_viewer, document_summaries, smoke) green.
106 lines
4.6 KiB
Python
106 lines
4.6 KiB
Python
"""Integration test: importer end-to-end against ``tests/fixtures/docs/``.
|
||
|
||
Runs the real import pipeline (walk → chunk → embed → upsert) into the
|
||
local compose Postgres, then checks the DB state *and* the API shape a
|
||
browser would consume. Embeddings come from a deterministic in-process
|
||
fake, so no network is needed.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from pathlib import Path
|
||
|
||
from sqlalchemy import func, select, text
|
||
|
||
from app.models import Chunk, Document
|
||
from app.rag.importer import import_sources
|
||
from tests.fakes import FakeEmbedder
|
||
|
||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs"
|
||
|
||
EXPECTED_DOCS = {
|
||
("docs", "homelab/kubernetes.md"),
|
||
("docs", "homelab/backups.md"),
|
||
("docs", "deployments/new-service.md"),
|
||
("docs", "homelab/container_gitlab/gitlab.md"),
|
||
("docs", "homelab/container_gitlab/gitlab-compose.yaml"),
|
||
("docs", "homelab/networking/static-dns.json"),
|
||
("docs", "homelab/scripts/uptime_probe.py"),
|
||
("docs", "homelab/ssh/ssh_aliases.txt"),
|
||
("docs", "homelab/tables.md"), # phase 44: the markdown-tables fixture
|
||
}
|
||
|
||
|
||
def test_import_fixtures_end_to_end(admin_client, db) -> None:
|
||
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
||
db.commit()
|
||
llm = FakeEmbedder()
|
||
|
||
summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
|
||
# Nine A9-format files (phase 44 added homelab/tables.md);
|
||
# .hidden/junk.md is out of scope (A9 revised).
|
||
assert (summary.files, summary.added, summary.unchanged) == (9, 9, 0)
|
||
assert summary.chunks >= 9
|
||
assert summary.formats == {"md": 5, "yaml": 1, "json": 1, "py": 1, "txt": 1}
|
||
# PLAN §9 per-format summary line: highest count first, then alpha.
|
||
assert summary.format_counts() == "md:5,json:1,py:1,txt:1,yaml:1"
|
||
|
||
docs = db.scalars(select(Document)).all()
|
||
assert {(d.source, d.path) for d in docs} == EXPECTED_DOCS
|
||
titles = {d.path: d.title for d in docs}
|
||
assert titles["homelab/kubernetes.md"] == "Kubernetes Homelab Cluster"
|
||
assert titles["deployments/new-service.md"] == "Deploying a New Service"
|
||
assert titles["homelab/container_gitlab/gitlab.md"] == "Gitlab"
|
||
# Non-markdown titles come from the file stem (a leading ``#`` or docstring
|
||
# line is a comment there, not a heading).
|
||
assert titles["homelab/container_gitlab/gitlab-compose.yaml"] == "gitlab-compose"
|
||
assert titles["homelab/scripts/uptime_probe.py"] == "uptime_probe"
|
||
assert titles["homelab/networking/static-dns.json"] == "static-dns"
|
||
assert titles["homelab/ssh/ssh_aliases.txt"] == "ssh_aliases"
|
||
# Hidden junk was never imported.
|
||
assert not any(".hidden" in d.path for d in docs)
|
||
# Full content is stored — that is what the RAG context will be.
|
||
k8s = next(d for d in docs if d.path == "homelab/kubernetes.md")
|
||
assert "Talos Linux" in k8s.content and k8s.content_hash
|
||
|
||
# Phase 30: the four non-markdown fixtures each gained one embedded
|
||
# ``is_summary`` chunk, so the DB holds content + summary chunks.
|
||
n_chunks = db.scalar(select(func.count()).select_from(Chunk))
|
||
assert n_chunks == summary.chunks + summary.summaries
|
||
for c in db.scalars(select(Chunk)).all():
|
||
assert c.embedding is not None and len(c.embedding) == 768
|
||
|
||
assert summary.summary_errors == 0
|
||
for d in docs:
|
||
non_md = Path(d.path).suffix.lower() not in (".md", ".markdown")
|
||
schunks = [c for c in d.chunks if c.is_summary]
|
||
if non_md:
|
||
# Lite summary stored + exactly one embedded summary chunk (−1).
|
||
assert d.summary is not None, f"{d.path} should have a summary"
|
||
assert len(schunks) == 1
|
||
assert schunks[0].position == -1
|
||
assert schunks[0].content == d.summary
|
||
assert schunks[0].embedding is not None
|
||
else:
|
||
# Markdown docs never get a summary (phase 30 scope).
|
||
assert d.summary is None and not schunks
|
||
assert summary.summaries == sum(
|
||
1 for d in docs if Path(d.path).suffix.lower() not in (".md", ".markdown")
|
||
)
|
||
|
||
# The Sources page consumes exactly this shape.
|
||
r = admin_client.get("/api/docs") # phase 16: the catalog is admin-only
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert len(body["documents"]) == 9
|
||
assert all(d["chunks"] >= 1 for d in body["documents"])
|
||
|
||
# Idempotent re-run: nothing re-embedded.
|
||
calls_before = len(llm.calls)
|
||
s2 = asyncio.run(import_sources([FIXTURES], llm, session=db))
|
||
assert s2.unchanged == 9 and s2.added == 0
|
||
assert len(llm.calls) == calls_before # unchanged → no embedding requests
|
||
|
||
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
||
db.commit()
|