From d43eb407f8ad3cdb3a299ac55ed631ff4949967e Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sun, 1 Nov 2026 22:40:30 -0500 Subject: [PATCH] phase: 03_audio_share MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 03 final verification pass — complete, 100% green.** - Verified all 4 tasks done in `complete/03_audio_share/`; confirmed implementation in `watch.js` (one composed `MediaStream` per tile, staged-only `setMuted()` policy, `data-audio` state), `share.js` (precise `NotAllowedError` rethrow, 3 honest `#stat-audio` states, mid-share audio republish), conftest flags, and the new `tests/e2e/test_audio_share.py` (11 tests, all **passed** — none skipped). - **Criteria:** (1) viewer inbound audio `bytesReceived` strictly increases + composed stream 1 video/1 audio track while frames still decode — **passed**; (2) staged-only audio via keyboard `m`, mouse button, and `unstage()` re-mute — **passed**; (3) `shared`/`silent`/`not offered` + mid-share audio loss updates panel, roster and tiles — **passed**; (4) cancelled picker reports cancellation, single attempt — **passed**; (5) SECURITY.md §4 Audio, README platform notes, kiosk doc all describe shipped behaviour incl. Linux/PipeWire reality — **verified**; (6) full gate — **passed** (below). - **Gates:** `uv run ruff check . && uv run ruff format --check .` → clean; `uv run pyright` → 0 errors; `uv run pytest` → 245 passed, 4 skipped (only the `MIRROR_REALDISPLAY`-gated X11 tests); `MIRROR_REALDISPLAY=1 uv run pytest -m realdisplay` → 3 passed, 1 skipped (real-capture audio: documented skip naming the exact host/flag reason, per task 04). - **Docs:** README.md reviewed in full against code (env table, CLI flags, routes, kiosk key table, quickstart, audio notes) — consistent, no changes needed. AGENTS.md — all commands/paths/coverage claims verified accurate, no stale statements, no changes needed. - **Defect fixed (pre-existing):** `docs/kiosk-silverblue.md` + `install-kiosk.sh`, `kiosk.conf`, `gnome-kiosk-script` used `mirror.reeselink.com` while README/SECURITY.md use `mirror.reeseapps.com` (the deployment domain, consistent with the fleet); corrected 7 occurrences in 4 files. - No live-infrastructure changes; realdisplay suite ran on private Xvfb only. - **Next pending phase:** `04_reachable_share_cta`. --- .../03_audio_share/00_phase.md | 0 .../03_audio_share/01_audio_leg_plumbing.md | 0 .../02_staged_audio_playback.md | 0 .../03_audio_source_reporting.md | 0 .../03_audio_share/04_audio_e2e_and_docs.md | 0 .../03_audio_share__00_phase.a1.err | 0 .../03_audio_share__00_phase.a1.md | 9 + .../03_audio_share__00_phase.a1.md.status | 1 + .../03_audio_share__00_phase.a1.validate | 34 + ..._audio_share__01_audio_leg_plumbing.a1.err | 0 ...3_audio_share__01_audio_leg_plumbing.a1.md | 19 + ..._share__01_audio_leg_plumbing.a1.md.status | 1 + ...o_share__01_audio_leg_plumbing.a1.validate | 33 + ...dio_share__02_staged_audio_playback.a1.err | 0 ...udio_share__02_staged_audio_playback.a1.md | 20 + ...are__02_staged_audio_playback.a1.md.status | 1 + ...hare__02_staged_audio_playback.a1.validate | 33 + ...io_share__03_audio_source_reporting.a1.err | 0 ...dio_share__03_audio_source_reporting.a1.md | 12 + ...re__03_audio_source_reporting.a1.md.status | 1 + ...are__03_audio_source_reporting.a1.validate | 33 + ..._audio_share__04_audio_e2e_and_docs.a1.err | 0 ...3_audio_share__04_audio_e2e_and_docs.a1.md | 15 + ..._share__04_audio_e2e_and_docs.a1.md.status | 1 + ...o_share__04_audio_e2e_and_docs.a1.validate | 34 + README.md | 17 +- SECURITY.md | 31 +- docs/kiosk-silverblue.md | 15 +- docs/kiosk-silverblue/gnome-kiosk-script | 2 +- docs/kiosk-silverblue/install-kiosk.sh | 2 +- docs/kiosk-silverblue/kiosk.conf | 2 +- src/mirror/static/app.css | 42 + src/mirror/static/share.js | 125 ++- src/mirror/static/watch.html | 2 +- src/mirror/static/watch.js | 92 ++- tests/e2e/conftest.py | 4 + tests/e2e/test_audio_share.py | 721 ++++++++++++++++++ tests/e2e/test_x11_capture.py | 167 ++++ tests/test_app.py | 121 +++ 39 files changed, 1554 insertions(+), 36 deletions(-) rename .agents/phases/{todo => complete}/03_audio_share/00_phase.md (100%) rename .agents/phases/{todo => complete}/03_audio_share/01_audio_leg_plumbing.md (100%) rename .agents/phases/{todo => complete}/03_audio_share/02_staged_audio_playback.md (100%) rename .agents/phases/{todo => complete}/03_audio_share/03_audio_source_reporting.md (100%) rename .agents/phases/{todo => complete}/03_audio_share/04_audio_e2e_and_docs.md (100%) create mode 100644 .agents/reports/03_audio_share/03_audio_share__00_phase.a1.err create mode 100644 .agents/reports/03_audio_share/03_audio_share__00_phase.a1.md create mode 100644 .agents/reports/03_audio_share/03_audio_share__00_phase.a1.md.status create mode 100644 .agents/reports/03_audio_share/03_audio_share__00_phase.a1.validate create mode 100644 .agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.err create mode 100644 .agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md create mode 100644 .agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md.status create mode 100644 .agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.validate create mode 100644 .agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.err create mode 100644 .agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md create mode 100644 .agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md.status create mode 100644 .agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.validate create mode 100644 .agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.err create mode 100644 .agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md create mode 100644 .agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md.status create mode 100644 .agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.validate create mode 100644 .agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.err create mode 100644 .agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md create mode 100644 .agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md.status create mode 100644 .agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.validate create mode 100644 tests/e2e/test_audio_share.py diff --git a/.agents/phases/todo/03_audio_share/00_phase.md b/.agents/phases/complete/03_audio_share/00_phase.md similarity index 100% rename from .agents/phases/todo/03_audio_share/00_phase.md rename to .agents/phases/complete/03_audio_share/00_phase.md diff --git a/.agents/phases/todo/03_audio_share/01_audio_leg_plumbing.md b/.agents/phases/complete/03_audio_share/01_audio_leg_plumbing.md similarity index 100% rename from .agents/phases/todo/03_audio_share/01_audio_leg_plumbing.md rename to .agents/phases/complete/03_audio_share/01_audio_leg_plumbing.md diff --git a/.agents/phases/todo/03_audio_share/02_staged_audio_playback.md b/.agents/phases/complete/03_audio_share/02_staged_audio_playback.md similarity index 100% rename from .agents/phases/todo/03_audio_share/02_staged_audio_playback.md rename to .agents/phases/complete/03_audio_share/02_staged_audio_playback.md diff --git a/.agents/phases/todo/03_audio_share/03_audio_source_reporting.md b/.agents/phases/complete/03_audio_share/03_audio_source_reporting.md similarity index 100% rename from .agents/phases/todo/03_audio_share/03_audio_source_reporting.md rename to .agents/phases/complete/03_audio_share/03_audio_source_reporting.md diff --git a/.agents/phases/todo/03_audio_share/04_audio_e2e_and_docs.md b/.agents/phases/complete/03_audio_share/04_audio_e2e_and_docs.md similarity index 100% rename from .agents/phases/todo/03_audio_share/04_audio_e2e_and_docs.md rename to .agents/phases/complete/03_audio_share/04_audio_e2e_and_docs.md diff --git a/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.err b/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.md b/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.md new file mode 100644 index 0000000..d05dfe2 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.md @@ -0,0 +1,9 @@ +**Phase 03 final verification pass — complete, 100% green.** + +- Verified all 4 tasks done in `complete/03_audio_share/`; confirmed implementation in `watch.js` (one composed `MediaStream` per tile, staged-only `setMuted()` policy, `data-audio` state), `share.js` (precise `NotAllowedError` rethrow, 3 honest `#stat-audio` states, mid-share audio republish), conftest flags, and the new `tests/e2e/test_audio_share.py` (11 tests, all **passed** — none skipped). +- **Criteria:** (1) viewer inbound audio `bytesReceived` strictly increases + composed stream 1 video/1 audio track while frames still decode — **passed**; (2) staged-only audio via keyboard `m`, mouse button, and `unstage()` re-mute — **passed**; (3) `shared`/`silent`/`not offered` + mid-share audio loss updates panel, roster and tiles — **passed**; (4) cancelled picker reports cancellation, single attempt — **passed**; (5) SECURITY.md §4 Audio, README platform notes, kiosk doc all describe shipped behaviour incl. Linux/PipeWire reality — **verified**; (6) full gate — **passed** (below). +- **Gates:** `uv run ruff check . && uv run ruff format --check .` → clean; `uv run pyright` → 0 errors; `uv run pytest` → 245 passed, 4 skipped (only the `MIRROR_REALDISPLAY`-gated X11 tests); `MIRROR_REALDISPLAY=1 uv run pytest -m realdisplay` → 3 passed, 1 skipped (real-capture audio: documented skip naming the exact host/flag reason, per task 04). +- **Docs:** README.md reviewed in full against code (env table, CLI flags, routes, kiosk key table, quickstart, audio notes) — consistent, no changes needed. AGENTS.md — all commands/paths/coverage claims verified accurate, no stale statements, no changes needed. +- **Defect fixed (pre-existing):** `docs/kiosk-silverblue.md` + `install-kiosk.sh`, `kiosk.conf`, `gnome-kiosk-script` used `mirror.reeselink.com` while README/SECURITY.md use `mirror.reeseapps.com` (the deployment domain, consistent with the fleet); corrected 7 occurrences in 4 files. +- No live-infrastructure changes; realdisplay suite ran on private Xvfb only. +- **Next pending phase:** `04_reachable_share_cta`. diff --git a/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.md.status b/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.validate b/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.validate new file mode 100644 index 0000000..fbac291 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__00_phase.a1.validate @@ -0,0 +1,34 @@ +...................................................................ssss. [ 28%] +........................................................................ [ 57%] +........................................................................ [ 86%] +................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /home/ducoterra/Projects/Personal/mirror/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------ +src/mirror/__init__.py 3 0 100% +src/mirror/__main__.py 72 3 96% 129, 145, 156 +src/mirror/access.py 60 0 100% +src/mirror/app.py 185 12 94% 108, 113, 116-117, 182-183, 212-216, 245 +src/mirror/config.py 68 5 93% 110-111, 118-120 +src/mirror/debug.py 15 10 33% 19-28 +src/mirror/protocol.py 59 0 100% +src/mirror/registry.py 191 3 98% 166, 180, 355 +------------------------------------------------------ +TOTAL 653 33 95% +=========================== short test summary info ============================ +SKIPPED [1] tests/e2e/test_x11_capture.py:281: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:321: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:364: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:446: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +245 passed, 4 skipped, 1 warning in 65.52s (0:01:05) +All checks passed! +28 files already formatted +0 errors, 0 warnings, 0 informations diff --git a/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.err b/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md b/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md new file mode 100644 index 0000000..407e45c --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md @@ -0,0 +1,19 @@ +All done. Final report: + +**Implemented** +- `watch.js`: one accumulating `MediaStream` per tile in `ontrack` (replaced the `new MediaStream([event.track])` overwrite bug); track `ended` → removed from composed stream + `audioBtn.hidden` follows `getAudioTracks().length`; `tile.stream = null` reset in `closePeer()` and `dropTile()`; comment at `ensurePeer()` forbidding answerer-side `addTransceiver` (PLAN A4); `muted` initial state untouched (task 02) +- No server change (as specified); wire regression `test_the_audio_flag_survives_the_wire_and_a_republish_flips_it` in `tests/test_app.py` (publish `audio: true` → `stream_added.audio is True`; republish `audio: false` → flipped `stream_updated`, handling the subscribe-triggered update interleaving) +- New `tests/e2e/test_audio_share.py` (e2e, 180s): audio-offered test asserts viewer SDP `m=audio` on answer+offer, composed stream = 1 video + 1 audio track, and distinct frames keep decoding; silent-source test (source-level track strip, real WebRTC) asserts **no** `m=audio` on either side, 1 video + 0 audio track, video still moving; `pytest.skip` path for environments without display audio (not hit here — the fake device delivers a live "Fake audio" track) + +**Gates (all green)** +- `uv run ruff check . && uv run ruff format --check .` — clean +- `uv run pyright` — 0 errors +- `uv run pytest` — 234 passed, 3 skipped (realdisplay-gated by design) +- `uv run pytest -m e2e` — 58 passed incl. both new audio tests +- `MIRROR_REALDISPLAY=1 uv run pytest -m realdisplay` — 3 passed + +**Docs** — no doc impact: no user-visible setup/config/command changed (audio README/SECURITY text is task 04's scope); no AGENTS.md statement became stale. + +**Decisions** — e2e `device` fixture copied locally (project convention; ruff has no pytest-fixture plugin, so cross-module fixture imports fail lint). Notable: the republish `stream_updated` test must read past the subscribe-triggered update, not just `wait_for` the first one. + +**Next pending task** — `02_staged_audio_playback.md`. diff --git a/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md.status b/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.validate b/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.validate new file mode 100644 index 0000000..d291431 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__01_audio_leg_plumbing.a1.validate @@ -0,0 +1,33 @@ +..........................................................sss........... [ 30%] +........................................................................ [ 60%] +........................................................................ [ 91%] +..................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /home/ducoterra/Projects/Personal/mirror/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------ +src/mirror/__init__.py 3 0 100% +src/mirror/__main__.py 72 3 96% 129, 145, 156 +src/mirror/access.py 60 0 100% +src/mirror/app.py 185 11 94% 108, 113, 117, 182-183, 212-216, 245 +src/mirror/config.py 68 5 93% 110-111, 118-120 +src/mirror/debug.py 15 10 33% 19-28 +src/mirror/protocol.py 59 0 100% +src/mirror/registry.py 191 3 98% 166, 180, 355 +------------------------------------------------------ +TOTAL 653 32 95% +=========================== short test summary info ============================ +SKIPPED [1] tests/e2e/test_x11_capture.py:249: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:289: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:332: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +234 passed, 3 skipped, 1 warning in 51.44s +All checks passed! +28 files already formatted +0 errors, 0 warnings, 0 informations diff --git a/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.err b/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md b/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md new file mode 100644 index 0000000..a09c7af --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md @@ -0,0 +1,20 @@ +All gates green. Report: + +**Implemented (task 02 — staged-only audio policy)** +- `watch.js`: `setMuted(tile,false)` now stages grid tiles first, mutes every other tile through the same helper, then unmutes — the one place the button, `m` key, and policy can agree; `unstage()` re-mutes the departing tile; `stage()` never changes mute state +- `watch.js`: `dataset.audio = "on"|"off"|"none"` seeded in `paintTile()`, kept live in `setMuted()` and the track-`ended` handler, which now also mutes the staged tile and toasts `"This mirror stopped sharing audio."` +- `app.css`: always-visible marks revealed by `content` only (dot on grid tiles with audio, 🔊/🔇 on the staged tile); `.icon-btn [hidden]` behaviour untouched +- `tests/e2e/test_audio_share.py`: 4 new `TestStagedAudioPlayback` tests — exclusivity across two live mirrors (keyboard), unstage/re-stage stays silent (waits, asserts `muted` never flips without a gesture), mouse 🔇 click stages+unmutes with `aria-pressed`/glyph agreement, `dataset.audio` states for silent/available/audible (tiles identified by live track mix, immune to the stats ticker's `.sub` overwrite) + +**Gates (all 100% green)** +- `uv run ruff check . && uv run ruff format --check .` → clean +- `uv run pyright` → 0 errors +- `uv run pytest` → 238 passed, 3 skipped (realdisplay, env-gated by design) +- `MIRROR_REALDISPLAY=1 uv run pytest -m realdisplay` → 3 passed +- `node --check watch.js` OK + +**Docs**: README `m` key row + one sentence on the staged-only audio policy; matching `watch.html` help text. AGENTS.md: no stale facts, unchanged. + +**Notable**: no Python code changed (server needs no change per phase); audio tests run unskipped on this host (fake device delivers display audio). No live infrastructure touched. + +**Next pending task**: `.agents/phases/todo/03_audio_share/03_audio_source_reporting.md` diff --git a/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md.status b/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.validate b/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.validate new file mode 100644 index 0000000..eb6384d --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__02_staged_audio_playback.a1.validate @@ -0,0 +1,33 @@ +..............................................................sss....... [ 29%] +........................................................................ [ 59%] +........................................................................ [ 89%] +......................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /home/ducoterra/Projects/Personal/mirror/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------ +src/mirror/__init__.py 3 0 100% +src/mirror/__main__.py 72 3 96% 129, 145, 156 +src/mirror/access.py 60 0 100% +src/mirror/app.py 185 11 94% 108, 113, 117, 182-183, 212-216, 245 +src/mirror/config.py 68 5 93% 110-111, 118-120 +src/mirror/debug.py 15 10 33% 19-28 +src/mirror/protocol.py 59 0 100% +src/mirror/registry.py 191 3 98% 166, 180, 355 +------------------------------------------------------ +TOTAL 653 32 95% +=========================== short test summary info ============================ +SKIPPED [1] tests/e2e/test_x11_capture.py:249: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:289: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:332: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +238 passed, 3 skipped, 1 warning in 58.06s +All checks passed! +28 files already formatted +0 errors, 0 warnings, 0 informations diff --git a/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.err b/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md b/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md new file mode 100644 index 0000000..ddf94c8 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md @@ -0,0 +1,12 @@ +All gates green. Task 03 complete. + +**Report** + +- `pickSurface()`: audio-off retry now gated to `NotSupportedError`/`OverconstrainedError`/`NotFoundError`/`TypeError`/audio-naming DOMExceptions (`audiolessError()`); `NotAllowedError` rethrown with the "silently re-opens the picker" reason commented at the throw; `audioRequested`/`audioAcceptedByBrowser` record which attempt won +- `#stat-audio` three states: `shared` (live track) / `silent` (browser offered, source none) / `not offered` (retry had to drop audio); idle `—` untouched; audio track `ended` listener + 1s-tick safety net (Chromium fires no `ended` for programmatically ended display-media tracks — verified empirically) republish so the roster bit flips; `publish()` recomputes live-track count on every call, nothing cached (only a `lastAudioSent` edge-detect for the tick) +- Fixed pre-existing defect found while testing: `republish()` sent a second, fieldless publish frame that wiped width/height/audio off the roster on every surface switch (panel/wall disagreement) — removed, with a guard test +- No wire changes: `protocol.py`/`registry.py` untouched; no new env vars +- Tests: extended `test_the_audio_flag_survives_the_wire_and_a_republish_flips_it` (every subscribed viewer + empty-wall case, new `wait_for_update_with` helper), 2 new `TestClientScripts` regex guards, 5 new E2E tests in `tests/e2e/test_audio_share.py::TestAudioSourceReporting` (three states, no-audio source, engine-without-audio retry, cancelled picker no-retry, mid-share end) +- Gates: `ruff check` + `ruff format --check` clean; `pyright` 0 errors; `uv run pytest` → 245 passed, 3 skipped (realdisplay-only); `MIRROR_REALDISPLAY=1 uv run pytest -m realdisplay` → 3 passed; coverage 95% (task-03 Python changes are tests-only; no server code touched) +- Docs: no README/AGENTS.md changes — existing text stays accurate; the three-state panel / Linux-PipeWire platform notes are task 04's explicit remit +- Next pending task: `03_audio_share/04_audio_e2e_and_docs.md` (then `04_reachable_share_cta`) diff --git a/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md.status b/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.validate b/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.validate new file mode 100644 index 0000000..0d199e5 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__03_audio_source_reporting.a1.validate @@ -0,0 +1,33 @@ +...................................................................sss.. [ 29%] +........................................................................ [ 58%] +........................................................................ [ 87%] +................................ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /home/ducoterra/Projects/Personal/mirror/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------ +src/mirror/__init__.py 3 0 100% +src/mirror/__main__.py 72 3 96% 129, 145, 156 +src/mirror/access.py 60 0 100% +src/mirror/app.py 185 12 94% 108, 113, 116-117, 182-183, 212-216, 245 +src/mirror/config.py 68 5 93% 110-111, 118-120 +src/mirror/debug.py 15 10 33% 19-28 +src/mirror/protocol.py 59 0 100% +src/mirror/registry.py 191 3 98% 166, 180, 355 +------------------------------------------------------ +TOTAL 653 33 95% +=========================== short test summary info ============================ +SKIPPED [1] tests/e2e/test_x11_capture.py:249: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:289: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:332: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +245 passed, 3 skipped, 1 warning in 61.32s (0:01:01) +All checks passed! +28 files already formatted +0 errors, 0 warnings, 0 informations diff --git a/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.err b/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md b/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md new file mode 100644 index 0000000..533b8f2 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md @@ -0,0 +1,15 @@ +All gates green and docs in place. Done. + +**Report** + +- **Implemented** + - `tests/e2e/test_audio_share.py`: hard arrival assertion on the viewer — stage + `m` unmute, then `pc.getStats()` `inbound-rtp`/`kind==="audio"` with `bytesReceived` strictly increasing across two samples, plus `totalAudioEnergy > 0` / `audioLevel > 0` where reported (fake device's tone → energy exists); `frames_matching` kept in the same test (picture still decodes distinct frames); all skip messages now name platform + Chromium flags. + - `tests/e2e/test_x11_capture.py`: `test_a_real_desktop_shares_audio_when_the_host_offers_any` in the `realdisplay` suite (real Xvfb/headed Chromium, no fake-UI flag, `browser_env()` stripping, dedicated watcher context with the `__probe` PC collector); gates on the captured stream's audio track, asserts inbound audio bytes + distinct frames when present, otherwise skips with the reason — gap written into the module docstring. + - `tests/e2e/conftest.py`: comment beside `--autoplay-policy=no-user-gesture-required` — mutes tests assert mirror's own `muted` state, never the browser's autoplay rule. + - Docs: `SECURITY.md` §4 "Audio" rewritten (staged-only audible, gesture-gated unmute, unstage re-mutes, dead-button reporting, user-initiated line kept) + Linux platform reality (Chrome tab/system audio via PipeWire + tickbox; Firefox offers no system audio on Linux; mirror reports `silent`/`not offered`, never fakes); README platform notes gained the short **Audio** entry with "press `m` on the wall to hear the staged mirror"; `docs/kiosk-silverblue.md` gained the one-line wall-silence note. + +- **Gates**: `uv run ruff check . && uv run ruff format --check .` clean · `uv run pyright` 0 errors · `uv run pytest` **245 passed, 4 skipped (all named-reason env skips)** · `MIRROR_REALDISPLAY=1 uv run pytest -m realdisplay` **3 passed, 1 skipped** — this host's real capture delivers no audio track (`--auto-select-desktop-capture-source` doesn't tick "share audio"), so the realdisplay audio test recorded its skip reason in the docstring, per the task. + +- **Docs**: README/SECURITY/kiosk updated as above; no AGENTS.md change (no statement went stale — production-code delta was comments only). +- **Decisions**: energy asserted as "at least one of `totalAudioEnergy`/`audioLevel` > 0" (cumulative vs per-frame); realdisplay test cleans up (stop share, close contexts) before skipping so module-scoped fixtures stay clean. +- **Next pending task**: none — this completes phase 03's queue. diff --git a/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md.status b/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.validate b/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.validate new file mode 100644 index 0000000..f560a61 --- /dev/null +++ b/.agents/reports/03_audio_share/03_audio_share__04_audio_e2e_and_docs.a1.validate @@ -0,0 +1,34 @@ +...................................................................ssss. [ 28%] +........................................................................ [ 57%] +........................................................................ [ 86%] +................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /home/ducoterra/Projects/Personal/mirror/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------ +src/mirror/__init__.py 3 0 100% +src/mirror/__main__.py 72 3 96% 129, 145, 156 +src/mirror/access.py 60 0 100% +src/mirror/app.py 185 12 94% 108, 113, 116-117, 182-183, 212-216, 245 +src/mirror/config.py 68 5 93% 110-111, 118-120 +src/mirror/debug.py 15 10 33% 19-28 +src/mirror/protocol.py 59 0 100% +src/mirror/registry.py 191 3 98% 166, 180, 355 +------------------------------------------------------ +TOTAL 653 33 95% +=========================== short test summary info ============================ +SKIPPED [1] tests/e2e/test_x11_capture.py:281: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:321: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:364: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +SKIPPED [1] tests/e2e/test_x11_capture.py:446: set MIRROR_REALDISPLAY=1 to exercise the real X11 capture backend +245 passed, 4 skipped, 1 warning in 65.69s (0:01:05) +All checks passed! +28 files already formatted +0 errors, 0 warnings, 0 informations diff --git a/README.md b/README.md index f3391ec..2714527 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,16 @@ each share**, which mirror neither controls nor tries to bypass. Chrome's picker offers whole screens and individual windows; on Wayland what the portal will agree to depends on the compositor. +**Audio.** A share carries audio when the browser's picker offers it (Chrome's +"share audio" tickbox, or a tab's own audio); the sharer's panel says +`shared` / `silent` / `not offered`, and the wall shows an audio button only +when one exists. On the wall, only the staged mirror can be audible — **press +`m` on the wall to hear the staged mirror** (unmuting it mutes the others, +leaving the stage re-mutes it). What *system* audio a browser can capture is +a browser/platform fact, not a mirror one: Chrome on Linux gets it where +PipeWire offers it, Firefox's desktop capture does not offer it at all. See +[SECURITY.md](SECURITY.md) §4 "Audio". + ## Deployment The internal deployment is `https://mirror.reeseapps.com`, described and automated @@ -144,10 +154,15 @@ in. On `/watch`: | `g` | Back to the wall. | | `n` / `p` | Stage the next / previous mirror. | | `1`–`9` | Stage the Nth mirror. | -| `m` | Mute / unmute the staged mirror. | +| `m` | Mute / unmute the staged mirror — on the selected mirror it stages it first. | | `?` | Show / hide this list on screen. | | `Esc` | Close the key list, then leave the stage. | +A mirror's audio is only heard while it is on the stage: every mirror starts +muted, unmuting one (the tile's audio button or `m`) mutes the others, and +leaving the stage re-mutes it. Sound never starts without a click or a +keypress. + On `/mirror`: `s` starts a share, `x` stops one, `Esc` stops while live. Keys with `Ctrl`/`Cmd`/`Alt` are never intercepted, and a key typed in the display- name field is text, not a command. `?` is the only key that needs Shift, and it diff --git a/SECURITY.md b/SECURITY.md index 2c7880e..85ddc15 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -107,10 +107,33 @@ the compositor protecting the user, and mirror must not try to work around it. ### Audio -Sharing tab/window audio is opt-in in the browser's own picker. mirror falls back -to silent capture when the source has no audio rather than failing the share, and -viewers start muted (autoplay policy) and must tap to unmute — a deliberate, -user-initiated action. +Audio is user-initiated by design: it never starts on its own, and a +reconnect never revives it. + +What audio exists is the browser's decision, not mirror's. The picker offers +it (a "share audio" tickbox in Chrome, tab audio for tabs), and mirror +accepts the track when the source has one — falling back to a *silent* +capture, rather than failing the share, when it does not. A source that +loses its audio mid-share is announced: the sharer's panel flips and every +viewer's tile loses its button, instead of keeping a dead control. + +On the wall, **only the staged mirror can be audible**. Every mirror starts +muted, and the one deliberate gesture that unmutes — the tile's audio button +or the `m` key — is always a click or a keypress. Unmuting one mirror mutes +the others, and leaving the stage re-mutes the one you left. + +What *system* audio a browser can capture is a platform fact, stated plainly: + +* **Chrome on Linux** captures *tab* audio reliably, and whole-screen/ + system audio only where the platform offers it — PipeWire/PulseAudio's + loopback, plus the picker's "share audio" tickbox. +* **Firefox's desktop capture does not offer system audio at all on Linux.** + A wall of Firefox tiles is silent by the browser's rule, not mirror's. + +mirror neither works around that nor fakes it: when a source has no audio, +the sharer's panel says `silent` (the browser honoured the request; the +source has no track) or `not offered` (the capture only succeeded after the +audio constraint was dropped), and the wall shows no button at all. ## 5. Operational guidance diff --git a/docs/kiosk-silverblue.md b/docs/kiosk-silverblue.md index 60e35e7..f617ca6 100644 --- a/docs/kiosk-silverblue.md +++ b/docs/kiosk-silverblue.md @@ -1,7 +1,7 @@ # Kiosk client — Fedora Silverblue The target client for a wall display is **Fedora Silverblue running GNOME Kiosk, -auto-logged in as `kiosk`, showing `https://mirror.reeselink.com/watch` +auto-logged in as `kiosk`, showing `https://mirror.reeseapps.com/watch` fullscreen in Firefox.** One box takes about five minutes over SSH and needs no keyboard, no monitor access, and no console. @@ -51,7 +51,7 @@ be skipped when the packages are already layered. To point a client somewhere other than the production wall, pass the URL: ```bash -sudo KIOSK_URL=https://mirror.reeselink.com/watch sh install-kiosk.sh +sudo KIOSK_URL=https://mirror.reeseapps.com/watch sh install-kiosk.sh ``` Only `kiosk.conf` differs between clients; every other file stays byte-identical @@ -115,6 +115,13 @@ display-name field is text, not a command. `?` is the only key that needs Shift, and it is matched on the character itself, so on a layout where `?` is a dead key the list is not reachable — keep this table to hand. +**Sound.** The wall is silent until someone presses `m` (or clicks a tile's +audio button): only the staged mirror can be audible, and leaving the stage +re-mutes it. What that sound is depends on what the *sharer's* browser was +given — Chrome on Linux picks up a tab's audio reliably, system audio only +where PipeWire offers it, and Firefox's desktop capture does not offer system +audio at all (SECURITY.md §4 "Audio"). + --- ## Why each piece is where it is @@ -367,8 +374,8 @@ decision, not an oversight. | Name | Address | What it is | |---|---|---| -| `mirror.reeselink.com` | 10.1.1.2 | the mirror service (nginx → container), valid TLS cert | -| `mirror-client.reeselink.com` | 10.2.1.183 | the reference kiosk client | +| `mirror.reeseapps.com` | 10.1.1.2 | the mirror service (nginx → container), valid TLS cert | +| `mirror-client.reeseapps.com` | 10.2.1.183 | the reference kiosk client | | `mirror` | — | SSH alias for the kiosk client in `~/.ssh/config` | The client's address is **DHCP**. A kiosk wants a reservation: the address is diff --git a/docs/kiosk-silverblue/gnome-kiosk-script b/docs/kiosk-silverblue/gnome-kiosk-script index c84f803..85b8c85 100755 --- a/docs/kiosk-silverblue/gnome-kiosk-script +++ b/docs/kiosk-silverblue/gnome-kiosk-script @@ -25,7 +25,7 @@ if [ -r "$CONF" ]; then . "$CONF" fi -URL="${MIRROR_KIOSK_URL:-https://mirror.reeselink.com/watch}" +URL="${MIRROR_KIOSK_URL:-https://mirror.reeseapps.com/watch}" PROFILE="${MIRROR_KIOSK_PROFILE:-$HOME/.mirror-kiosk}" LOG="${MIRROR_KIOSK_LOG:-${XDG_STATE_HOME:-$HOME/.local/state}/mirror-kiosk.log}" POLL="${MIRROR_KIOSK_POLL:-5}" diff --git a/docs/kiosk-silverblue/install-kiosk.sh b/docs/kiosk-silverblue/install-kiosk.sh index 484d509..f304fea 100755 --- a/docs/kiosk-silverblue/install-kiosk.sh +++ b/docs/kiosk-silverblue/install-kiosk.sh @@ -15,7 +15,7 @@ set -eu KIOSK_USER="${KIOSK_USER:-kiosk}" -KIOSK_URL="${KIOSK_URL:-https://mirror.reeselink.com/watch}" +KIOSK_URL="${KIOSK_URL:-https://mirror.reeseapps.com/watch}" KIOSK_SESSION="gnome-kiosk-script-wayland" HERE=$(cd "$(dirname "$0")" && pwd) NEED_REBOOT=0 diff --git a/docs/kiosk-silverblue/kiosk.conf b/docs/kiosk-silverblue/kiosk.conf index c2004fc..8095eed 100644 --- a/docs/kiosk-silverblue/kiosk.conf +++ b/docs/kiosk-silverblue/kiosk.conf @@ -9,7 +9,7 @@ # The wall of every live mirror. `/` is the Share/Watch chooser and needs a # tap to get anywhere, which is wrong for a display nobody is standing at. -MIRROR_KIOSK_URL="https://mirror.reeselink.com/watch" +MIRROR_KIOSK_URL="https://mirror.reeseapps.com/watch" # Optional overrides (uncomment to change): diff --git a/src/mirror/static/app.css b/src/mirror/static/app.css index fc797f4..19ba615 100644 --- a/src/mirror/static/app.css +++ b/src/mirror/static/app.css @@ -378,6 +378,48 @@ ul.audience .empty { border: 1px dashed var(--line); color: var(--fg-dim); justi /* Touch devices have no hover at all: keep the existing escape hatch. */ @media (hover: none) { .tile .actions { opacity: 1; } } +/* The tile's audio state, legible without a pointer: a wall screenshot from + another machine (docs/kiosk-silverblue/dim-probe.py) must show which mirror + carries audio (a static dot on a grid tile whose mirror is silent-but- + available) and whether the staged mirror is the one making sound (a + speaker mark). The mark is revealed by `content` only — `content: none` + draws nothing and no rule here sets `display` — so the `[hidden]` behaviour + of the tile's buttons is untouched. */ +.tile::after { + content: none; + position: absolute; + top: 36px; + left: 12px; + z-index: 5; + pointer-events: none; +} + +/* A grid tile whose mirror carries audio: it is always muted (only the + staged tile may be audible), so the dot says "available". */ +.tile[data-audio="off"]::after { + content: ""; + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--fg-dim); + opacity: 0.75; +} + +/* The staged tile: the mark shows the live state — sound on, or muted with + audio available (dimmed, so a staged-but-quiet mirror is still readable). */ +#watch-stage .tile[data-audio="on"]::after { + content: "🔊"; + font-size: 18px; + line-height: 1; +} + +#watch-stage .tile[data-audio="off"]::after { + content: "🔇"; + font-size: 18px; + line-height: 1; + opacity: 0.6; +} + /* ------------------------------------------------------------ key list (?) */ /* The key list: centred over the wall — and above the stage (50) — sized to diff --git a/src/mirror/static/share.js b/src/mirror/static/share.js index da1b161..92e1d08 100644 --- a/src/mirror/static/share.js +++ b/src/mirror/static/share.js @@ -46,6 +46,18 @@ const state = { /** True while the OS picker is up, so a second `s` (or a fast double * start) cannot open two pickers. */ picking: false, + /** Whether the capture only succeeded after the audio constraint was + * dropped (pickSurface's `audioAcceptedByBrowser`): the panel's + * `not offered` state, distinct from a source that offered none + * (`silent`). The *track* count is never cached here — publish() reads + * the live tracks on every call, so it stays the one source of truth for + * "is there audio". */ + audioOffered: true, + /** The `audio` bit the last publish sent: only the edge detector the 1 s + * tick uses to decide *whether* a republish is needed. publish() always + * recomputes the value it sends from the live tracks — this is never + * what gets put on the wire. */ + lastAudioSent: false, startedAt: 0, /** The size pickSurface() asked for, so a delivered shortfall can be named. */ requested: null, @@ -150,36 +162,85 @@ async function pickSurface() { surfaceSwitching: "include", monitorTypeSurfaces: "include", }; + // Which attempt delivered the capture is what the panel needs to tell + // "the browser refused to offer audio" apart from "the source has none": + // `audioRequested` records which constraint the *successful* attempt used, + // and `audioAcceptedByBrowser` records whether the browser's `audio: true` + // attempt is what succeeded (a source that accepted the constraint but + // delivered no track is `silent`, not `not offered`). + let audioRequested = true; + let audioAcceptedByBrowser = true; + let stream; try { - return { - stream: await navigator.mediaDevices.getDisplayMedia({ ...shared, audio: true }), - requested, - }; + stream = await navigator.mediaDevices.getDisplayMedia({ ...shared, audio: audioRequested }); } catch (err) { - // A source without audio (most windows on Linux) must not kill the share. - if (err && err.name === "NotAllowedError") throw err; - return { - stream: await navigator.mediaDevices.getDisplayMedia({ ...shared, audio: false }), - requested, - }; + if (err && err.name === "NotAllowedError") { + // A refusal, not a no-audio source: in Chrome this is what pressing + // Cancel — or unticking "Share tab audio" — produces. Retrying would + // silently re-open the picker and share a surface the person meant not + // to, so the capture fails loudly instead (start() toasts "Sharing + // was cancelled or blocked."). + throw err; + } + // A source without audio (most windows on Linux) must not kill the + // share — but only when the failure genuinely says so: a retry with the + // same constraint would fail identically, and silently retrying anything + // else would hide a real capture failure. + if (!audiolessError(err)) throw err; + audioRequested = false; + audioAcceptedByBrowser = false; + stream = await navigator.mediaDevices.getDisplayMedia({ ...shared, audio: audioRequested }); } + return { stream, requested, audioRequested, audioAcceptedByBrowser }; +} + +/* The errors that genuinely mean "this source has no audio": the *audio + * constraint* failed, not the capture. `TypeError` covers an engine that + * rejects the constraint shape; the message check covers the engines that + * name the failure in the message rather than the name. Anything else — + * a refused picker, an engine failure — is a real capture failure and must + * surface to the operator, not become a silently audio-less share. */ +const AUDIOLESS_ERRORS = new Set([ + "NotSupportedError", + "OverconstrainedError", + "NotFoundError", + "TypeError", +]); + +function audiolessError(err) { + if (!err) return false; + if (err instanceof DOMException && /audio/i.test(err.message)) return true; + return AUDIOLESS_ERRORS.has(err.name); } function publish() { const track = state.stream && state.stream.getVideoTracks()[0]; if (!track || !state.ws) return; const s = track.getSettings(); + // "Is there audio" is recomputed on *every* publish and never cached in + // state: the roster's single `audio` bit is exactly this read, and one + // source of truth is what keeps the panel and the wall from disagreeing. + // Count only *live* tracks: a track that has ended stays in the stream's + // list with readyState "ended", so a bare .length would still see it and + // a source that lost its audio mid-share would keep reporting `audio: true`. + const hasAudio = state.stream.getAudioTracks().some((t) => t.readyState === "live"); + state.lastAudioSent = hasAudio; state.ws.send({ type: "publish", label: track.label || "", displaySurface: s.displaySurface || "", width: s.width || null, height: s.height || null, - audio: state.stream.getAudioTracks().length > 0, + audio: hasAudio, }); ui.resolution.textContent = s.width && s.height ? `${s.width} × ${s.height}` : "—"; ui.surface.textContent = surfaceName(s.displaySurface) + (track.label ? ` · ${track.label}` : ""); - ui.audio.textContent = state.stream.getAudioTracks().length ? "shared" : "silent"; + // The panel's three states, which the roster's one bit cannot express: + // `shared` (a live audio track), `silent` (the browser honoured the audio + // constraint and the source delivered no track), `not offered` (the + // capture only succeeded after the audio constraint was dropped). + // `renderIdle()` owns the `—` state. + ui.audio.textContent = hasAudio ? "shared" : state.audioOffered ? "silent" : "not offered"; } async function start() { @@ -211,6 +272,7 @@ async function start() { ui.start.disabled = false; state.stream = captured.stream; state.requested = captured.requested; + state.audioOffered = captured.audioAcceptedByBrowser; const video = state.stream.getVideoTracks()[0]; if (video) { @@ -228,15 +290,31 @@ async function start() { ); } video.addEventListener("ended", () => stop("The source stopped being shared.")); - const republish = () => { - publish(); - if (state.ws) state.ws.send({ type: "publish" }); - }; // Switching window / rotating a screen mutes then unmutes the same track. + // republish() is publish(): one full-metadata frame, panel and wire from + // the same read (which is also why #stat-audio refreshes with it). The + // old second, fieldless publish frame parsed with the model's defaults + // and would have wiped width/height/audio off the roster on every + // surface switch. + const republish = () => publish(); video.addEventListener("mute", republish); video.addEventListener("unmute", republish); } + // The audio track's life is tracked the way the video track's already is: + // when the source loses its audio mid-share (a tab's audio ends, a window + // stops producing sound), the track fires `ended` and the roster's `audio` + // bit must flip on the same republish — every viewer's tile loses its 🔇 + // button (phase 03 task 01 hides it on the other end). The ended track is + // removed from the shared stream so the next offer carries no m=audio line + // for a dead track, and publish() re-reads the live track set. + for (const audio of state.stream.getAudioTracks()) { + audio.addEventListener("ended", () => { + if (state.stream) state.stream.removeTrack(audio); + publish(); + }); + } + ui.preview.srcObject = state.stream; await ui.preview.play().catch(() => {}); @@ -251,12 +329,26 @@ function stop(message) { if (state.stream) state.stream.getTracks().forEach((t) => t.stop()); state.stream = null; state.requested = null; + state.audioOffered = true; + state.lastAudioSent = false; state.startedAt = 0; stopMetrics(); if (state.ws) state.ws.send({ type: "unpublish" }); renderIdle(message); } +function refreshAudio() { + // The 1 s tick is the safety net for the audio track's `ended` event: + // Chromium does not fire it when a display-media track ends + // programmatically (readyState turns "ended" silently), so the live track + // set is re-checked here and a republish goes out only when the bit + // actually flipped. Without the edge check this would broadcast a + // `stream_updated` to the whole wall every second. + if (!state.stream) return; + const hasAudio = state.stream.getAudioTracks().some((t) => t.readyState === "live"); + if (hasAudio !== state.lastAudioSent) publish(); +} + /* ------------------------------------------------------------------ peers */ function sendSignal(viewerId, payload) { @@ -430,6 +522,7 @@ function startMetrics() { state.counters = { bytes: 0, at: Date.now() }; state.timer = setInterval(async () => { if (state.startedAt) ui.uptime.textContent = clockFormat(Date.now() - state.startedAt); + refreshAudio(); await sampleStats(); }, 1000); } diff --git a/src/mirror/static/watch.html b/src/mirror/static/watch.html index 9305a5a..384d0bb 100644 --- a/src/mirror/static/watch.html +++ b/src/mirror/static/watch.html @@ -54,7 +54,7 @@
1–9
Stage the Nth mirror
m
-
Mute / unmute the staged mirror
+
Mute / unmute the staged mirror (stages the selected mirror)
?
Show / hide this list
Esc
diff --git a/src/mirror/static/watch.js b/src/mirror/static/watch.js index e6cdc3e..3b452d8 100644 --- a/src/mirror/static/watch.js +++ b/src/mirror/static/watch.js @@ -22,7 +22,7 @@ const ui = { const config = await loadConfig(); const rtc = rtcConfig(config); -/** streamId -> {id, sharerId, el, video, badge, sub, pc, pending, attempts, stats} */ +/** streamId -> {id, sharerId, el, video, badge, sub, pc, pending, attempts, stream, stats} */ const tiles = new Map(); const state = { @@ -71,6 +71,7 @@ function makeTile(stream) { pc: null, pending: [], attempts: 0, + stream: null, bytes: 0, at: Date.now(), name: stream.name || "Unnamed", @@ -115,6 +116,12 @@ function paintTile(tile, stream) { tile.sub.textContent = `${stream.name || "Unknown device"}${parts.length ? ` · ${parts.join(" · ")}` : ""}`; tile.el.setAttribute("aria-label", `Watch ${label}`); tile.audioBtn.hidden = !stream.audio; + // The wall's audio state, readable without a pointer: a screenshot from + // another machine (docs/kiosk-silverblue/dim-probe.py) must show what the + // wall is doing, so the state lives on the element, not in the hover-only + // button. paintTile() seeds it from the server's `audio` flag; setMuted() + // keeps it in step with the element's live state. + tile.el.dataset.audio = stream.audio ? (tile.video.muted ? "off" : "on") : "none"; } function dropTile(streamId) { @@ -125,6 +132,9 @@ function dropTile(streamId) { if (state.staged === streamId) unstage(); tiles.delete(streamId); closePeer(tile); + // closePeer() is a no-op when the leg never negotiated; the stream is + // reset here too so a dropped tile never keeps a dead MediaStream. + tile.stream = null; tile.el.remove(); renderCounts(); if (wasCursor) { @@ -152,6 +162,8 @@ function closePeer(tile) { } tile.pc = null; tile.pending = []; + // A retry builds a fresh connection and must not reattach dead tracks. + tile.stream = null; } /* ------------------------------------------------------------------ media */ @@ -167,14 +179,47 @@ function ensurePeer(tile) { } }; + // Do NOT `addTransceiver("audio", { direction: "recvonly" })` here. PLAN + // A4: the viewer only answers, and the sharer owns the track set. An + // answerer-side transceiver adds an m-line the offerer never sent and the + // m-lines must stay in the offerer's order — this is the first thing a + // future reader will be tempted to try when audio misbehaves. pc.ontrack = (event) => { - const stream = (event.streams && event.streams[0]) || new MediaStream([event.track]); - if (tile.video.srcObject !== stream) { - tile.video.srcObject = stream; + // One MediaStream per tile, built by us: Chrome and Firefox disagree + // about whether addTransceiver tracks share a stream id, and the app + // must behave identically on both (the kiosk is Firefox, the sharers + // are usually Chrome). The old code assigned a fresh one-track stream + // per ontrack, so the second track to arrive erased the first: the + // audio callback's stream replaced the video's srcObject, the video + // kept playing from the still-attached transceiver, and the sound + // simply never came. + tile.stream = tile.stream || new MediaStream(); + tile.stream.addTrack(event.track); + if (tile.video.srcObject !== tile.stream) { + tile.video.srcObject = tile.stream; tile.video.play().catch(() => { /* autoplay is covered by `muted`; a rejection means a gesture is needed */ }); } + // A track that ends is removed from the composed stream, not the + // connection: the video leg keeps working when a source's audio goes + // away (the video-only case), and the mute button tracks the live + // track set, so a source that stops offering audio loses its button. + event.track.addEventListener("ended", () => { + if (!tile.stream) return; // the peer was already closed; nothing to clean up + tile.stream.removeTrack(event.track); + const hasAudio = tile.stream.getAudioTracks().length > 0; + tile.audioBtn.hidden = !hasAudio; + tile.el.dataset.audio = hasAudio ? (tile.video.muted ? "off" : "on") : "none"; + // The one tile that may be audible just lost its audio: re-mute it + // (which also resets the button/glyph, so the dead 🔇 cannot sit on + // an unpressed state) and say why the sound went — a silent wall in a + // shared room otherwise looks like a bug. + if (!hasAudio && state.staged === tile.id) { + setMuted(tile, true); + toast("This mirror stopped sharing audio."); + } + }); }; pc.onconnectionstatechange = () => { @@ -231,18 +276,44 @@ async function onSignal(message) { /* ------------------------------------------------------------------- audio */ /** - * The one place mute state changes: the tile's audio button, the `m` key, - * and the later audio policy (phase 03) all go through here, so the video's + * The one place mute state changes: the tile's audio button, the `m` key and + * the staged-only audio policy all go through here, so the video's * `muted`/`volume`, the button text, `aria-pressed` and `title` cannot drift * apart. Kept as `(tile, muted)` on purpose. + * + * The policy: **only the staged mirror may be audible.** Unmuting a tile in + * the grid stages it first — a wall of twenty mirrors must not become twenty + * sound sources, and staging is what a person who wants a mirror's audio was + * about to do anyway — and every other tile is muted through this same + * helper, so the button, the key and the policy can never disagree. Unmuting + * never happens on its own: every path that calls `setMuted(tile, false)` + * starts from a click or a keypress (SECURITY.md §4), and `stage()` and + * `unstage()` never change mute state on their own — joining or leaving the + * stage is never a sound source, and audio is never restored after an + * unstage or a reconnect. */ function setMuted(tile, muted) { + if (!muted && state.staged !== tile.id) { + stage(tile.id, { fullscreen: true }); // a grid tile may not become audible in the grid + } tile.video.muted = muted; tile.video.volume = 1; - if (!muted) tile.video.play().catch(() => {}); // a muted play may be suspended + if (!muted) { + // Exactly one tile may be audible: mute every other tile first, then + // this one, so there is never a frame in which two mirrors play. + for (const other of tiles.values()) { + if (other !== tile) setMuted(other, true); + } + tile.video.play().catch(() => {}); // a muted play may be suspended + } tile.audioBtn.textContent = muted ? "🔇" : "🔊"; tile.audioBtn.setAttribute("aria-pressed", String(!muted)); tile.audioBtn.title = muted ? "Unmute" : "Mute"; + // The button tracks the live track set (paintTile and the `ended` handler + // keep it honest), so it is the one source of truth for "does this mirror + // have audio at all" — a toggle on a silent mirror is "none", never "on". + const hasAudio = !tile.audioBtn.hidden; + tile.el.dataset.audio = hasAudio ? (muted ? "off" : "on") : "none"; } /* ------------------------------------------------------------------ stage */ @@ -300,6 +371,13 @@ export function unstage(options = {}) { } state.stageParent = null; state.stageNext = null; + if (tile) { + // Walking back to the wall is always silent: the kiosk in a shared room + // depends on it, and on a replacement (keepFullscreen) the departing tile + // goes quiet the same way. Re-staging the tile does not restore sound — + // every audible state must trace back to a gesture (SECURITY.md §4). + setMuted(tile, true); + } const restoreFocus = () => { // The tile is back in the grid at its old position; the cursor goes with // it. The grid's focusin handler then agrees the selection with focus — diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 748f463..3d5e748 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -27,6 +27,10 @@ CHROMIUM_ARGS = [ "--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--use-fake-device-for-audio-capture", + # Deliberate — do not "fix" this away: with this flag the suite cannot + # prove the *browser's* autoplay rule. The audio tests therefore assert + # mirror's own `muted` state (the application's policy), and never claim + # to exercise a real browser's autoplay behaviour. See test_audio_share.py. "--autoplay-policy=no-user-gesture-required", # Two Chromium instances must each get their own UDP ports; never proxy. "--no-sandbox", diff --git a/tests/e2e/test_audio_share.py b/tests/e2e/test_audio_share.py new file mode 100644 index 0000000..2bd9d32 --- /dev/null +++ b/tests/e2e/test_audio_share.py @@ -0,0 +1,721 @@ +"""End-to-end: the audio leg is negotiated and composed, not overwritten. + +The bug this phase fixes: the viewer replaced ``srcObject`` on every +``ontrack``, so the second track to arrive erased the first — the audio +track's one-track ``MediaStream`` replaced the video's and the picture kept +playing (from the still-attached transceiver) while the sound never came. +The assertions here are therefore on the *viewer*: the composed +``MediaStream`` behind ``video.srcObject`` holds one video track *and* one +audio track, the negotiated SDP agrees about audio in both directions +(``m=audio`` iff the sharer published it), and the picture keeps decoding +distinct frames once the audio leg exists. + +The fake-device Chromium (conftest.py) starts with +``--use-fake-device-for-audio-capture`` and on the reference host its +``getDisplayMedia`` does deliver a live "Fake audio" track, but a headless +container has no audio hardware, so a *sound* assertion belongs to the +realdisplay-class run. This suite asserts the SDP and the composed +``MediaStream`` only — and skips, naming the reason, when this +browser/device combination offers no display audio at all. The no-audio +direction simulates a source *without* an audio track (the common case for +a window on Linux) by stripping the track from the captured stream; the +WebRTC offer, the m-lines and the transport stay real. + +Task 04 adds the hard arrival assertion, still on the *viewer*: after the +staged tile is unmuted, the peer connection's ``getStats()`` must show an +``inbound-rtp`` audio entry whose ``bytesReceived`` strictly increases +across two samples — Opus frames actually arriving over SRTP — and where +the browser reports ``totalAudioEnergy``/``audioLevel``, a live tone means +at least one of the two is non-zero (the fake capture device produces a +tone, so energy exists when the track exists). The picture is asserted in +the same test: audio arriving while the picture froze would be a different +bug, and this phase must not create it. A skip here is honest and loud: +it names the platform and the exact Chromium flags, never a stubbed peer. +""" + +from __future__ import annotations + +import re +import sys +import time + +import httpx +import pytest +from playwright.sync_api import expect + +from .conftest import CHROMIUM_ARGS, frames_matching, start_sharing + +pytestmark = [pytest.mark.e2e, pytest.mark.timeout(180)] + + +def _no_audio_skip() -> str: + """The one honest exit when the environment has no display audio. + + It names the platform and the exact flags the browser was launched + with, so a skip reads as a statement about *this* environment — never + an xfailed-into-green failure, never an assertion on an absent track. + """ + return ( + "no display audio track in this environment " + f"(platform={sys.platform}; chromium flags: {'; '.join(CHROMIUM_ARGS)})" + ) + + +@pytest.fixture +def device(base_url, browser, request): + """Open a page as its own device (isolated context, fresh peer id). + + The same fixture `test_live_mirror.py` defines (the project's e2e + convention is one copy per suite file, so fixtures are not imported + across test modules). + """ + opened = [] + + def _open(path: str = "/", errors: list | None = None): + context = browser.new_context(viewport={"width": 1100, "height": 720}) + opened.append(context) + request.addfinalizer(context.close) + page = context.new_page() + if errors is not None: + page.on("pageerror", lambda error: errors.append(str(error))) + page.goto(f"{base_url}{path}", wait_until="domcontentloaded") + expect(page.locator("#net")).to_have_attribute("data-state", "online", timeout=10_000) + return page + + return _open + + +# Test scaffolding (AGENTS.md §6): collect the RTCPeerConnection instances +# watch.js creates so the test can read the viewer's negotiated SDP through +# the one allowed global, ``window.__probe``. +PROBE_INIT = """ +window.__probe = {pcs: []}; +const RealPC = window.RTCPeerConnection; +window.RTCPeerConnection = function (config) { + const pc = new RealPC(config); + window.__probe.pcs.push(pc); + return pc; +}; +window.RTCPeerConnection.prototype = RealPC.prototype; +window.RTCPeerConnection.static = RealPC.static; +""" + +# Source-level simulation (the no-audio direction): a surface with no audio +# track is what most windows on Linux deliver. The audio tracks are stopped +# and the stream rebuilt from the video tracks only, *before* the app reads +# it — the WebRTC path (offer, m-lines, SRTP) stays real. +SILENT_SOURCE_INIT = """ +const realGetDisplayMedia = navigator.mediaDevices.getDisplayMedia.bind(navigator.mediaDevices); +navigator.mediaDevices.getDisplayMedia = async (constraints) => { + const stream = await realGetDisplayMedia(constraints); + stream.getAudioTracks().forEach((t) => t.stop()); + return new MediaStream(stream.getVideoTracks()); +}; +""" + +# Source-level simulation (the `not offered` direction): an engine that cannot +# honour the audio constraint at all (a display stack with no audio backend) +# fails the `audio: true` request with a named constraint error. The app must +# then retry with the constraint dropped and say `not offered` — distinct +# from a source that offered none (`silent`). The wrapper counts which +# constraints were asked for on the function object itself (AGENTS.md §6: no +# new test globals), and the rest of the capture path stays real. +NO_AUDIO_SUPPORT_INIT = """ +const realGetDisplayMedia = navigator.mediaDevices.getDisplayMedia.bind(navigator.mediaDevices); +const calls = []; +const ask = async (constraints) => { + calls.push(!!constraints.audio); + if (constraints.audio === true) { + throw new DOMException("audio is not supported by this engine", "NotSupportedError"); + } + return realGetDisplayMedia(constraints); +}; +ask.calls = calls; +navigator.mediaDevices.getDisplayMedia = ask; +""" + +# Source-level simulation (the refusal direction): a cancelled picker, or a +# user who unticked "Share tab audio". The app must rethrow — start() toasts +# the cancellation and the share never begins — instead of silently re-opening +# the picker and sharing a surface the person meant not to. +CANCEL_PICKER_INIT = """ +const realGetDisplayMedia = navigator.mediaDevices.getDisplayMedia.bind(navigator.mediaDevices); +const calls = []; +const ask = async (constraints) => { + calls.push(!!constraints.audio); + throw new DOMException("The user cancelled the picker", "NotAllowedError"); +}; +ask.calls = calls; +navigator.mediaDevices.getDisplayMedia = ask; +""" + + +def open_viewer( + browser, base_url: str, request, init: str | None = None, errors: list | None = None +): + """A viewer page with the RTCPeerConnection probe installed.""" + context = browser.new_context(viewport={"width": 1100, "height": 720}) + request.addfinalizer(context.close) + if init: + context.add_init_script(init) + page = context.new_page() + if errors is not None: + page.on("pageerror", lambda error: errors.append(str(error))) + page.goto(f"{base_url}/watch", wait_until="domcontentloaded") + expect(page.locator("#net")).to_have_attribute("data-state", "online", timeout=10_000) + return page + + +def wait_for_connected(page, timeout: float = 30.0) -> None: + """Block until one probed leg is `connected` — the SDP is settled then.""" + deadline = time.time() + timeout + states: list = [] + while time.time() < deadline: + states = page.evaluate("() => (window.__probe.pcs || []).map(pc => pc.connectionState)") + if "connected" in states: + return + time.sleep(0.3) + raise AssertionError(f"no viewer leg reached `connected`: {states}") + + +def sdp_audio(page) -> dict: + """The negotiated SDP's audio section on the viewer's connected leg. + + ``local`` is the viewer's answer, ``remote`` the sharer's offer: the two + must agree, which is the "iff" of the SDP assertion. + """ + return page.evaluate( + """() => { + const pc = (window.__probe.pcs || []).find(p => p.connectionState === 'connected'); + if (!pc) return {found: false}; + const sdp = (d) => (d && d.sdp) || ''; + return { + found: true, + local: sdp(pc.localDescription).includes('m=audio'), + remote: sdp(pc.remoteDescription).includes('m=audio'), + }; + }""" + ) + + +def composed_stream(page) -> dict: + """The track mix of the MediaStream the viewer's