From 1e67a2d1412e6cd20f7b52bcdb885ac9a6c75697 Mon Sep 17 00:00:00 2001 From: toorux Date: Fri, 9 Oct 2026 23:23:13 +0800 Subject: [PATCH] Add built-in headset casting and media controls --- .github/workflows/release.yml | 7 +- .gitignore | 5 + docs/casting.md | 72 ++ docs/releases.md | 2 + media/THIRD_PARTY_NOTICES.md | 27 + media/capture/Cargo.lock | 750 ++++++++++++++++++ media/capture/Cargo.toml | 21 + media/capture/LICENSE.framecorder | 21 + media/capture/build.rs | 16 + media/capture/shaders/convert.comp | 227 ++++++ media/capture/src/bin/framely-panel-grab.rs | 1 + media/capture/src/clock.rs | 8 + media/capture/src/encoder.rs | 571 ++++++++++++++ media/capture/src/gpu.rs | 829 ++++++++++++++++++++ media/capture/src/gpu_copy.rs | 168 ++++ media/capture/src/grab.rs | 243 ++++++ media/capture/src/kms.rs | 214 +++++ media/capture/src/lib.rs | 2 + media/capture/src/lut.rs | 467 +++++++++++ media/capture/src/main.rs | 85 ++ media/capture/src/openvr.rs | 264 +++++++ media/dependencies.json | 12 + media/native/airplay_sink.cpp | 30 + media/native/frame_sink.h | 47 ++ media/native/receiver.cpp | 277 +++++++ media/patch-uxplay.py | 37 + media/upnp/AVTransport.xml | 2 + media/upnp/ConnectionManager.xml | 2 + media/upnp/RenderingControl.xml | 2 + media/upnp/generate.py | 31 + native/cast_frame.h | 31 + native/host.cpp | 23 +- native/plugin_navigation.h | 2 +- packaging/install.sh | 30 +- src/cast/mod.rs | 6 + src/cast/receiver.rs | 541 +++++++++++++ src/cast/runtime.rs | 660 ++++++++++++++++ src/cast/settings.rs | 124 +++ src/main.rs | 1 + src/model.rs | 2 + src/service.rs | 55 +- src/session.rs | 247 +++++- src/visibility.rs | 9 +- tests/cast_frame.cpp | 20 + tests/casting_browser_probe.cpp | 43 + tests/plugin_navigation.cpp | 2 +- tests/test_install_handoff.py | 82 ++ tools/build-media.sh | 31 + tools/fetch-media.py | 15 + tools/package-release.sh | 20 +- tools/test-casting-media.py | 181 +++++ tools/test-casting-panel.py | 29 + tools/test-install-handoff.py | 52 ++ ui/locales/en-US.json | 172 +++- ui/locales/zh-CN.json | 170 +++- ui/src/cast-framing-editor.tsx | 49 ++ ui/src/cast-framing.ts | 26 + ui/src/cast-player.tsx | 20 + ui/src/cast-viewer.tsx | 55 ++ ui/src/casting.css | 13 + ui/src/casting.tsx | 82 ++ ui/src/main.tsx | 12 +- ui/src/manager-switcher.tsx | 9 +- ui/src/mobile-manager.css | 2 +- ui/src/mobile-manager.tsx | 4 +- ui/src/notifications.tsx | 2 +- 66 files changed, 7216 insertions(+), 46 deletions(-) create mode 100644 docs/casting.md create mode 100644 media/THIRD_PARTY_NOTICES.md create mode 100644 media/capture/Cargo.lock create mode 100644 media/capture/Cargo.toml create mode 100644 media/capture/LICENSE.framecorder create mode 100644 media/capture/build.rs create mode 100644 media/capture/shaders/convert.comp create mode 100644 media/capture/src/bin/framely-panel-grab.rs create mode 100644 media/capture/src/clock.rs create mode 100644 media/capture/src/encoder.rs create mode 100644 media/capture/src/gpu.rs create mode 100644 media/capture/src/gpu_copy.rs create mode 100644 media/capture/src/grab.rs create mode 100644 media/capture/src/kms.rs create mode 100644 media/capture/src/lib.rs create mode 100644 media/capture/src/lut.rs create mode 100644 media/capture/src/main.rs create mode 100644 media/capture/src/openvr.rs create mode 100644 media/dependencies.json create mode 100644 media/native/airplay_sink.cpp create mode 100644 media/native/frame_sink.h create mode 100644 media/native/receiver.cpp create mode 100644 media/patch-uxplay.py create mode 100644 media/upnp/AVTransport.xml create mode 100644 media/upnp/ConnectionManager.xml create mode 100644 media/upnp/RenderingControl.xml create mode 100644 media/upnp/generate.py create mode 100644 native/cast_frame.h create mode 100644 src/cast/mod.rs create mode 100644 src/cast/receiver.rs create mode 100644 src/cast/runtime.rs create mode 100644 src/cast/settings.rs create mode 100644 tests/cast_frame.cpp create mode 100644 tests/casting_browser_probe.cpp create mode 100644 tests/test_install_handoff.py create mode 100755 tools/build-media.sh create mode 100644 tools/fetch-media.py create mode 100644 tools/test-casting-media.py create mode 100644 tools/test-casting-panel.py create mode 100644 tools/test-install-handoff.py create mode 100644 ui/src/cast-framing-editor.tsx create mode 100644 ui/src/cast-framing.ts create mode 100644 ui/src/cast-player.tsx create mode 100644 ui/src/cast-viewer.tsx create mode 100644 ui/src/casting.css create mode 100644 ui/src/casting.tsx diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 550e41f..2dc43bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,12 @@ jobs: node-version: '22' cache: npm - name: Install native build dependencies - run: sudo apt-get update && sudo apt-get install -y g++ libgl1-mesa-dev libx11-dev + run: | + sudo apt-get update + sudo apt-get install -y g++ cmake pkg-config libgl1-mesa-dev libx11-dev \ + libdrm-dev libvulkan-dev glslc libclang-dev libssl-dev libplist-dev \ + libavahi-compat-libdnssd-dev libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev libgupnp-1.6-dev - name: Check release version env: RELEASE_REF: ${{ github.ref }} diff --git a/.gitignore b/.gitignore index e54f25a..046fbdb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,8 @@ # Generated UI review screenshots /docs/previews/ /templates/plugin/payload/ + +/media/capture/target/ +/media/bin/ + +/media/source/ diff --git a/docs/casting.md b/docs/casting.md new file mode 100644 index 0000000..23ab18c --- /dev/null +++ b/docs/casting.md @@ -0,0 +1,72 @@ +# Framely 内置投屏 + +管理网页顶部的 **投屏** 面板使用侧边菜单分为 **网页观看**、**画面与声音**、**接收投屏** 和 **投到设备**。手机通过页面选择器切换，底部六个管理入口保持单行。设置保存在核心数据库。接收协议默认关闭；采集默认 **屏幕采集**，也可手动选择 **SteamVR**。没有自动采集模式，也不会因失败自行切换来源。 + +## 采集与网页观看 + +**屏幕采集**读取头显实际扫描输出，包括透视、Steam 菜单和空间窗口。核心 root 服务只启动固定的 `framely-panel-grab` 助手，通过受 Steam 用户身份检查的私人 Unix socket 导出 DRM framebuffer 的 DMA-BUF。其余采集、Vulkan 畸变校正、取景及 Iris H.264 编码均以 Steam 用户运行；不设置应用 capability，也不依赖 Framecorder 插件。 + +Framecorder 原插件的名称是“面板画面”和“SteamVR 备用画面”（CLI 为 `panel` / `headset`）。Framely 使用“屏幕采集”和“SteamVR”，避免把参考实现的应用名当作采集方式。 + +- **单眼画面**：校正镜头畸变，读取 SteamVR 的主力眼或手动选左/右眼。使用实时完整视角预览，拖动取景框移动中心、拖动四角缩放，支持方向键和视野滑杆。开始调整会临时切换共享画面到完整参考视角，应用或取消后恢复之前的串流状态。画幅、分辨率和常见帧率使用预设按钮。 +- **双眼原始画面**：保留扫描方向、双眼布局及镜头畸变。高度由原始比例和输出宽度计算，不使用主力眼或取景设置。 +- **SteamVR**：读取 `/dev/video99`，由头显现有 FFmpeg 缩放、限帧及软件编码。实测输入固定为 RGB3 1920×1080；更改 V4L2 格式不能改变输入尺寸。设置帧率会改变驱动的报告值，但实际独立输入帧仍约 90 fps，因此设置调整的是串流输出。 + +启动一次采集，多个网页观看者共享同一条 H.264 + Opus 串流。管理页预览和 `/cast/watch` 使用同源 WHEP 信令代理和 WebRTC；沿用网络面板的登录与 Origin 检查。观看网页的视频占满视口，播放、音量、重连、画幅和全屏控件叠加在画面上，并在静止三秒后收起。默认填满屏幕，可切换完整画面；默认静音便于浏览器自动播放。手机适配安全区域和横竖屏。 + +网页优先协商 H.264。当前官方 CEF 运行包没有 H.264 解码能力时，头显自带 FFmpeg 以 libvpx 提供 VP8 + Opus 兼容输出；所有需要 VP8 的观看者共享一个额外的软件编码进程。采集来源不变。支持 H.264 的观看者仍直接接收原串流，DLNA 仍使用 H.264。VP8 增加 CPU 消耗；兼容观看结束后收回该进程。Framely 不安装、调用或依赖外部 Chrome。 + +## 接收与发送 + +AirPlay 由固定版本 UxPlay 处理 RAOP 音频和屏幕镜像；只允许一个发送设备，不实现 AirPlay 发送。自定义 GStreamer sink 在用户确认前丢弃音频和画面。断开、拒绝或停止后清除同意状态，新连接重新询问。 + +DLNA 使用 GUPnP 的标准发现、SOAP 和事件订阅，接收 HTTP(S) 媒体地址。每个新请求先显示通知，接受后才下载、解码和播放。接收的音频打开内置控制窗口；视频采用旋转及像素比例校正后的实际画幅，控制区另占 112 像素。不同 DLNA 请求可以保持多个空间窗口。播放器支持暂停、音量、停止以及媒体支持时的跳转；关闭空间窗口会停止该投屏。未处理请求 60 秒后过期。 + +GStreamer 解码结果经用户私人运行目录中的带锁 BGRA frame 文件交给原生 OpenGL 空间窗口，CEF 负责播放器控制，不承担视频解码。播放结束、错误及接收进程退出会关闭对应窗口并清理状态。 + +DLNA 发送需要开启网络管理面板并启动串流。GUPnP 发现设备后，使用该设备所在网卡的地址发送带随机会话令牌的 HLS 地址。FFmpeg 复制 H.264 视频，仅在需要时将 Opus 转为 AAC。令牌只允许读取该串流的播放列表和 MPEG-TS 分段，停止串流后失效。设备需要支持 HLS；接收端支持的具体媒体格式取决于头显已安装的 GStreamer 解码器。DRM 保护媒体不在此实现范围。 + +## 构建与打包 + +头显运行依赖现有 FFmpeg（libx264、libvpx、libopus、AAC、V4L2、PulseAudio）、GStreamer 1.20+ 和其解码器、GUPnP 1.6、Avahi、OpenVR、Vulkan/DRM、Qualcomm Iris。设备上已确认 FFmpeg、GStreamer 1.24.2、GUPnP 1.6.6 和上述编码器可用。 + +构建需要 Rust、C/C++ 编译器、CMake、pkg-config、glslc，以及 DRM/Vulkan、GStreamer app/video、GUPnP 1.6、Avahi、OpenSSL/libplist 开发文件。`tools/build-media.sh` 构建采集助手、接收器和 UxPlay，并下载校验固定版本的 MediaMTX。可用 `FRAMELY_SYSROOT` 为采集 crate 指定 ARM64 sysroot；其他组件使用目标系统的开发库。 + +`tools/package-release.sh ` 同时构建媒体能力；助手放入 `bin`，其他媒体程序放入 `lib/media`。第三方许可证和媒体改动源码随包提供，版本和 SHA-256 位于 `media/dependencies.json`。 + +## 验证 + +```bash +CARGO_HOME=/tmp/framely-cast-cargo cargo test --locked +npm run typecheck +node tools/test-localization.mjs +bash tools/test-native.sh +python3 tools/test-casting-media.py --binaries media/bin --upnp media/upnp +``` + +`test-casting-media.py` 使用生成的测试媒体与临时音频 sink，不改变系统音频设置；验证确认门控、DLNA SOAP、解码、暂停、音量、精确跳转、并发请求及超过会话上限次数的释放。测试结束回收进程和临时设备。 + +实机采集与实际 CEF WebRTC 测试： + +```bash +FRAMELY_MEDIA_BIN=/绝对路径/media/bin cargo test --locked steamvr_stream_has_requested_output_and_stops_all_workers -- --ignored --nocapture +DISPLAY=:0 FRAMELY_MEDIA_BIN=/绝对路径/media/bin \ + FRAMELY_TEST_ASSETS=/绝对路径/ui/dist FRAMELY_CAST_WEB_PORT=28189 \ + FRAMELY_CEF_ROOT=/绝对路径/CEF运行目录 \ + FRAMELY_CAST_BROWSER_PROBE=/绝对路径/cast-browser-probe \ + cargo test --locked casting_browser_fixture -- --ignored --nocapture +``` + +`tests/casting_browser_probe.cpp` 可使用现有 CEF 开发头和 libcef 编译。运行时需要 ICU、资源和 locales 与测试可执行文件一起可用。测试使用独立的临时数据库、网络端口及浏览器缓存，检查网页解码尺寸、连续播放时间和实际帧数，不修改设备上已安装的 Framely。设置 `FRAMELY_CAST_TEST_SOURCE=screen` 可以通过已安装核心的私有 IPC 请求屏幕助手；启动请求走实际 HTTP 接口，以验证媒体进程不随请求线程结束而退出。 + +需用户在头显上完成的验收： + +1. 戴上/唤醒头显，以 root 执行 `tools/test-casting-panel.py --binaries /绝对路径/media/bin`，验证普通、取景及原始双眼三种屏幕输出。 +2. 打开 **投屏**，验证默认屏幕串流、视野与主力眼设置、应用后输出变化、网页画面和声音。 +3. 用真实 AirPlay 发送端分别发送音频和屏幕镜像，确认拒绝前没有声音或画面；接受后可控，断开后重连重新询问。 +4. 用真实 DLNA 发送端打开两个媒体窗口，核对各自比例、独立暂停/停止，关闭一个不影响另一个。 +5. 选择真实 DLNA 播放设备，验证头显串流播放和停止；比较 H.264 与 CEF 的 VP8 兼容输出在目标分辨率下的流畅度和延迟。 + +自动通过的协议/编解码测试不能替代真实发送端兼容性、头显内视觉比例和用户感知延迟验收。 + +`tests/cast_frame.cpp` 验证原生视频与透明控件的预乘 Alpha 合成、控件纹理保留和收起后的完整画面，需要 GLX 环境。 diff --git a/docs/releases.md b/docs/releases.md index 8d162ef..c1f479d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -66,6 +66,8 @@ The entry fetches the release engine. Default is latest stable; `--version TAG` Installation initializes the official update descriptor URL only if no source is configured, preserving existing sources for future in-app updates. +When run from Framely's built-in terminal, the package installer moves into an independent systemd service before stopping Framely. This prevents the terminal shutdown from killing the installer. Its output remains in the journal under the `framely-install-*` unit printed by the command; the UI closes during installation and starts again on completion. + ## Unified uninstallation Scripts and desktop UI expose one uninstall operation. It stops session/downloads, then calls root-only `prepare-uninstall --approve`: persistently disable plugins, stop them in dependency order, run uninstall hooks, remove payloads. Only complete success removes services/core/runtime. Failure retains Framely, restores session access and leaves plugins disabled for troubleshooting/retry. There is no skip-failed-plugin manager uninstall. diff --git a/media/THIRD_PARTY_NOTICES.md b/media/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..825739b --- /dev/null +++ b/media/THIRD_PARTY_NOTICES.md @@ -0,0 +1,27 @@ +# Media components + +- Panel capture code in `capture/src/{encoder,gpu,gpu_copy,kms,lut,grab,openvr}.rs` + and `capture/shaders/convert.comp` derives from [coah80/framecorder](https://github.com/coah80/framecorder), MIT. + The original license is preserved in `capture/LICENSE.framecorder`. + Framely adds streaming output, field-of-view offsets, browser-compatible H.264 + encoding and a core-managed framebuffer helper. It does not invoke or require + the Framecorder application or plugin. +- [MediaMTX](https://github.com/bluenviron/mediamtx), MIT, supplies RTSP, WHEP/WebRTC + and HLS transport. Its version, source URL and archive checksum are pinned in + `dependencies.json`; the release includes its license. +- [UxPlay](https://github.com/FDH2/UxPlay), GPL-3.0-or-later, supplies AirPlay/RAOP + reception. Its version and source checksum are pinned in `dependencies.json`. + `patch-uxplay.py` adds session-end events and enforces one sending device. + Releases include the patched source archive, build recipe and UxPlay/llhttp licenses. +- GStreamer and GUPnP are linked as system libraries (LGPL). GStreamer performs + decoding, audio playback, rotation and pixel aspect correction; GUPnP provides + UPnP discovery, SOAP and event subscriptions. FFmpeg is invoked as the installed + system executable for SteamVR capture, software encoding and audio muxing. + These system components are not copied into Framely packages. +- `native/json.hpp` is the existing nlohmann/json dependency (MIT), included with + the main Framely release notices. + +Framely's added code uses the repository's AGPL-3.0-only license. See +`docs/casting.md` for build dependencies and verification commands. Source and +licenses for the bundled media additions ship under `share/source/media` and +`share/licenses` in a release. diff --git a/media/capture/Cargo.lock b/media/capture/Cargo.lock new file mode 100644 index 0000000..46bb69a --- /dev/null +++ b/media/capture/Cargo.lock @@ -0,0 +1,750 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.21", +] + +[[package]] +name = "drm" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a41816e58f47f49acfd956651055ddcf137c4882c2098c30c448817af21183a" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "bytemuck_derive", + "drm-ffi", + "drm-fourcc", + "libc", + "rustix", +] + +[[package]] +name = "drm-ffi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a91c9b32ac4e8105dec255e849e0d66e27d7c34d184364fb93e469db08f690" +dependencies = [ + "drm-sys", + "rustix", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8e1361066d91f5ffccff060a3c3be9c3ecde15be2959c1937595f7a82a9f8" +dependencies = [ + "libc", + "linux-raw-sys 0.9.4", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "enumn" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "framely-capture" +version = "0.1.0" +dependencies = [ + "anyhow", + "ash", + "clap", + "drm", + "env_logger", + "libc", + "log", + "serde_json", + "v4l2r", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "v4l2r" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67d06043afd259b7e3f0b70bb8cc590b35c99006abc595399956392b4ebbe717" +dependencies = [ + "anyhow", + "bindgen", + "bitflags 2.13.2", + "enumn", + "log", + "nix", + "pastey", + "thiserror 1.0.69", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/media/capture/Cargo.toml b/media/capture/Cargo.toml new file mode 100644 index 0000000..eba9aad --- /dev/null +++ b/media/capture/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "framely-capture" +version = "0.1.0" +edition = "2021" +license = "AGPL-3.0-only" + +[dependencies] +anyhow = "1" +ash = { version = "0.38", default-features = false, features = ["loaded", "debug"] } +clap = { version = "4", features = ["derive"] } +drm = "0.15" +env_logger = { version = "0.11", default-features = false, features = ["auto-color", "humantime"] } +libc = "0.2" +log = "0.4" +serde_json = "1" +v4l2r = "0.0.8" + +[profile.release] +lto = "thin" +strip = true + diff --git a/media/capture/LICENSE.framecorder b/media/capture/LICENSE.framecorder new file mode 100644 index 0000000..230bce9 --- /dev/null +++ b/media/capture/LICENSE.framecorder @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 coah80 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/media/capture/build.rs b/media/capture/build.rs new file mode 100644 index 0000000..78cf52b --- /dev/null +++ b/media/capture/build.rs @@ -0,0 +1,16 @@ +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let out = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let src = "shaders/convert.comp"; + println!("cargo::rerun-if-changed={src}"); + + let status = Command::new("glslc") + .args(["-O", "--target-env=vulkan1.1", "-o"]) + .arg(out.join("convert.spv")) + .arg(src) + .status() + .expect("glslc not found, install shaderc"); + assert!(status.success(), "failed to compile {src}"); +} diff --git a/media/capture/shaders/convert.comp b/media/capture/shaders/convert.comp new file mode 100644 index 0000000..1c406fc --- /dev/null +++ b/media/capture/shaders/convert.comp @@ -0,0 +1,227 @@ +#version 450 + +// Turns the panel scanout into NV12 for the hardware encoder in one pass. +// +// Every output pixel gets its source position from a coarse lookup grid. The +// grid either maps the output to a straight crop of the panel (raw view) or +// undoes the lens distortion for one eye. Each grid point stores a position per +// color channel, because the compositor pre-corrects chromatic aberration and +// we need to undo that too. A negative position means "outside the lens", which +// gets painted black. +// +// One invocation writes a 4x2 block: two words of luma and one word of +// interleaved chroma, so there are no byte stores and no atomics. A workgroup +// covers 32x16 pixels, which is exactly 4x2 grid cells, so the workgroup pulls +// the grid points it needs into shared memory once and every invocation reads +// its cell from there. + +#define STEP 8u +#define WG_X 8u +#define WG_Y 8u +#define CELLS_X (WG_X * 4u / STEP) +#define CELLS_Y (WG_Y * 2u / STEP) +#define POINTS_X (CELLS_X + 1u) +#define POINTS_Y (CELLS_Y + 1u) +#define POINTS (POINTS_X * POINTS_Y) + +layout(local_size_x = WG_X, local_size_y = WG_Y, local_size_z = 1) in; + +layout(set = 0, binding = 0) uniform sampler2D src_tex; +layout(set = 0, binding = 1, std430) readonly buffer Lut { vec2 lut[]; }; +layout(set = 0, binding = 2, std430) writeonly buffer Nv12 { uint nv12[]; }; + +layout(push_constant) uniform Params { + uvec2 out_size; + uvec2 lut_size; + uint y_stride; + uint uv_offset; + float lut_step; + uint flags; +} p; + +const uint FLAG_PER_CHANNEL = 1u; +const uint FLAG_SUPERSAMPLE = 2u; +const uint FLAG_ROTATED_EYES = 4u; +const uint FLAG_SHARP = 8u; + +// Fringes narrower than this, in output pixels, aren't worth extra reads. +const float FRINGE_MIN = 0.3; + +shared vec2 s_lut[3u * POINTS]; + +// One grid cell for one channel: a corner plus how the source moves per +// output pixel along x and y, and the bilinear twist term. +struct Cell { + vec2 origin; + vec2 dx; + vec2 dy; + vec2 dxy; + bool valid; +}; + +Cell load_cell(uint i, uint j, uint c) { + uint base = c * POINTS + j * POINTS_X + i; + vec2 a = s_lut[base]; + vec2 b = s_lut[base + 1u]; + vec2 d = s_lut[base + POINTS_X]; + vec2 e = s_lut[base + POINTS_X + 1u]; + + Cell cell; + cell.valid = min(min(a.x, b.x), min(d.x, e.x)) >= 0.0; + const float inv = 1.0 / float(STEP); + cell.origin = a; + cell.dx = (b - a) * inv; + cell.dy = (d - a) * inv; + cell.dxy = (e - d - b + a) * (inv * inv); + return cell; +} + +// Largest distance between two channels' positions at the cell's corners. +float cell_split(Cell a, Cell b) { + const float s = float(STEP); + vec2 o = a.origin - b.origin; + vec2 x = (a.dx - b.dx) * s; + vec2 y = (a.dy - b.dy) * s; + vec2 xy = (a.dxy - b.dxy) * (s * s); + return max(max(length(o), length(o + x)), max(length(o + y), length(o + x + y + xy))); +} + +vec2 cell_uv(Cell cell, vec2 local) { + return cell.origin + cell.dx * local.x + cell.dy * local.y + cell.dxy * (local.x * local.y); +} + +// The grid is in the eyes' own panel space, as SteamVR describes it. The +// Frame scans each eye's half out turned 180° in place, so turn it back +// here, per sample, which keeps the seam between the eyes clean. +// Catmull-Rom in 5 bilinear taps (the four far corners dropped, they +// barely matter). Crisper than plain bilinear, which matters here because +// the panel image has already been resampled once by the compositor. +vec4 catmull_rom(vec2 uv) { + vec2 size = vec2(textureSize(src_tex, 0)); + vec2 pos = uv * size; + vec2 center = floor(pos - 0.5) + 0.5; + vec2 f = pos - center; + vec2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f)); + vec2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f); + vec2 w2 = f * (0.5 + f * (2.0 - 1.5 * f)); + vec2 w3 = f * f * (-0.5 + 0.5 * f); + vec2 w12 = w1 + w2; + vec2 tc0 = (center - 1.0) / size; + vec2 tc3 = (center + 2.0) / size; + vec2 tc12 = (center + w2 / w12) / size; + vec4 c = textureLod(src_tex, vec2(tc12.x, tc0.y), 0.0) * (w12.x * w0.y) + + textureLod(src_tex, vec2(tc0.x, tc12.y), 0.0) * (w0.x * w12.y) + + textureLod(src_tex, tc12, 0.0) * (w12.x * w12.y) + + textureLod(src_tex, vec2(tc3.x, tc12.y), 0.0) * (w3.x * w12.y) + + textureLod(src_tex, vec2(tc12.x, tc3.y), 0.0) * (w12.x * w3.y); + float norm = w12.x * w0.y + w0.x * w12.y + w12.x * w12.y + w3.x * w12.y + w12.x * w3.y; + return max(c / norm, 0.0); +} + +vec4 fetch(vec2 uv) { + if ((p.flags & FLAG_ROTATED_EYES) != 0u) { + uv = vec2(uv.x < 0.5 ? 0.5 - uv.x : 1.5 - uv.x, 1.0 - uv.y); + } + if ((p.flags & FLAG_SHARP) != 0u) { + return catmull_rom(uv); + } + return textureLod(src_tex, uv, 0.0); +} + +vec4 sample_at(Cell cell, vec2 local) { + vec2 uv = cell_uv(cell, local); + if ((p.flags & FLAG_SUPERSAMPLE) == 0u) { + return fetch(uv); + } + // 4 taps on a rotated grid, a cheap box filter for when we shrink. + vec2 fx = cell.dx + cell.dxy * local.y; + vec2 fy = cell.dy + cell.dxy * local.x; + return 0.25 * (fetch(uv + fx * -0.375 + fy * -0.125) + + fetch(uv + fx * 0.125 + fy * -0.375) + + fetch(uv + fx * 0.375 + fy * 0.125) + + fetch(uv + fx * -0.125 + fy * 0.375)); +} + +// BT.709, limited range. The scanout is already gamma encoded, which is what +// the video matrix expects, so no transfer conversion here. +uint luma(vec3 c) { + float y = dot(c, vec3(0.2126, 0.7152, 0.0722)); + return uint(clamp(16.0 + 219.0 * y + 0.5, 0.0, 255.0)); +} + +uint chroma(vec3 c) { + float y = dot(c, vec3(0.2126, 0.7152, 0.0722)); + float cb = (c.b - y) / 1.8556; + float cr = (c.r - y) / 1.5748; + uint u = uint(clamp(128.0 + 224.0 * cb + 0.5, 0.0, 255.0)); + uint v = uint(clamp(128.0 + 224.0 * cr + 0.5, 0.0, 255.0)); + return u | (v << 8); +} + +void load_grid() { + uint li = gl_LocalInvocationIndex; + uint channels = (p.flags & FLAG_PER_CHANNEL) != 0u ? 3u : 1u; + if (li < channels * POINTS) { + // Green goes in slot 1 either way, so single channel mode only loads that. + uint c = channels == 3u ? li / POINTS : 1u; + uint r = li % POINTS; + uint gx = min(gl_WorkGroupID.x * CELLS_X + r % POINTS_X, p.lut_size.x - 1u); + uint gy = min(gl_WorkGroupID.y * CELLS_Y + r / POINTS_X, p.lut_size.y - 1u); + s_lut[c * POINTS + r] = lut[(gy * p.lut_size.x + gx) * 3u + c]; + } + barrier(); +} + +void main() { + load_grid(); + + uint x0 = gl_GlobalInvocationID.x * 4u; + uint y0 = gl_GlobalInvocationID.y * 2u; + if (x0 >= p.out_size.x || y0 >= p.out_size.y) { + return; + } + + uint ci = gl_LocalInvocationID.x * 4u / STEP; + uint cj = gl_LocalInvocationID.y * 2u / STEP; + vec2 base = vec2(float(x0 % STEP), float(y0 % STEP)) + 0.5; + + Cell green = load_cell(ci, cj, 1u); + Cell red = green; + Cell blue = green; + bool per_channel = false; + if ((p.flags & FLAG_PER_CHANNEL) != 0u) { + red = load_cell(ci, cj, 0u); + blue = load_cell(ci, cj, 2u); + // Near the middle of the lens the channels line up; one read per + // pixel is enough there, and that's most of the picture. + float limit = FRINGE_MIN * length(green.dx); + per_channel = cell_split(red, green) > limit || cell_split(blue, green) > limit; + } + + vec3 px[8]; + for (uint k = 0u; k < 8u; k++) { + vec2 local = base + vec2(float(k & 3u), float(k >> 2)); + if (!green.valid) { + px[k] = vec3(0.0); + } else if (!per_channel) { + px[k] = sample_at(green, local).rgb; + } else { + px[k] = vec3(red.valid ? sample_at(red, local).r : 0.0, + sample_at(green, local).g, + blue.valid ? sample_at(blue, local).b : 0.0); + } + } + + uint row0 = 0u; + uint row1 = 0u; + for (uint dx = 0u; dx < 4u; dx++) { + row0 |= luma(px[dx]) << (8u * dx); + row1 |= luma(px[4u + dx]) << (8u * dx); + } + nv12[(y0 * p.y_stride + x0) >> 2] = row0; + nv12[((y0 + 1u) * p.y_stride + x0) >> 2] = row1; + + vec3 left = (px[0] + px[1] + px[4] + px[5]) * 0.25; + vec3 right = (px[2] + px[3] + px[6] + px[7]) * 0.25; + nv12[(p.uv_offset + (y0 >> 1) * p.y_stride + x0) >> 2] = chroma(left) | (chroma(right) << 16); +} diff --git a/media/capture/src/bin/framely-panel-grab.rs b/media/capture/src/bin/framely-panel-grab.rs new file mode 100644 index 0000000..1ede149 --- /dev/null +++ b/media/capture/src/bin/framely-panel-grab.rs @@ -0,0 +1 @@ +fn main() { if let Err(e) = framely_capture::grab::serve(&std::path::PathBuf::from("/dev/dri/card0")) { eprintln!("{e:#}"); std::process::exit(1); } } diff --git a/media/capture/src/clock.rs b/media/capture/src/clock.rs new file mode 100644 index 0000000..374d70b --- /dev/null +++ b/media/capture/src/clock.rs @@ -0,0 +1,8 @@ +use std::time::Duration; + +/// CLOCK_MONOTONIC, the same clock DRM vblank stamps and PipeWire use. +pub fn now() -> Duration { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }; + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) +} diff --git a/media/capture/src/encoder.rs b/media/capture/src/encoder.rs new file mode 100644 index 0000000..bd916c4 --- /dev/null +++ b/media/capture/src/encoder.rs @@ -0,0 +1,571 @@ +//! Drives the Snapdragon's hardware video encoder (qcom iris) through the +//! V4L2 stateful encoder interface. Frames go in as dmabufs straight from the +//! GPU, compressed packets come out of mmapped buffers. + +use std::fs::{File, OpenOptions}; +use std::io; +use std::os::fd::{AsRawFd, RawFd}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use v4l2r::bindings as v4l2; + +use crate::gpu::Nv12Layout; + +const BUF_TYPE_CAPTURE: u32 = 9; // VIDEO_CAPTURE_MPLANE +const BUF_TYPE_OUTPUT: u32 = 10; // VIDEO_OUTPUT_MPLANE +const MEMORY_MMAP: u32 = 1; +const MEMORY_DMABUF: u32 = 4; +const FIELD_NONE: u32 = 1; +const BUF_FLAG_KEYFRAME: u32 = 0x8; +const BUF_FLAG_ERROR: u32 = 0x40; +const BUF_FLAG_LAST: u32 = 0x0010_0000; +const ENC_CMD_STOP: u32 = 1; + +const COLORSPACE_REC709: u32 = 3; +const YCBCR_ENC_709: u8 = 2; +const QUANTIZATION_LIM_RANGE: u8 = 2; +const XFER_FUNC_709: u8 = 1; + +const CID_CODEC_BASE: u32 = 0x0099_0900; +const CID_B_FRAMES: u32 = CID_CODEC_BASE + 202; +const CID_GOP_SIZE: u32 = CID_CODEC_BASE + 203; +const CID_BITRATE_MODE: u32 = CID_CODEC_BASE + 206; +const CID_BITRATE: u32 = CID_CODEC_BASE + 207; +const CID_BITRATE_PEAK: u32 = CID_CODEC_BASE + 208; +const CID_FRAME_RC_ENABLE: u32 = CID_CODEC_BASE + 215; +const CID_HEADER_MODE: u32 = CID_CODEC_BASE + 216; +const CID_FORCE_KEY_FRAME: u32 = CID_CODEC_BASE + 229; +const CID_H264_I_FRAME_QP: u32 = CID_CODEC_BASE + 350; +const CID_H264_P_FRAME_QP: u32 = CID_CODEC_BASE + 351; +const CID_H264_LEVEL: u32 = CID_CODEC_BASE + 359; +const CID_H264_PROFILE: u32 = CID_CODEC_BASE + 363; +const CID_HEVC_I_FRAME_QP: u32 = CID_CODEC_BASE + 602; +const CID_HEVC_P_FRAME_QP: u32 = CID_CODEC_BASE + 603; +const CID_HEVC_PROFILE: u32 = CID_CODEC_BASE + 615; +const CID_HEVC_LEVEL: u32 = CID_CODEC_BASE + 616; +const CID_HEVC_TIER: u32 = CID_CODEC_BASE + 618; + +const CAPTURE_BUFFERS: u32 = 4; +const STOP_TIMEOUT: Duration = Duration::from_secs(3); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum Codec { + Hevc, + H264, +} + +impl Codec { + fn fourcc(self) -> u32 { + u32::from_le_bytes(match self { + Codec::Hevc => *b"HEVC", + Codec::H264 => *b"H264", + }) + } +} + +#[derive(Clone, Copy, Debug)] +pub struct Config { + pub codec: Codec, + pub width: u32, + pub height: u32, + pub fps: u32, + pub bitrate: u32, + pub qp: Option, +} + +pub struct Packet<'a> { + pub data: &'a [u8], + pub pts_us: u64, + pub key: bool, +} + +struct Mapping { + ptr: *mut u8, + len: usize, +} + +impl Drop for Mapping { + fn drop(&mut self) { + unsafe { libc::munmap(self.ptr.cast(), self.len) }; + } +} + +pub struct Encoder { + file: File, + layout: Nv12Layout, + outputs: Vec, + captures: Vec, + streaming: bool, +} + +// The mmapped capture buffers are only touched from the thread that owns the encoder. +unsafe impl Send for Encoder {} + +// asm-generic ioctl encoding, which is what arm64 uses. +const fn ioc(dir: u32, nr: u32, size: usize) -> libc::c_ulong { + ((dir << 30) | ((size as u32) << 16) | ((b'V' as u32) << 8) | nr) as libc::c_ulong +} +const RW: u32 = 3; +const W: u32 = 1; +const VIDIOC_S_FMT: libc::c_ulong = ioc(RW, 5, std::mem::size_of::()); +const VIDIOC_REQBUFS: libc::c_ulong = ioc(RW, 8, std::mem::size_of::()); +const VIDIOC_QUERYBUF: libc::c_ulong = ioc(RW, 9, std::mem::size_of::()); +const VIDIOC_QBUF: libc::c_ulong = ioc(RW, 15, std::mem::size_of::()); +const VIDIOC_DQBUF: libc::c_ulong = ioc(RW, 17, std::mem::size_of::()); +const VIDIOC_STREAMON: libc::c_ulong = ioc(W, 18, std::mem::size_of::()); +const VIDIOC_STREAMOFF: libc::c_ulong = ioc(W, 19, std::mem::size_of::()); +const VIDIOC_S_PARM: libc::c_ulong = ioc(RW, 22, std::mem::size_of::()); +const VIDIOC_S_CTRL: libc::c_ulong = ioc(RW, 28, std::mem::size_of::()); +const VIDIOC_ENCODER_CMD: libc::c_ulong = ioc(RW, 77, std::mem::size_of::()); +const VIDIOC_S_SELECTION: libc::c_ulong = ioc(RW, 95, std::mem::size_of::()); +const BUF_TYPE_OUTPUT_SINGLE: u32 = 2; +const SEL_TGT_CROP: u32 = 0; + +#[repr(C)] +#[derive(Default)] +struct EncoderCmd { + cmd: u32, + flags: u32, + data: [u32; 8], +} + +unsafe fn xioctl(fd: RawFd, req: libc::c_ulong, arg: *mut T) -> io::Result<()> { + loop { + if libc::ioctl(fd, req, arg) == 0 { + return Ok(()); + } + let err = io::Error::last_os_error(); + if err.kind() != io::ErrorKind::Interrupted { + return Err(err); + } + } +} + +fn set_crop(fd: RawFd, width: u32, height: u32) -> Result<()> { + // Some drivers want the single planar type here even on mplane queues. + let mut last = None; + for ty in [BUF_TYPE_OUTPUT, BUF_TYPE_OUTPUT_SINGLE] { + let mut sel: v4l2::v4l2_selection = unsafe { std::mem::zeroed() }; + sel.type_ = ty; + sel.target = SEL_TGT_CROP; + sel.r.width = width; + sel.r.height = height; + match unsafe { xioctl(fd, VIDIOC_S_SELECTION, &mut sel) } { + Ok(()) if sel.r.width == width && sel.r.height == height => return Ok(()), + Ok(()) => bail!("the encoder cropped to {}x{} instead of {width}x{height}", sel.r.width, sel.r.height), + Err(e) => last = Some(e), + } + } + Err(last.unwrap()).context("setting the encoder crop") +} + +fn peak_bitrate(bitrate: u32) -> u32 { + (bitrate as u64 * 3 / 2).min(245_000_000) as u32 +} + +/// Lowest HEVC level (V4L2 menu index) and tier that fit the stream. +/// Limits from H.265 Annex A: luma picture size, luma samples per second, +/// and Main / High tier bitrate in kbit/s. +fn hevc_level(width: u32, height: u32, fps: u32, peak_kbps: u32) -> (i32, bool) { + const LEVELS: [(i32, u64, u64, u32, u32); 7] = [ + (4, 2_228_224, 66_846_720, 12_000, 30_000), // 4 + (6, 2_228_224, 133_693_440, 20_000, 50_000), // 4.1 + (7, 8_912_896, 267_386_880, 25_000, 100_000), // 5 + (8, 8_912_896, 534_773_760, 40_000, 160_000), // 5.1 + (9, 8_912_896, 1_069_547_520, 60_000, 240_000), // 5.2 + (10, 35_651_584, 1_069_547_520, 60_000, 240_000), // 6 + (11, 35_651_584, 2_139_095_040, 120_000, 480_000), // 6.1 + ]; + let size = width as u64 * height as u64; + let rate = size * fps as u64; + for &(level, max_size, max_rate, main, high) in &LEVELS { + if size <= max_size && rate <= max_rate { + if peak_kbps <= main { + return (level, false); + } + if peak_kbps <= high { + return (level, true); + } + } + } + (11, true) +} + +/// Lowest H.264 level (V4L2 menu index) that fits the stream. Limits from +/// H.264 Annex A in macroblocks, with High profile's 1.25x bitrate allowance. +fn h264_level(width: u32, height: u32, fps: u32, peak_kbps: u32) -> i32 { + const LEVELS: [(i32, u64, u64, u32); 5] = [ + (12, 8_192, 245_760, 62_500), // 4.1 + (13, 8_704, 522_240, 62_500), // 4.2 + (14, 22_080, 589_824, 168_750), // 5 + (15, 36_864, 983_040, 300_000), // 5.1 + (16, 36_864, 2_073_600, 300_000), // 5.2 + ]; + let mbs = width.div_ceil(16) as u64 * height.div_ceil(16) as u64; + let rate = mbs * fps as u64; + LEVELS + .iter() + .find(|&&(_, max_fs, max_rate, max_kbps)| mbs <= max_fs && rate <= max_rate && peak_kbps <= max_kbps) + .map_or(16, |l| l.0) +} + +/// Finds the hardware encoder node. +pub fn find_device() -> Result { + let link = Path::new("/dev/video-enc0"); + if link.exists() { + return Ok(link.into()); + } + for entry in std::fs::read_dir("/sys/class/video4linux")? { + let entry = entry?; + let name = std::fs::read_to_string(entry.path().join("name")).unwrap_or_default(); + if name.contains("encoder") { + return Ok(Path::new("/dev").join(entry.file_name())); + } + } + bail!("no hardware video encoder found") +} + +impl Encoder { + pub fn open(path: &Path, cfg: Config, output_buffers: u32) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_NONBLOCK | libc::O_CLOEXEC) + .open(path) + .with_context(|| format!("opening encoder {}", path.display()))?; + let fd = file.as_raw_fd(); + + unsafe { + // Coded format first, the raw side's constraints depend on it. + let mut fmt: v4l2::v4l2_format = std::mem::zeroed(); + fmt.type_ = BUF_TYPE_CAPTURE; + fmt.fmt.pix_mp.width = cfg.width; + fmt.fmt.pix_mp.height = cfg.height; + fmt.fmt.pix_mp.pixelformat = cfg.codec.fourcc(); + fmt.fmt.pix_mp.field = FIELD_NONE; + fmt.fmt.pix_mp.num_planes = 1; + xioctl(fd, VIDIOC_S_FMT, &mut fmt).context("setting coded format")?; + if fmt.fmt.pix_mp.pixelformat != cfg.codec.fourcc() { + bail!("the encoder doesn't do {:?}", cfg.codec); + } + + let mut fmt: v4l2::v4l2_format = std::mem::zeroed(); + fmt.type_ = BUF_TYPE_OUTPUT; + let pix = &mut fmt.fmt.pix_mp; + pix.width = cfg.width; + pix.height = cfg.height; + pix.pixelformat = u32::from_le_bytes(*b"NV12"); + pix.field = FIELD_NONE; + pix.num_planes = 1; + pix.colorspace = COLORSPACE_REC709; + pix.__bindgen_anon_1.ycbcr_enc = YCBCR_ENC_709; + pix.quantization = QUANTIZATION_LIM_RANGE; + pix.xfer_func = XFER_FUNC_709; + xioctl(fd, VIDIOC_S_FMT, &mut fmt).context("setting raw format")?; + let pix = fmt.fmt.pix_mp; + let (got_w, got_h) = (pix.width, pix.height); + if got_w < cfg.width || got_h < cfg.height { + bail!("the encoder wants {got_w}x{got_h} instead of {}x{}", cfg.width, cfg.height); + } + if (got_w, got_h) != (cfg.width, cfg.height) { + // Buffers are padded to the macroblock size, the crop says what's real. + set_crop(fd, cfg.width, cfg.height)?; + } + + let plane = pix.plane_fmt[0]; + let (y_stride, size) = (plane.bytesperline, plane.sizeimage); + // Venus/iris NV12: luma rows padded to 32, chroma follows right after. + let uv_offset = y_stride * got_h.next_multiple_of(32); + let expected = uv_offset + y_stride * (got_h / 2).next_multiple_of(16); + if y_stride % 4 != 0 || uv_offset % 4 != 0 || expected > size { + bail!("unexpected NV12 layout from the encoder: stride {y_stride}, size {size}"); + } + log::info!("encoder input: NV12 stride {y_stride}, chroma at {uv_offset}, {size} bytes"); + + let layout = Nv12Layout { width: cfg.width, height: cfg.height, y_stride, uv_offset, size }; + + for ty in [BUF_TYPE_OUTPUT, BUF_TYPE_CAPTURE] { + let mut parm: v4l2::v4l2_streamparm = std::mem::zeroed(); + parm.type_ = ty; + // Same offset for capture and output. + parm.parm.output.timeperframe.numerator = 1; + parm.parm.output.timeperframe.denominator = cfg.fps; + if let Err(e) = xioctl(fd, VIDIOC_S_PARM, &mut parm) { + log::warn!("couldn't set frame rate on queue {ty}: {e}"); + } + } + + let mut enc = Self { file, layout, outputs: Vec::new(), captures: Vec::new(), streaming: false }; + enc.set_controls(&cfg); + enc.setup_buffers(output_buffers)?; + Ok(enc) + } + } + + fn set_ctrl(&self, name: &str, id: u32, value: i32) { + let mut ctrl = v4l2::v4l2_control { id, value }; + if let Err(e) = unsafe { xioctl(self.file.as_raw_fd(), VIDIOC_S_CTRL, &mut ctrl) } { + log::warn!("encoder ignored {name}={value}: {e}"); + } + } + + fn set_controls(&self, cfg: &Config) { + let gop = cfg.fps as i32; + self.set_ctrl("b_frames", CID_B_FRAMES, 0); + self.set_ctrl("gop_size", CID_GOP_SIZE, gop); + self.set_ctrl("header_mode", CID_HEADER_MODE, 0); + + // With a fixed quantizer the bitrate is whatever it turns out to be, + // so leave room for the top of the range. + let peak_kbps = if cfg.qp.is_some() { 160_000 } else { peak_bitrate(cfg.bitrate) / 1000 }; + match cfg.codec { + Codec::Hevc => { + let (level, high_tier) = hevc_level(cfg.width, cfg.height, cfg.fps, peak_kbps); + self.set_ctrl("hevc_profile", CID_HEVC_PROFILE, 0); // Main + self.set_ctrl("hevc_tier", CID_HEVC_TIER, high_tier as i32); + self.set_ctrl("hevc_level", CID_HEVC_LEVEL, level); + } + Codec::H264 => { + self.set_ctrl("h264_profile", CID_H264_PROFILE, 0); // Baseline for browser interoperability + self.set_ctrl("h264_level", CID_H264_LEVEL, h264_level(cfg.width, cfg.height, cfg.fps, peak_kbps)); + } + } + + match cfg.qp { + Some(qp) => { + self.set_ctrl("frame_rc", CID_FRAME_RC_ENABLE, 0); + let (i, p) = match cfg.codec { + Codec::Hevc => (CID_HEVC_I_FRAME_QP, CID_HEVC_P_FRAME_QP), + Codec::H264 => (CID_H264_I_FRAME_QP, CID_H264_P_FRAME_QP), + }; + self.set_ctrl("i_qp", i, qp as i32); + self.set_ctrl("p_qp", p, qp as i32); + } + None => { + self.set_ctrl("frame_rc", CID_FRAME_RC_ENABLE, 1); + self.set_ctrl("bitrate_mode", CID_BITRATE_MODE, 0); // VBR + self.set_ctrl("bitrate", CID_BITRATE, cfg.bitrate as i32); + self.set_ctrl("bitrate_peak", CID_BITRATE_PEAK, peak_bitrate(cfg.bitrate) as i32); + } + } + } + + unsafe fn setup_buffers(&mut self, output_buffers: u32) -> Result<()> { + let fd = self.file.as_raw_fd(); + + let mut req: v4l2::v4l2_requestbuffers = std::mem::zeroed(); + req.count = output_buffers; + req.type_ = BUF_TYPE_OUTPUT; + req.memory = MEMORY_DMABUF; + xioctl(fd, VIDIOC_REQBUFS, &mut req).context("allocating encoder input slots")?; + if req.count < output_buffers { + bail!("encoder only gave us {} input slots", req.count); + } + self.outputs = vec![false; output_buffers as usize]; + + let mut req: v4l2::v4l2_requestbuffers = std::mem::zeroed(); + req.count = CAPTURE_BUFFERS; + req.type_ = BUF_TYPE_CAPTURE; + req.memory = MEMORY_MMAP; + xioctl(fd, VIDIOC_REQBUFS, &mut req).context("allocating encoder output buffers")?; + + for index in 0..req.count { + let mut plane: v4l2::v4l2_plane = std::mem::zeroed(); + let mut buf: v4l2::v4l2_buffer = std::mem::zeroed(); + buf.index = index; + buf.type_ = BUF_TYPE_CAPTURE; + buf.memory = MEMORY_MMAP; + buf.length = 1; + buf.m.planes = &mut plane; + xioctl(fd, VIDIOC_QUERYBUF, &mut buf).context("querying encoder output buffer")?; + + let len = plane.length as usize; + let ptr = libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ, + libc::MAP_SHARED, + fd, + plane.m.mem_offset as libc::off_t, + ); + if ptr == libc::MAP_FAILED { + return Err(io::Error::last_os_error()).context("mapping encoder output buffer"); + } + self.captures.push(Mapping { ptr: ptr.cast(), len }); + self.queue_capture(index)?; + } + log::debug!("encoder: {} input slots, {} output buffers", self.outputs.len(), self.captures.len()); + + for ty in [BUF_TYPE_OUTPUT, BUF_TYPE_CAPTURE] { + let mut t = ty as libc::c_int; + xioctl(fd, VIDIOC_STREAMON, &mut t).context("starting the encoder")?; + } + self.streaming = true; + Ok(()) + } + + fn queue_capture(&self, index: u32) -> Result<()> { + unsafe { + let mut plane: v4l2::v4l2_plane = std::mem::zeroed(); + plane.length = self.captures[index as usize].len as u32; + let mut buf: v4l2::v4l2_buffer = std::mem::zeroed(); + buf.index = index; + buf.type_ = BUF_TYPE_CAPTURE; + buf.memory = MEMORY_MMAP; + buf.length = 1; + buf.m.planes = &mut plane; + xioctl(self.file.as_raw_fd(), VIDIOC_QBUF, &mut buf).context("queueing encoder output buffer") + } + } + + /// Makes the next frame a keyframe, so a recording started mid-session + /// doesn't wait for the next one. + pub fn force_keyframe(&self) { + self.set_ctrl("force_key_frame", CID_FORCE_KEY_FRAME, 1); + } + + pub fn layout(&self) -> Nv12Layout { + self.layout + } + + /// An input slot the GPU can write into, or None if the encoder is + /// still chewing on all of them. + pub fn free_slot(&mut self) -> Result> { + self.reclaim_outputs()?; + Ok(self.outputs.iter().position(|busy| !busy)) + } + + fn reclaim_outputs(&mut self) -> Result<()> { + loop { + let mut plane: v4l2::v4l2_plane = unsafe { std::mem::zeroed() }; + let mut buf: v4l2::v4l2_buffer = unsafe { std::mem::zeroed() }; + buf.type_ = BUF_TYPE_OUTPUT; + buf.memory = MEMORY_DMABUF; + buf.length = 1; + buf.m.planes = &mut plane; + match unsafe { xioctl(self.file.as_raw_fd(), VIDIOC_DQBUF, &mut buf) } { + Ok(()) => self.outputs[buf.index as usize] = false, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(()), + Err(e) if e.raw_os_error() == Some(libc::EPIPE) => return Ok(()), + Err(e) => return Err(e).context("reclaiming encoder input"), + } + } + } + + pub fn queue_frame(&mut self, slot: usize, dmabuf: RawFd, pts_us: u64) -> Result<()> { + unsafe { + let mut plane: v4l2::v4l2_plane = std::mem::zeroed(); + plane.bytesused = self.layout.size; + plane.length = self.layout.size; + plane.m.fd = dmabuf; + let mut buf: v4l2::v4l2_buffer = std::mem::zeroed(); + buf.index = slot as u32; + buf.type_ = BUF_TYPE_OUTPUT; + buf.memory = MEMORY_DMABUF; + buf.field = FIELD_NONE; + buf.length = 1; + buf.m.planes = &mut plane; + buf.timestamp.tv_sec = (pts_us / 1_000_000) as _; + buf.timestamp.tv_usec = (pts_us % 1_000_000) as _; + xioctl(self.file.as_raw_fd(), VIDIOC_QBUF, &mut buf).context("feeding the encoder")?; + } + self.outputs[slot] = true; + Ok(()) + } + + /// Hands every finished packet to `sink`. Returns true once the encoder + /// has sent its last packet after `finish`. + pub fn poll_packets(&mut self, mut sink: impl FnMut(Packet)) -> Result { + loop { + let mut plane: v4l2::v4l2_plane = unsafe { std::mem::zeroed() }; + let mut buf: v4l2::v4l2_buffer = unsafe { std::mem::zeroed() }; + buf.type_ = BUF_TYPE_CAPTURE; + buf.memory = MEMORY_MMAP; + buf.length = 1; + buf.m.planes = &mut plane; + match unsafe { xioctl(self.file.as_raw_fd(), VIDIOC_DQBUF, &mut buf) } { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(false), + Err(e) if e.raw_os_error() == Some(libc::EPIPE) => return Ok(true), + Err(e) => return Err(e).context("reading from the encoder"), + } + + let map = &self.captures[buf.index as usize]; + let start = (plane.data_offset as usize).min(map.len); + let end = (plane.bytesused as usize).min(map.len); + if buf.flags & BUF_FLAG_ERROR != 0 { + log::warn!("encoder flagged a corrupt packet, dropping it"); + } else if end > start { + let data = unsafe { std::slice::from_raw_parts(map.ptr.add(start), end - start) }; + sink(Packet { + data, + pts_us: buf.timestamp.tv_sec as u64 * 1_000_000 + buf.timestamp.tv_usec as u64, + key: buf.flags & BUF_FLAG_KEYFRAME != 0, + }); + } + + if buf.flags & BUF_FLAG_LAST != 0 { + return Ok(true); + } + self.queue_capture(buf.index)?; + } + } + + /// Flushes everything still inside the encoder. + pub fn finish(&mut self, mut sink: impl FnMut(Packet)) -> Result<()> { + let mut cmd = EncoderCmd { cmd: ENC_CMD_STOP, ..Default::default() }; + if let Err(e) = unsafe { xioctl(self.file.as_raw_fd(), VIDIOC_ENCODER_CMD, &mut cmd) } { + log::warn!("encoder refused to drain: {e}"); + return Ok(()); + } + let deadline = Instant::now() + STOP_TIMEOUT; + while Instant::now() < deadline { + if self.poll_packets(&mut sink)? { + return Ok(()); + } + let mut pfd = libc::pollfd { fd: self.file.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + unsafe { libc::poll(&mut pfd, 1, 50) }; + } + log::warn!("encoder didn't finish draining in time, the last frames may be missing"); + Ok(()) + } +} + +impl Drop for Encoder { + fn drop(&mut self) { + if self.streaming { + for ty in [BUF_TYPE_OUTPUT, BUF_TYPE_CAPTURE] { + let mut t = ty as libc::c_int; + let _ = unsafe { xioctl(self.file.as_raw_fd(), VIDIOC_STREAMOFF, &mut t) }; + } + } + self.captures.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hevc_levels() { + // A 60 Mbit/s peak is past 4.1 High's 50, so level 5 High. 30 fits 4.1 High. + assert_eq!(hevc_level(1920, 1080, 60, 60_000), (7, true)); + assert_eq!(hevc_level(1920, 1080, 60, 30_000), (6, true)); + assert_eq!(hevc_level(1920, 1080, 60, 15_000), (6, false)); + // 1080p72 needs level 5 for the sample rate alone. + assert_eq!(hevc_level(1920, 1080, 72, 20_000), (7, false)); + // Square 1440 at 30 fps is light enough for level 4. + assert_eq!(hevc_level(1440, 1440, 30, 10_000), (4, false)); + } + + #[test] + fn h264_levels() { + assert_eq!(h264_level(1920, 1080, 60, 60_000), 13); + assert_eq!(h264_level(1920, 1080, 72, 60_000), 14); + assert_eq!(h264_level(1920, 1080, 30, 120_000), 14); + } +} diff --git a/media/capture/src/gpu.rs b/media/capture/src/gpu.rs new file mode 100644 index 0000000..17fee1d --- /dev/null +++ b/media/capture/src/gpu.rs @@ -0,0 +1,829 @@ +//! The only GPU work framely-capture does: one small compute dispatch per recorded +//! frame. It reads the compositor's scanout buffer in place and writes NV12 +//! into dmabufs that the hardware encoder reads directly. No CPU copies. +//! +//! Its queue priority is picked by the caller, see `Priority`. + +use std::ffi::{c_char, CStr, CString}; +use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; + +use anyhow::{bail, Context, Result}; +use ash::vk; + +use crate::kms::ScanoutBuffer; + +#[path = "gpu_copy.rs"] +mod copy; +use crate::lut::Lut; + +const SHADER: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/convert.spv")); +const MAX_SOURCES: usize = 8; + +const DRM_FORMAT_XRGB8888: u32 = u32::from_le_bytes(*b"XR24"); +const DRM_FORMAT_ARGB8888: u32 = u32::from_le_bytes(*b"AR24"); +const DRM_FORMAT_XBGR8888: u32 = u32::from_le_bytes(*b"XB24"); +const DRM_FORMAT_ABGR8888: u32 = u32::from_le_bytes(*b"AB24"); + +#[repr(C)] +#[derive(Clone, Copy)] +struct Params { + out_size: [u32; 2], + lut_size: [u32; 2], + y_stride: u32, + uv_offset: u32, + lut_step: f32, + flags: u32, +} + +/// Layout of the NV12 buffers, dictated by the encoder. +#[derive(Clone, Copy, Debug)] +pub struct Nv12Layout { + pub width: u32, + pub height: u32, + pub y_stride: u32, + pub uv_offset: u32, + pub size: u32, +} + +struct Source { + fb_id: u32, + /// Identity of the buffer behind the framebuffer id, see `kms::buffer_id`. + buffer: u64, + last_used: u64, + image: vk::Image, + memory: vk::DeviceMemory, + view: vk::ImageView, + /// One descriptor set and pre-recorded command buffer per target. + sets: Vec, + cmds: Vec, +} + +pub struct Target { + buffer: vk::Buffer, + memory: vk::DeviceMemory, + pub fd: OwnedFd, +} + +pub struct Gpu { + _entry: ash::Entry, + instance: ash::Instance, + device: ash::Device, + ext_fd: ash::khr::external_memory_fd::Device, + mem_props: vk::PhysicalDeviceMemoryProperties, + queue: vk::Queue, + queue_family: u32, + cmd_pool: vk::CommandPool, + desc_pool: vk::DescriptorPool, + set_layout: vk::DescriptorSetLayout, + pipeline_layout: vk::PipelineLayout, + pipeline: vk::Pipeline, + shader: vk::ShaderModule, + sampler: vk::Sampler, + fence: vk::Fence, + /// Two timestamps around the dispatch, to report what the shader costs. + queries: vk::QueryPool, + timestamp_ns: f64, + lut_buffer: vk::Buffer, + lut_memory: vk::DeviceMemory, + params: Params, + layout: Nv12Layout, + targets: Vec, + sources: Vec, + tick: u64, + /// Where images handed to us by someone else get copied before converting. + copy: Option, + physical_device: vk::PhysicalDevice, +} + +/// How the shader reads the source. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Filter { + /// One bilinear read per pixel. + Bilinear, + /// Catmull-Rom: crisper, five reads per pixel. + Sharp, + /// Four averaged reads, for when the output shrinks the source a lot. + Supersample, +} + +/// How our GPU work ranks against the game's. The scanout buffer gets reused +/// a frame later, so work that waits for the game to leave the GPU idle can +/// miss frames in heavy games. +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum Priority { + /// Only runs when the game leaves the GPU idle. + Low, + /// Takes turns with the game. + Medium, + /// Cuts in ahead of the game, briefly. + High, +} + +impl Priority { + fn vk(self) -> vk::QueueGlobalPriorityKHR { + match self { + Priority::Low => vk::QueueGlobalPriorityKHR::LOW, + Priority::Medium => vk::QueueGlobalPriorityKHR::MEDIUM, + Priority::High => vk::QueueGlobalPriorityKHR::HIGH, + } + } +} + +/// Extensions to enable beyond our own: fixed ones for the instance, and a +/// function giving the device ones for the physical device we pick. +pub struct Extensions { + pub instance: Vec, + pub device: Box Vec>, +} + +/// Raw handles for handing our device to other Vulkan users. +pub struct NativeHandles { + pub instance: vk::Instance, + pub device: vk::Device, + pub physical_device: vk::PhysicalDevice, + pub queue: vk::Queue, + pub queue_family: u32, +} + +struct CopyTarget { + pool: vk::CommandPool, + cmd: vk::CommandBuffer, + image: vk::Image, + width: u32, + height: u32, +} + +/// Framebuffer id used for the copy source. +const COPY_SOURCE: u32 = u32::MAX; + +impl Gpu { + /// `extra` adds extensions someone else needs on our instance and device + /// (SteamVR, to hand us its headset view). + pub fn new( + layout: Nv12Layout, + target_count: usize, + lut: &Lut, + filter: Filter, + priority: Priority, + extra: Option<&Extensions>, + ) -> Result { + let entry = unsafe { ash::Entry::load() }.context("loading Vulkan")?; + let app = vk::ApplicationInfo::default() + .application_name(c"framely-capture") + .api_version(vk::API_VERSION_1_1); + let instance_exts: Vec<*const c_char> = extra.map_or_else(Vec::new, |e| e.instance.iter().map(|x| x.as_ptr()).collect()); + let instance = unsafe { + entry.create_instance( + &vk::InstanceCreateInfo::default().application_info(&app).enabled_extension_names(&instance_exts), + None, + ) + } + .context("creating Vulkan instance")?; + + Self::with_instance(entry, instance, layout, target_count, lut, filter, priority, extra) + } + + #[allow(clippy::too_many_arguments)] + fn with_instance( + entry: ash::Entry, + instance: ash::Instance, + layout: Nv12Layout, + target_count: usize, + lut: &Lut, + filter: Filter, + priority: Priority, + extra: Option<&Extensions>, + ) -> Result { + let pdev = unsafe { instance.enumerate_physical_devices() }? + .into_iter() + .next() + .context("no Vulkan device")?; + let props = unsafe { instance.get_physical_device_properties(pdev) }; + let name = unsafe { CStr::from_ptr(props.device_name.as_ptr()) }; + log::info!("gpu: {}", name.to_string_lossy()); + + let queue_family = unsafe { instance.get_physical_device_queue_family_properties(pdev) } + .iter() + .position(|q| q.queue_flags.contains(vk::QueueFlags::COMPUTE)) + .context("no compute queue")? as u32; + + let mut exts: Vec<*const c_char> = vec![ + ash::khr::external_memory_fd::NAME.as_ptr(), + ash::ext::external_memory_dma_buf::NAME.as_ptr(), + ash::ext::image_drm_format_modifier::NAME.as_ptr(), + // Required by the modifier extension on Vulkan 1.1. + ash::khr::image_format_list::NAME.as_ptr(), + ash::ext::queue_family_foreign::NAME.as_ptr(), + ]; + let wanted = extra.map(|e| (e.device)(pdev)).unwrap_or_default(); + for name in &wanted { + if !exts.iter().any(|&e| unsafe { CStr::from_ptr(e) } == name.as_c_str()) { + exts.push(name.as_ptr()); + } + } + // Last, so it can be dropped if the driver refuses the priority. + exts.push(ash::ext::global_priority::NAME.as_ptr()); + let priorities = [0.0f32]; + let device = [true, false] + .into_iter() + .find_map(|with_priority| { + let mut prio = vk::DeviceQueueGlobalPriorityCreateInfoKHR::default().global_priority(priority.vk()); + let mut qinfo = vk::DeviceQueueCreateInfo::default() + .queue_family_index(queue_family) + .queue_priorities(&priorities); + if with_priority { + qinfo = qinfo.push_next(&mut prio); + } + let qinfos = [qinfo]; + let n = if with_priority { exts.len() } else { exts.len() - 1 }; + let info = vk::DeviceCreateInfo::default() + .queue_create_infos(&qinfos) + .enabled_extension_names(&exts[..n]); + match unsafe { instance.create_device(pdev, &info, None) } { + Ok(d) => Some(d), + Err(e) => { + log::warn!("device creation failed ({priority:?} priority queue: {with_priority}): {e}"); + None + } + } + }) + .context("creating Vulkan device")?; + + let ext_fd = ash::khr::external_memory_fd::Device::new(&instance, &device); + let mem_props = unsafe { instance.get_physical_device_memory_properties(pdev) }; + let queue = unsafe { device.get_device_queue(queue_family, 0) }; + + let mut gpu = Self { + _entry: entry, + instance, + device, + ext_fd, + mem_props, + timestamp_ns: props.limits.timestamp_period as f64, + queue, + queue_family, + cmd_pool: vk::CommandPool::null(), + desc_pool: vk::DescriptorPool::null(), + set_layout: vk::DescriptorSetLayout::null(), + pipeline_layout: vk::PipelineLayout::null(), + pipeline: vk::Pipeline::null(), + shader: vk::ShaderModule::null(), + sampler: vk::Sampler::null(), + fence: vk::Fence::null(), + queries: vk::QueryPool::null(), + lut_buffer: vk::Buffer::null(), + lut_memory: vk::DeviceMemory::null(), + params: Params { + out_size: [layout.width, layout.height], + lut_size: [lut.width, lut.height], + y_stride: layout.y_stride, + uv_offset: layout.uv_offset, + lut_step: lut.step, + flags: (lut.per_channel as u32) + | (((filter == Filter::Supersample) as u32) << 1) + | ((lut.rotated_eyes as u32) << 2) + | (((filter == Filter::Sharp) as u32) << 3), + }, + layout, + targets: Vec::new(), + sources: Vec::new(), + tick: 0, + copy: None, + physical_device: pdev, + }; + gpu.create_pipeline(target_count)?; + gpu.upload_lut(lut)?; + for _ in 0..target_count { + let t = gpu.create_target()?; + gpu.targets.push(t); + } + Ok(gpu) + } + + pub fn targets(&self) -> &[Target] { + &self.targets + } + + fn create_pipeline(&mut self, target_count: usize) -> Result<()> { + let d = &self.device; + unsafe { + let code = ash::util::read_spv(&mut std::io::Cursor::new(SHADER))?; + self.shader = d.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&code), None)?; + + let bindings = [ + vk::DescriptorSetLayoutBinding::default() + .binding(0) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::COMPUTE), + vk::DescriptorSetLayoutBinding::default() + .binding(1) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::COMPUTE), + vk::DescriptorSetLayoutBinding::default() + .binding(2) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::COMPUTE), + ]; + self.set_layout = d.create_descriptor_set_layout( + &vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings), + None, + )?; + + let ranges = [vk::PushConstantRange::default() + .stage_flags(vk::ShaderStageFlags::COMPUTE) + .size(std::mem::size_of::() as u32)]; + let set_layouts = [self.set_layout]; + self.pipeline_layout = d.create_pipeline_layout( + &vk::PipelineLayoutCreateInfo::default() + .set_layouts(&set_layouts) + .push_constant_ranges(&ranges), + None, + )?; + + let stage = vk::PipelineShaderStageCreateInfo::default() + .stage(vk::ShaderStageFlags::COMPUTE) + .module(self.shader) + .name(c"main"); + let info = vk::ComputePipelineCreateInfo::default() + .stage(stage) + .layout(self.pipeline_layout); + self.pipeline = d + .create_compute_pipelines(vk::PipelineCache::null(), &[info], None) + .map_err(|(_, e)| e)?[0]; + + self.sampler = d.create_sampler( + &vk::SamplerCreateInfo::default() + .mag_filter(vk::Filter::LINEAR) + .min_filter(vk::Filter::LINEAR) + .mipmap_mode(vk::SamplerMipmapMode::NEAREST) + .address_mode_u(vk::SamplerAddressMode::CLAMP_TO_EDGE) + .address_mode_v(vk::SamplerAddressMode::CLAMP_TO_EDGE) + .address_mode_w(vk::SamplerAddressMode::CLAMP_TO_EDGE), + None, + )?; + + let max_sets = (MAX_SOURCES * target_count) as u32; + let sizes = [ + vk::DescriptorPoolSize::default() + .ty(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .descriptor_count(max_sets), + vk::DescriptorPoolSize::default() + .ty(vk::DescriptorType::STORAGE_BUFFER) + .descriptor_count(max_sets * 2), + ]; + self.desc_pool = d.create_descriptor_pool( + &vk::DescriptorPoolCreateInfo::default() + .flags(vk::DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET) + .max_sets(max_sets) + .pool_sizes(&sizes), + None, + )?; + + self.cmd_pool = d.create_command_pool( + &vk::CommandPoolCreateInfo::default().queue_family_index(self.queue_family), + None, + )?; + self.fence = d.create_fence(&vk::FenceCreateInfo::default(), None)?; + self.queries = d.create_query_pool( + &vk::QueryPoolCreateInfo::default().query_type(vk::QueryType::TIMESTAMP).query_count(2), + None, + )?; + } + Ok(()) + } + + fn find_memory_type(&self, bits: u32, want: vk::MemoryPropertyFlags) -> Option { + (0..self.mem_props.memory_type_count).find(|&i| { + bits & (1 << i) != 0 + && self.mem_props.memory_types[i as usize].property_flags.contains(want) + }) + } + + fn upload_lut(&mut self, lut: &Lut) -> Result<()> { + let bytes: &[u8] = unsafe { + std::slice::from_raw_parts(lut.points.as_ptr().cast(), std::mem::size_of_val(lut.points.as_slice())) + }; + let d = &self.device; + unsafe { + self.lut_buffer = d.create_buffer( + &vk::BufferCreateInfo::default() + .size(bytes.len() as u64) + .usage(vk::BufferUsageFlags::STORAGE_BUFFER), + None, + )?; + let req = d.get_buffer_memory_requirements(self.lut_buffer); + let ty = self + .find_memory_type( + req.memory_type_bits, + vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT, + ) + .context("no host visible memory for the lookup grid")?; + self.lut_memory = d.allocate_memory( + &vk::MemoryAllocateInfo::default().allocation_size(req.size).memory_type_index(ty), + None, + )?; + d.bind_buffer_memory(self.lut_buffer, self.lut_memory, 0)?; + let ptr = d.map_memory(self.lut_memory, 0, vk::WHOLE_SIZE, vk::MemoryMapFlags::empty())?; + std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.cast(), bytes.len()); + d.unmap_memory(self.lut_memory); + } + Ok(()) + } + + fn create_target(&self) -> Result { + let d = &self.device; + let handle = vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT; + unsafe { + let mut ext = vk::ExternalMemoryBufferCreateInfo::default().handle_types(handle); + let buffer = d.create_buffer( + &vk::BufferCreateInfo::default() + .size(self.layout.size as u64) + .usage(vk::BufferUsageFlags::STORAGE_BUFFER) + .push_next(&mut ext), + None, + )?; + let req = d.get_buffer_memory_requirements(buffer); + let ty = self + .find_memory_type(req.memory_type_bits, vk::MemoryPropertyFlags::DEVICE_LOCAL) + .context("no device memory for encoder buffers")?; + let mut export = vk::ExportMemoryAllocateInfo::default().handle_types(handle); + let mut dedicated = vk::MemoryDedicatedAllocateInfo::default().buffer(buffer); + let memory = d.allocate_memory( + &vk::MemoryAllocateInfo::default() + .allocation_size(req.size) + .memory_type_index(ty) + .push_next(&mut export) + .push_next(&mut dedicated), + None, + )?; + d.bind_buffer_memory(buffer, memory, 0)?; + let fd = self.ext_fd.get_memory_fd( + &vk::MemoryGetFdInfoKHR::default().memory(memory).handle_type(handle), + )?; + Ok(Target { buffer, memory, fd: OwnedFd::from_raw_fd(fd) }) + } + } + + pub fn has_source(&self, fb_id: u32, buffer: u64) -> bool { + self.sources.iter().any(|s| s.fb_id == fb_id && s.buffer == buffer) + } + + /// Forgets every imported scanout buffer. + pub fn clear_sources(&mut self) { + let (keep, gone): (Vec<_>, Vec<_>) = std::mem::take(&mut self.sources).into_iter().partition(|s| s.fb_id == COPY_SOURCE); + self.sources = keep; + for src in gone { + self.destroy_source(src); + } + } + + /// Imports a scanout buffer. The compositor flips between a few buffers, + /// so this happens a couple of times at startup and then never again. + pub fn add_source(&mut self, buf: ScanoutBuffer, buffer: u64) -> Result<()> { + // A recycled framebuffer id now points at a different buffer. + if let Some(i) = self.sources.iter().position(|s| s.fb_id == buf.fb_id) { + let src = self.sources.swap_remove(i); + self.destroy_source(src); + } + if self.sources.len() >= MAX_SOURCES { + let oldest = (0..self.sources.len()) + .min_by_key(|&i| self.sources[i].last_used) + .unwrap(); + let src = self.sources.swap_remove(oldest); + self.destroy_source(src); + } + let mut src = Source { + fb_id: buf.fb_id, + buffer, + last_used: 0, + image: vk::Image::null(), + memory: vk::DeviceMemory::null(), + view: vk::ImageView::null(), + sets: Vec::new(), + cmds: Vec::new(), + }; + match self.import(buf, &mut src) { + Ok(()) => { + self.sources.push(src); + Ok(()) + } + Err(e) => { + self.destroy_source(src); + Err(e) + } + } + } + + /// Fills in `src`; whatever got created is in there even on failure. + fn import(&self, buf: ScanoutBuffer, src: &mut Source) -> Result<()> { + let format = match buf.fourcc { + DRM_FORMAT_XRGB8888 | DRM_FORMAT_ARGB8888 => vk::Format::B8G8R8A8_UNORM, + DRM_FORMAT_XBGR8888 | DRM_FORMAT_ABGR8888 => vk::Format::R8G8B8A8_UNORM, + other => bail!("unsupported scanout format {:?}", other.to_le_bytes().map(char::from)), + }; + log::info!( + "importing scanout fb {} ({}x{}, modifier {:#x}, pitch {})", + buf.fb_id, + buf.width, + buf.height, + buf.modifier, + buf.pitch + ); + + let d = &self.device; + let handle = vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT; + unsafe { + let planes = [vk::SubresourceLayout { + offset: buf.offset as u64, + size: 0, + row_pitch: buf.pitch as u64, + array_pitch: 0, + depth_pitch: 0, + }]; + let mut modifier = vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default() + .drm_format_modifier(buf.modifier) + .plane_layouts(&planes); + let mut ext = vk::ExternalMemoryImageCreateInfo::default().handle_types(handle); + src.image = d + .create_image( + &vk::ImageCreateInfo::default() + .image_type(vk::ImageType::TYPE_2D) + .format(format) + .extent(vk::Extent3D { width: buf.width, height: buf.height, depth: 1 }) + .mip_levels(1) + .array_layers(1) + .samples(vk::SampleCountFlags::TYPE_1) + .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(vk::ImageUsageFlags::SAMPLED) + .sharing_mode(vk::SharingMode::EXCLUSIVE) + .initial_layout(vk::ImageLayout::UNDEFINED) + .push_next(&mut modifier) + .push_next(&mut ext), + None, + ) + .context("creating image for the scanout buffer (modifier not supported?)")?; + let image = src.image; + + let mut fd_props = vk::MemoryFdPropertiesKHR::default(); + self.ext_fd + .get_memory_fd_properties(handle, buf.fd.as_raw_fd(), &mut fd_props)?; + let req = d.get_image_memory_requirements(image); + let ty = self + .find_memory_type(req.memory_type_bits & fd_props.memory_type_bits, vk::MemoryPropertyFlags::empty()) + .context("no memory type can import the scanout buffer")?; + + // Vulkan takes ownership of the fd on success. + let fd = buf.fd.into_raw_fd(); + let mut import = vk::ImportMemoryFdInfoKHR::default().handle_type(handle).fd(fd); + let mut dedicated = vk::MemoryDedicatedAllocateInfo::default().image(image); + src.memory = match d.allocate_memory( + &vk::MemoryAllocateInfo::default() + .allocation_size(req.size) + .memory_type_index(ty) + .push_next(&mut import) + .push_next(&mut dedicated), + None, + ) { + Ok(m) => m, + Err(e) => { + libc::close(fd); + return Err(e).context("importing scanout memory"); + } + }; + d.bind_image_memory(image, src.memory, 0)?; + + src.view = d.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(image) + .view_type(vk::ImageViewType::TYPE_2D) + .format(format) + .subresource_range(color_range()), + None, + )?; + + let layouts = vec![self.set_layout; self.targets.len()]; + src.sets = d.allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(self.desc_pool) + .set_layouts(&layouts), + )?; + src.cmds = d.allocate_command_buffers( + &vk::CommandBufferAllocateInfo::default() + .command_pool(self.cmd_pool) + .level(vk::CommandBufferLevel::PRIMARY) + .command_buffer_count(self.targets.len() as u32), + )?; + + for (i, target) in self.targets.iter().enumerate() { + self.write_set(src.sets[i], src.view, target); + self.record(src.cmds[i], src.sets[i], image, target, true)?; + } + Ok(()) + } + } + + unsafe fn write_set(&self, set: vk::DescriptorSet, view: vk::ImageView, target: &Target) { + let image_info = [vk::DescriptorImageInfo::default() + .sampler(self.sampler) + .image_view(view) + .image_layout(vk::ImageLayout::GENERAL)]; + let lut_info = [vk::DescriptorBufferInfo::default().buffer(self.lut_buffer).range(vk::WHOLE_SIZE)]; + let out_info = [vk::DescriptorBufferInfo::default().buffer(target.buffer).range(vk::WHOLE_SIZE)]; + let writes = [ + vk::WriteDescriptorSet::default() + .dst_set(set) + .dst_binding(0) + .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER) + .image_info(&image_info), + vk::WriteDescriptorSet::default() + .dst_set(set) + .dst_binding(1) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .buffer_info(&lut_info), + vk::WriteDescriptorSet::default() + .dst_set(set) + .dst_binding(2) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .buffer_info(&out_info), + ]; + self.device.update_descriptor_sets(&writes, &[]); + } + + /// `external` images belong to someone else (the display) and get + /// borrowed and handed back around the dispatch; our own copy doesn't. + unsafe fn record( + &self, + cmd: vk::CommandBuffer, + set: vk::DescriptorSet, + image: vk::Image, + target: &Target, + external: bool, + ) -> Result<()> { + let d = &self.device; + let foreign = vk::QUEUE_FAMILY_FOREIGN_EXT; + let ours = self.queue_family; + let (img_from, img_to) = if external { (foreign, ours) } else { (vk::QUEUE_FAMILY_IGNORED, vk::QUEUE_FAMILY_IGNORED) }; + d.begin_command_buffer(cmd, &vk::CommandBufferBeginInfo::default())?; + d.cmd_reset_query_pool(cmd, self.queries, 0, 2); + + // Borrow the scanout image from the display and the NV12 buffer from + // the encoder. GENERAL in and out, so nothing gets decompressed or + // discarded behind the compositor's back. + let acquire_image = vk::ImageMemoryBarrier::default() + .src_queue_family_index(img_from) + .dst_queue_family_index(img_to) + .old_layout(vk::ImageLayout::GENERAL) + .new_layout(vk::ImageLayout::GENERAL) + .dst_access_mask(vk::AccessFlags::SHADER_READ) + .image(image) + .subresource_range(color_range()); + let acquire_buffer = vk::BufferMemoryBarrier::default() + .src_queue_family_index(foreign) + .dst_queue_family_index(ours) + .dst_access_mask(vk::AccessFlags::SHADER_WRITE) + .buffer(target.buffer) + .size(vk::WHOLE_SIZE); + d.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::DependencyFlags::empty(), + &[], + &[acquire_buffer], + &[acquire_image], + ); + + d.cmd_bind_pipeline(cmd, vk::PipelineBindPoint::COMPUTE, self.pipeline); + d.cmd_bind_descriptor_sets(cmd, vk::PipelineBindPoint::COMPUTE, self.pipeline_layout, 0, &[set], &[]); + let params: [u8; std::mem::size_of::()] = std::mem::transmute(self.params); + d.cmd_push_constants(cmd, self.pipeline_layout, vk::ShaderStageFlags::COMPUTE, 0, ¶ms); + let groups_x = (self.layout.width / 4).div_ceil(8); + let groups_y = (self.layout.height / 2).div_ceil(8); + d.cmd_write_timestamp(cmd, vk::PipelineStageFlags::TOP_OF_PIPE, self.queries, 0); + d.cmd_dispatch(cmd, groups_x, groups_y, 1); + d.cmd_write_timestamp(cmd, vk::PipelineStageFlags::BOTTOM_OF_PIPE, self.queries, 1); + + let release_image = vk::ImageMemoryBarrier::default() + .src_queue_family_index(img_to) + .dst_queue_family_index(img_from) + .old_layout(vk::ImageLayout::GENERAL) + .new_layout(vk::ImageLayout::GENERAL) + .src_access_mask(vk::AccessFlags::SHADER_READ) + .image(image) + .subresource_range(color_range()); + let release_buffer = vk::BufferMemoryBarrier::default() + .src_queue_family_index(ours) + .dst_queue_family_index(foreign) + .src_access_mask(vk::AccessFlags::SHADER_WRITE) + .buffer(target.buffer) + .size(vk::WHOLE_SIZE); + d.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::PipelineStageFlags::BOTTOM_OF_PIPE, + vk::DependencyFlags::empty(), + &[], + &[release_buffer], + &[release_image], + ); + d.end_command_buffer(cmd)?; + Ok(()) + } + + /// Converts the given scanout buffer into target `index` and waits for the + /// GPU to finish, so the encoder can take the buffer right away. + /// Returns how long the shader ran on the GPU. + pub fn convert(&mut self, fb_id: u32, buffer: u64, index: usize) -> Result { + self.tick += 1; + let tick = self.tick; + let src = self + .sources + .iter_mut() + .find(|s| s.fb_id == fb_id && s.buffer == buffer) + .context("scanout buffer wasn't imported")?; + src.last_used = tick; + let cmd = src.cmds[index]; + self.submit(&[cmd]) + } + + /// Runs command buffers, waits for them, and returns the conversion + /// shader's GPU time from the timestamps it wrote. + fn submit(&self, cmds: &[vk::CommandBuffer]) -> Result { + unsafe { + self.device.queue_submit( + self.queue, + &[vk::SubmitInfo::default().command_buffers(cmds)], + self.fence, + )?; + self.device.wait_for_fences(&[self.fence], true, 1_000_000_000)?; + self.device.reset_fences(&[self.fence])?; + + let mut stamps = [0u64; 2]; + self.device + .get_query_pool_results(self.queries, 0, &mut stamps, vk::QueryResultFlags::TYPE_64)?; + let ns = stamps[1].saturating_sub(stamps[0]) as f64 * self.timestamp_ns; + Ok(std::time::Duration::from_nanos(ns as u64)) + } + } + + fn destroy_source(&self, src: Source) { + unsafe { + let d = &self.device; + let _ = d.device_wait_idle(); + if !src.cmds.is_empty() { + d.free_command_buffers(self.cmd_pool, &src.cmds); + } + if !src.sets.is_empty() { + let _ = d.free_descriptor_sets(self.desc_pool, &src.sets); + } + d.destroy_image_view(src.view, None); + d.destroy_image(src.image, None); + d.free_memory(src.memory, None); + } + } +} + +fn color_range() -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + } +} + +impl Drop for Gpu { + fn drop(&mut self) { + unsafe { + let _ = self.device.device_wait_idle(); + for src in std::mem::take(&mut self.sources) { + self.destroy_source(src); + } + let d = &self.device; + if let Some(copy) = self.copy.take() { + d.destroy_command_pool(copy.pool, None); + } + for t in self.targets.drain(..) { + d.destroy_buffer(t.buffer, None); + d.free_memory(t.memory, None); + } + d.destroy_buffer(self.lut_buffer, None); + d.free_memory(self.lut_memory, None); + d.destroy_fence(self.fence, None); + d.destroy_query_pool(self.queries, None); + d.destroy_command_pool(self.cmd_pool, None); + d.destroy_descriptor_pool(self.desc_pool, None); + d.destroy_sampler(self.sampler, None); + d.destroy_pipeline(self.pipeline, None); + d.destroy_pipeline_layout(self.pipeline_layout, None); + d.destroy_descriptor_set_layout(self.set_layout, None); + d.destroy_shader_module(self.shader, None); + d.destroy_device(None); + self.instance.destroy_instance(None); + } + } +} diff --git a/media/capture/src/gpu_copy.rs b/media/capture/src/gpu_copy.rs new file mode 100644 index 0000000..3ff2a53 --- /dev/null +++ b/media/capture/src/gpu_copy.rs @@ -0,0 +1,168 @@ +//! Converting images someone else hands us on our own device (SteamVR's +//! headset view): each frame gets copied into an image of ours, then runs +//! through the same conversion as the panel scanout. + +use anyhow::{Context, Result}; +use ash::vk; + +use super::{color_range, CopyTarget, Gpu, NativeHandles, Source, COPY_SOURCE}; + +impl Gpu { + pub fn native(&self) -> NativeHandles { + NativeHandles { + instance: self.instance.handle(), + device: self.device.handle(), + physical_device: self.physical_device, + queue: self.queue, + queue_family: self.queue_family, + } + } + + /// Size of the image frames get copied into, if it's set up. + pub fn copy_size(&self) -> Option<(u32, u32)> { + self.copy.as_ref().map(|c| (c.width, c.height)) + } + + /// Sets up (or replaces) the image frames get copied into, `width`x`height` RGBA. + pub fn add_copy_source(&mut self, width: u32, height: u32) -> Result<()> { + if let Some(i) = self.sources.iter().position(|s| s.fb_id == COPY_SOURCE) { + let old = self.sources.swap_remove(i); + self.destroy_source(old); + } + if let Some(old) = self.copy.take() { + unsafe { self.device.destroy_command_pool(old.pool, None) }; + } + log::info!("copying {width}x{height} frames from SteamVR"); + let d = &self.device; + let mut src = Source { + fb_id: COPY_SOURCE, + buffer: 0, + last_used: 0, + image: vk::Image::null(), + memory: vk::DeviceMemory::null(), + view: vk::ImageView::null(), + sets: Vec::new(), + cmds: Vec::new(), + }; + let format = vk::Format::R8G8B8A8_UNORM; + unsafe { + src.image = d.create_image( + &vk::ImageCreateInfo::default() + .image_type(vk::ImageType::TYPE_2D) + .format(format) + .extent(vk::Extent3D { width, height, depth: 1 }) + .mip_levels(1) + .array_layers(1) + .samples(vk::SampleCountFlags::TYPE_1) + .tiling(vk::ImageTiling::OPTIMAL) + .usage(vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST) + .initial_layout(vk::ImageLayout::UNDEFINED), + None, + )?; + let req = d.get_image_memory_requirements(src.image); + let ty = self + .find_memory_type(req.memory_type_bits, vk::MemoryPropertyFlags::DEVICE_LOCAL) + .context("no memory for the copy image")?; + src.memory = d.allocate_memory(&vk::MemoryAllocateInfo::default().allocation_size(req.size).memory_type_index(ty), None)?; + d.bind_image_memory(src.image, src.memory, 0)?; + src.view = d.create_image_view( + &vk::ImageViewCreateInfo::default() + .image(src.image) + .view_type(vk::ImageViewType::TYPE_2D) + .format(format) + .subresource_range(color_range()), + None, + )?; + let layouts = vec![self.set_layout; self.targets.len()]; + src.sets = d.allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default().descriptor_pool(self.desc_pool).set_layouts(&layouts), + )?; + src.cmds = d.allocate_command_buffers( + &vk::CommandBufferAllocateInfo::default() + .command_pool(self.cmd_pool) + .level(vk::CommandBufferLevel::PRIMARY) + .command_buffer_count(self.targets.len() as u32), + )?; + for (i, target) in self.targets.iter().enumerate() { + self.write_set(src.sets[i], src.view, target); + self.record(src.cmds[i], src.sets[i], src.image, target, false)?; + } + + let pool = d.create_command_pool( + &vk::CommandPoolCreateInfo::default() + .queue_family_index(self.queue_family) + .flags(vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER), + None, + )?; + let cmd = d.allocate_command_buffers(&vk::CommandBufferAllocateInfo::default().command_pool(pool).command_buffer_count(1))?[0]; + self.copy = Some(CopyTarget { pool, cmd, image: src.image, width, height }); + } + self.sources.push(src); + Ok(()) + } + + /// Copies `from` (in TRANSFER_SRC_OPTIMAL, the region starting at + /// `origin`) into our image and converts it into target `index`. + /// Returns how long the conversion shader ran. + pub fn convert_copied(&mut self, from: vk::Image, origin: (i32, i32), index: usize) -> Result { + let copy = self.copy.as_ref().context("no copy source set up")?; + let (cmd, to, width, height) = (copy.cmd, copy.image, copy.width, copy.height); + let d = &self.device; + unsafe { + d.reset_command_buffer(cmd, vk::CommandBufferResetFlags::empty())?; + d.begin_command_buffer(cmd, &vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT))?; + let to_dst = vk::ImageMemoryBarrier::default() + .old_layout(vk::ImageLayout::UNDEFINED) + .new_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) + .src_access_mask(vk::AccessFlags::SHADER_READ) + .dst_access_mask(vk::AccessFlags::TRANSFER_WRITE) + .image(to) + .subresource_range(color_range()); + d.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::PipelineStageFlags::TRANSFER, + vk::DependencyFlags::empty(), + &[], + &[], + &[to_dst], + ); + let layers = vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }; + let region = vk::ImageCopy::default() + .src_subresource(layers) + .src_offset(vk::Offset3D { x: origin.0, y: origin.1, z: 0 }) + .dst_subresource(layers) + .extent(vk::Extent3D { width, height, depth: 1 }); + d.cmd_copy_image(cmd, from, vk::ImageLayout::TRANSFER_SRC_OPTIMAL, to, vk::ImageLayout::TRANSFER_DST_OPTIMAL, &[region]); + let to_general = vk::ImageMemoryBarrier::default() + .old_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) + .new_layout(vk::ImageLayout::GENERAL) + .src_access_mask(vk::AccessFlags::TRANSFER_WRITE) + .dst_access_mask(vk::AccessFlags::SHADER_READ) + .image(to) + .subresource_range(color_range()); + d.cmd_pipeline_barrier( + cmd, + vk::PipelineStageFlags::TRANSFER, + vk::PipelineStageFlags::COMPUTE_SHADER, + vk::DependencyFlags::empty(), + &[], + &[], + &[to_general], + ); + d.end_command_buffer(cmd)?; + } + let convert = self + .sources + .iter() + .find(|s| s.fb_id == COPY_SOURCE) + .context("no copy source set up")? + .cmds[index]; + self.submit(&[cmd, convert]) + } +} diff --git a/media/capture/src/grab.rs b/media/capture/src/grab.rs new file mode 100644 index 0000000..9f8d9b3 --- /dev/null +++ b/media/capture/src/grab.rs @@ -0,0 +1,243 @@ +//! Framely's root framebuffer export helper and its private session socket. +//! The worker holds no capabilities. The core starts a version-matched helper, +//! which exports framebuffer DMA-BUF handles until the session socket closes. + +use std::io::{Read, Write}; +use std::os::fd::{AsFd, AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::os::unix::net::UnixStream; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use drm::control::{framebuffer, Device as ControlDevice}; + +/// IPC protocol shared by the packaged helper and capture worker. +pub const VERSION: u32 = 1; +const REPLY_LEN: usize = 32; + +/// One scanout buffer, as a dmabuf. +pub struct Exported { + pub fd: OwnedFd, + pub width: u32, + pub height: u32, + pub fourcc: u32, + pub modifier: u64, + pub pitch: u32, + pub offset: u32, +} + +/// Exports a framebuffer, which only works with the permission. None when +/// the kernel kept the buffer to itself, i.e. without it. +pub fn export(card: &D, fb_id: u32) -> Result> { + let handle: framebuffer::Handle = drm::control::from_u32(fb_id).context("bad framebuffer id")?; + let info = card.get_planar_framebuffer(handle).with_context(|| format!("reading framebuffer {fb_id}"))?; + let buffers = info.buffers(); + let Some(bo) = buffers[0] else { return Ok(None) }; + if buffers[1..].iter().any(Option::is_some) { + bail!("multi-plane scanout buffers aren't supported yet"); + } + let fd = card.buffer_to_prime_fd(bo, libc::O_CLOEXEC as u32).context("exporting scanout buffer as dmabuf"); + // The export holds its own reference, the GEM handle is ours to drop. + let _ = card.close_buffer(bo); + let (width, height) = info.size(); + Ok(Some(Exported { + fd: fd?, + width, + height, + fourcc: info.pixel_format() as u32, + modifier: info.modifier().map(u64::from).unwrap_or(0), + pitch: info.pitches()[0], + offset: info.offsets()[0], + })) +} + +struct Card(std::fs::File); + +impl AsFd for Card { + fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl drm::Device for Card {} +impl ControlDevice for Card {} + +/// The helper's side: says its version, then answers framebuffer ids on the +/// socket it was started with (its stdin) until the recorder hangs up. +pub fn serve(card_path: &Path) -> Result<()> { + // Only ever the display, nothing else gets opened with the permission. + use std::os::unix::fs::FileTypeExt; + let real = std::fs::canonicalize(card_path).with_context(|| format!("finding {}", card_path.display()))?; + let is_device = std::fs::metadata(&real).is_ok_and(|m| m.file_type().is_char_device()); + if !real.to_string_lossy().starts_with("/dev/dri/card") || !is_device { + bail!("{} isn't a display", card_path.display()); + } + let file = std::fs::OpenOptions::new().read(true).write(true).open(&real); + let card = Card(file.with_context(|| format!("opening {}", real.display()))?); + let mut sock = unsafe { UnixStream::from_raw_fd(0) }; + sock.write_all(&VERSION.to_le_bytes())?; + let mut id = [0u8; 4]; + loop { + match sock.read_exact(&mut id) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), + Err(e) => return Err(e.into()), + } + let fb_id = u32::from_le_bytes(id); + match export(&card, fb_id) { + Ok(Some(buf)) => send(&sock, &reply(0, &buf), Some(buf.fd.as_raw_fd()))?, + Ok(None) => { + eprintln!("framely-panel-grab: the kernel hid framebuffer {fb_id}, is the permission gone?"); + send(&sock, &[1; REPLY_LEN], None)? + } + Err(e) => { + eprintln!("framely-panel-grab: {e:#}"); + send(&sock, &[1; REPLY_LEN], None)? + } + } + } +} + +fn reply(status: u32, b: &Exported) -> [u8; REPLY_LEN] { + let mut out = [0u8; REPLY_LEN]; + for (i, v) in [status, b.width, b.height, b.fourcc, b.pitch, b.offset].into_iter().enumerate() { + out[i * 4..i * 4 + 4].copy_from_slice(&v.to_le_bytes()); + } + out[24..32].copy_from_slice(&b.modifier.to_le_bytes()); + out +} + +/// The recorder's side: the helper, started and asked per frame. +pub struct Grabber { + sock: UnixStream, +} + +impl Grabber { + /// A dedicated socket passed by Framely's root supervisor. It grants no + /// general capability to the Steam user process and is never public. + pub fn inherited() -> Result> { + let Some(raw) = std::env::var_os("FRAMELY_GRAB_FD") else { return Ok(None) }; + std::env::remove_var("FRAMELY_GRAB_FD"); + let fd: RawFd = raw.to_string_lossy().parse().context("invalid helper fd")?; + if fd < 3 { bail!("invalid helper fd"); } + let mut sock = unsafe { UnixStream::from_raw_fd(fd) }; + unsafe { libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); } + sock.set_read_timeout(Some(std::time::Duration::from_secs(5)))?; + sock.set_write_timeout(Some(std::time::Duration::from_secs(5)))?; + let mut version = [0; 4]; + sock.read_exact(&mut version).context("Framely panel helper did not answer")?; + if u32::from_le_bytes(version) != VERSION { bail!("panel helper version mismatch"); } + Ok(Some(Self { sock })) + } + + pub fn export(&mut self, fb_id: u32) -> Result { + self.sock.write_all(&fb_id.to_le_bytes()).context("asking the panel helper")?; + let (bytes, fd) = recv(&self.sock)?; + let word = |i: usize| u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap()); + let (Some(fd), 0) = (fd, word(0)) else { bail!("the panel helper couldn't read framebuffer {fb_id}") }; + Ok(Exported { + fd, + width: word(1), + height: word(2), + fourcc: word(3), + pitch: word(4), + offset: word(5), + modifier: u64::from_le_bytes(bytes[24..32].try_into().unwrap()), + }) + } +} + + +fn send(sock: &UnixStream, bytes: &[u8; REPLY_LEN], fd: Option) -> Result<()> { + let mut iov = libc::iovec { iov_base: bytes.as_ptr() as *mut _, iov_len: bytes.len() }; + let space = unsafe { libc::CMSG_SPACE(std::mem::size_of::() as u32) } as usize; + let mut control = vec![0u8; space]; + let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; + msg.msg_iov = &mut iov; + msg.msg_iovlen = 1; + if let Some(fd) = fd { + msg.msg_control = control.as_mut_ptr().cast(); + msg.msg_controllen = space as _; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&msg); + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = libc::CMSG_LEN(std::mem::size_of::() as u32) as _; + std::ptr::write_unaligned(libc::CMSG_DATA(cmsg).cast::(), fd); + } + } + if unsafe { libc::sendmsg(sock.as_raw_fd(), &msg, libc::MSG_NOSIGNAL) } != REPLY_LEN as isize { + return Err(std::io::Error::last_os_error()).context("answering the recorder"); + } + Ok(()) +} + +fn recv(sock: &UnixStream) -> Result<([u8; REPLY_LEN], Option)> { + let mut bytes = [0u8; REPLY_LEN]; + let mut iov = libc::iovec { iov_base: bytes.as_mut_ptr().cast(), iov_len: bytes.len() }; + let space = unsafe { libc::CMSG_SPACE(std::mem::size_of::() as u32) } as usize; + let mut control = vec![0u8; space]; + let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; + msg.msg_iov = &mut iov; + msg.msg_iovlen = 1; + msg.msg_control = control.as_mut_ptr().cast(); + msg.msg_controllen = space as _; + let got = unsafe { libc::recvmsg(sock.as_raw_fd(), &mut msg, libc::MSG_CMSG_CLOEXEC | libc::MSG_WAITALL) }; + if got != REPLY_LEN as isize { + bail!("the panel helper hung up"); + } + let mut fd = None; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&msg); + if !cmsg.is_null() && (*cmsg).cmsg_level == libc::SOL_SOCKET && (*cmsg).cmsg_type == libc::SCM_RIGHTS { + fd = Some(OwnedFd::from_raw_fd(std::ptr::read_unaligned(libc::CMSG_DATA(cmsg).cast::()))); + } + } + Ok((bytes, fd)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supervisor_socket_uses_existing_helper_protocol() { + use std::os::fd::IntoRawFd; + let (ours, mut helper) = UnixStream::pair().unwrap(); + let server = std::thread::spawn(move || { + helper.write_all(&VERSION.to_le_bytes()).unwrap(); + let mut id = [0; 4]; helper.read_exact(&mut id).unwrap(); + assert_eq!(u32::from_le_bytes(id), 42); + let file = std::fs::File::open("/dev/null").unwrap(); + let buf = Exported {fd: OwnedFd::from(file),width:1920,height:1080,fourcc:0x34325258,modifier:0,pitch:7680,offset:0}; + send(&helper, &reply(0, &buf), Some(buf.fd.as_raw_fd())).unwrap(); + }); + std::env::set_var("FRAMELY_GRAB_FD", ours.into_raw_fd().to_string()); + let mut grabber = Grabber::inherited().unwrap().unwrap(); + assert!(std::env::var_os("FRAMELY_GRAB_FD").is_none()); + let buf = grabber.export(42).unwrap(); + assert_eq!((buf.width, buf.height), (1920, 1080)); + server.join().unwrap(); + } + + #[test] + fn replies_round_trip_with_a_file() { + let (a, b) = UnixStream::pair().unwrap(); + let file = std::fs::File::open("/dev/null").unwrap(); + let buf = Exported { + fd: OwnedFd::from(file), + width: 1920, + height: 1080, + fourcc: 0x34325258, + modifier: 0x0500000000000001, + pitch: 7680, + offset: 64, + }; + send(&a, &reply(0, &buf), Some(buf.fd.as_raw_fd())).unwrap(); + let (bytes, fd) = recv(&b).unwrap(); + assert!(fd.is_some()); + assert_eq!(bytes, reply(0, &buf)); + send(&a, &[1; REPLY_LEN], None).unwrap(); + let (_, fd) = recv(&b).unwrap(); + assert!(fd.is_none()); + } +} diff --git a/media/capture/src/kms.rs b/media/capture/src/kms.rs new file mode 100644 index 0000000..3a9c560 --- /dev/null +++ b/media/capture/src/kms.rs @@ -0,0 +1,214 @@ +//! Finds the plane the VR compositor scans out and hands its buffers over as +//! dmabufs. The compositor keeps DRM master; we only look. +//! +//! Getting buffer handles for someone else's framebuffer needs CAP_SYS_ADMIN. +//! The recorder uses it if it has it itself, the panel helper otherwise (see +//! framely_capture::grab). + +use std::fs::{File, OpenOptions}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; +use std::cell::{Cell, RefCell}; +use std::path::Path; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use drm::control::{crtc, plane, Device as ControlDevice}; +use framely_capture::grab; +use drm::{ClientCapability, Device, VblankWaitFlags, VblankWaitTarget}; + +struct Card(File); + +impl AsFd for Card { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl Device for Card {} +impl ControlDevice for Card {} + +/// One scanout buffer, exported as a dmabuf. +pub struct ScanoutBuffer { + pub fb_id: u32, + pub fd: OwnedFd, + pub width: u32, + pub height: u32, + pub fourcc: u32, + pub modifier: u64, + pub pitch: u32, + pub offset: u32, +} + +pub struct Kms { + card: Card, + helper: RefCell>, + pipe: u32, + plane: Cell, + crtc: crtc::Handle, + next_plane_probe: Cell, + pub refresh_hz: f64, + pub mode_size: (u32, u32), +} + +impl Kms { + pub fn open(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .with_context(|| format!("opening {}", path.display()))?; + let card = Card(file); + card.set_client_capability(ClientCapability::UniversalPlanes, true) + .context("enabling universal planes")?; + + let res = card.resource_handles().context("reading DRM resources")?; + let (pipe, crtc, mode) = res + .crtcs() + .iter() + .enumerate() + .find_map(|(i, &h)| { + let info = card.get_crtc(h).ok()?; + Some((i as u32, h, info.mode()?)) + }) + .context("no active display found, is the headset on?")?; + + let plane = find_scanout_plane(&card, crtc)?; + let (w, h) = mode.size(); + let refresh_hz = mode_refresh(&mode); + log::info!( + "display: {}x{} @ {:.2} Hz, crtc {:?}, plane {:?}", + w, + h, + refresh_hz, + crtc, + plane + ); + + Ok(Self { + card, + helper: RefCell::new(grab::Grabber::inherited()?), + pipe, + plane: Cell::new(plane), + crtc, + next_plane_probe: Cell::new(Instant::now()), + refresh_hz, + mode_size: (w as u32, h as u32), + }) + } + + /// Blocks until the next vblank and returns its CLOCK_MONOTONIC timestamp. + pub fn wait_vblank(&self) -> Result { + let reply = loop { + match self.card.wait_vblank(VblankWaitTarget::Relative(1), VblankWaitFlags::empty(), self.pipe, 0) { + // A pause or stop signal landed mid-wait; just wait again. + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + other => break other.context("waiting for vblank")?, + } + }; + Ok(reply.time().unwrap_or_else(crate::clock::now)) + } + + /// Id of the framebuffer being scanned out right now, if any. + pub fn current_fb(&self) -> Result> { + let info = self.card.get_plane(self.plane.get()).context("reading scanout plane")?; + if info.crtc() == Some(self.crtc) && info.framebuffer().is_some() { + return Ok(info.framebuffer().map(u32::from)); + } + // SteamVR can move scanout to another plane when waking or restarting. + // Keep normal capture cheap, and bound probes while the display is off. + if Instant::now() < self.next_plane_probe.get() { return Ok(None); } + self.next_plane_probe.set(Instant::now() + Duration::from_secs(1)); + let Ok(plane) = find_scanout_plane(&self.card, self.crtc) else { return Ok(None) }; + if plane != self.plane.get() { + log::info!("scanout moved to plane {plane:?}"); + self.plane.set(plane); + } + Ok(self.card.get_plane(plane).context("reading replacement scanout plane")?.framebuffer().map(u32::from)) + } + + pub fn export(&self, fb_id: u32) -> Result { + let buf = match grab::export(&self.card, fb_id)? { + Some(buf) => buf, + // No permission of our own: the helper has it, if it's installed. + None => self.export_through_helper(fb_id)?, + }; + Ok(ScanoutBuffer { + fb_id, + fd: buf.fd, + width: buf.width, + height: buf.height, + fourcc: buf.fourcc, + modifier: buf.modifier, + pitch: buf.pitch, + offset: buf.offset, + }) + } + + fn export_through_helper(&self, fb_id: u32) -> Result { + self.helper.borrow_mut().as_mut() + .context("Framely panel helper is not connected; start capture from Framely's casting page")? + .export(fb_id) + } + +} + +fn find_scanout_plane(card: &Card, crtc: crtc::Handle) -> Result { + let mut best: Option<(plane::Handle, u64)> = None; + for handle in card.plane_handles().context("listing planes")? { + let Ok(info) = card.get_plane(handle) else { continue }; + if info.crtc() != Some(crtc) { + continue; + } + let Some(fb) = info.framebuffer() else { continue }; + // GETFB2 exposes dimensions even when the unprivileged recorder cannot + // see GEM handles. Legacy GETFB can fail on modern scanout formats. + // If it fails for every plane, choosing the first one can select a cursor. + let props = card.get_properties(handle).ok(); + let mut cursor = false; + let (mut width, mut height) = (0, 0); + if let Some(props) = props { + for (&property, &value) in props.iter() { + let Ok(info) = card.get_property(property) else { continue }; + match info.name().to_bytes() { + b"type" => cursor = value == 2, // DRM_PLANE_TYPE_CURSOR + b"CRTC_W" => width = value, + b"CRTC_H" => height = value, + _ => {} + } + } + } + let size = card.get_planar_framebuffer(fb).map(|f| f.size()) + .or_else(|_| card.get_framebuffer(fb).map(|f| f.size())); + let area = size.map(|(w,h)| w as u64 * h as u64).unwrap_or(width.saturating_mul(height)); + if cursor || area == 0 { continue; } + if best.is_none_or(|(_, a)| area > a) { + best = Some((handle, area)); + } + } + best.map(|(h, _)| h).context("no plane is scanning out, is the VR compositor running?") +} + +fn mode_refresh(mode: &drm::control::Mode) -> f64 { + let (htotal, vtotal) = (mode.hsync().2 as f64, mode.vsync().2 as f64); + if htotal > 0.0 && vtotal > 0.0 { + mode.clock() as f64 * 1000.0 / (htotal * vtotal) + } else { + mode.vrefresh() as f64 + } +} + +/// Identifies the buffer behind a dmabuf fd. Every export of the same buffer +/// shares one dma_buf file, so its inode is stable for the buffer's lifetime. +pub fn buffer_id(fd: &OwnedFd) -> Result { + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { libc::fstat(fd.as_raw_fd(), &mut st) } != 0 { + return Err(std::io::Error::last_os_error()).context("looking at a scanout buffer"); + } + Ok(st.st_ino) +} + +/// Waits (briefly) until nobody is still drawing into the buffer. +pub fn wait_idle(fd: &OwnedFd, timeout: Duration) { + let mut pfd = libc::pollfd { fd: fd.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + unsafe { libc::poll(&mut pfd, 1, timeout.as_millis() as libc::c_int) }; +} diff --git a/media/capture/src/lib.rs b/media/capture/src/lib.rs new file mode 100644 index 0000000..0e58859 --- /dev/null +++ b/media/capture/src/lib.rs @@ -0,0 +1,2 @@ +pub mod openvr; +pub mod grab; diff --git a/media/capture/src/lut.rs b/media/capture/src/lut.rs new file mode 100644 index 0000000..8a34abd --- /dev/null +++ b/media/capture/src/lut.rs @@ -0,0 +1,467 @@ +//! The lookup grid the shader uses to find, for each output pixel, where on +//! the panel to read from. Built once at startup, so the per-frame GPU work is +//! just a few texture reads per pixel. + +use anyhow::{bail, Result}; + +use framely_capture::openvr::OpenVr; + +/// Output pixels between grid points. The mapping is smooth, 8 is plenty. +const STEP: u32 = 8; +/// Samples per side when reading SteamVR's distortion function. +const DISTORTION_GRID: usize = 97; +const INVALID: [f32; 2] = [-1.0, -1.0]; +/// Color fringes narrower than this (in output pixels) are left alone. +const MIN_FRINGE_PX: f64 = 0.35; +/// Supersample once an output pixel covers more than this many panel pixels. +const SUPERSAMPLE_ABOVE: f64 = 1.5; +/// Panel pixels across one eye. +const EYE_PX: f64 = 2160.0; +/// The Frame's compositor scans each eye out turned 180° in its own half, +/// while SteamVR's distortion data describes the panels upright. Checked on +/// a headset: straight ahead lands where the in-place rotation puts it (not +/// where a whole-display rotation would), and recordings came out upside +/// down without this. +const PANELS_ROTATED: bool = true; + +pub struct Lut { + pub view: Option<(f64, f64, f64)>, + pub width: u32, + pub height: u32, + pub step: f32, + /// Whether the red and blue channels have their own positions (they do + /// when the compositor corrects chromatic aberration). + pub per_channel: bool, + /// Whether the output shrinks the panel enough that single taps would + /// alias, so the shader should average a few. + pub supersample: bool, + /// Grid positions are in SteamVR's panel space; the scanout has each + /// eye's half turned 180° from that. See `PANELS_ROTATED`. + pub rotated_eyes: bool, + /// Scanout texture coordinates, 3 per grid point (r, g, b). + pub points: Vec<[f32; 2]>, +} + +impl Lut { + fn new(out_w: u32, out_h: u32) -> Self { + let width = out_w.div_ceil(STEP) + 1; + let height = out_h.div_ceil(STEP) + 1; + Self { + view: None, + width, + height, + step: STEP as f32, + per_channel: false, + supersample: false, + rotated_eyes: PANELS_ROTATED, + points: vec![INVALID; (width * height * 3) as usize], + } + } + + fn grid(&self) -> impl Iterator + '_ { + (0..self.height).flat_map(move |j| { + (0..self.width).map(move |i| (((j * self.width + i) * 3) as usize, (i * STEP) as f64, (j * STEP) as f64)) + }) + } +} + +/// The whole panel, both eyes, lens distortion and all, fitted into the +/// output with black bars. +pub fn raw(out_w: u32, out_h: u32, src_w: u32, src_h: u32) -> Lut { + let mut lut = Lut::new(out_w, out_h); + let scale = (out_w as f64 / src_w as f64).min(out_h as f64 / src_h as f64); + lut.supersample = 1.0 / scale > SUPERSAMPLE_ABOVE; + let (fit_w, fit_h) = (src_w as f64 * scale, src_h as f64 * scale); + let (ox, oy) = ((out_w as f64 - fit_w) / 2.0, (out_h as f64 - fit_h) / 2.0); + + let cells: Vec<_> = lut.grid().collect(); + for (idx, x, y) in cells { + let u = (x - ox) / fit_w; + let v = (y - oy) / fit_h; + // Let the grid run a bit past the picture so the edges interpolate + // cleanly, anything further out is a black bar. + let margin = STEP as f64 / fit_w.min(fit_h); + let p = if (-margin..=1.0 + margin).contains(&u) && (-margin..=1.0 + margin).contains(&v) { + [u.clamp(0.0, 1.0) as f32, v.clamp(0.0, 1.0) as f32] + } else { + INVALID + }; + lut.points[idx..idx + 3].fill(p); + } + lut +} + +/// Output edge points checked per side when looking for the widest clean view. +const EDGE_SAMPLES: usize = 64; +/// Pulls the automatic view in a little from the very edge of what's drawn, +/// where the lens is at its blurriest. +const AUTO_FOV_MARGIN: f64 = 0.97; +/// Resolution of the hidden area mask, per side. +const MASK_SIZE: usize = 512; +/// How far off straight ahead the automatic view may center itself. Zero: +/// a few extra degrees aren't worth the middle of the video not being where +/// you looked. +const MAX_CENTER_SHIFT_DEG: f64 = 0.0; +/// Candidate centers per direction on each side of straight ahead. +const CENTER_STEPS: i32 = 5; + +/// Everything we know about how one eye maps to the panel. +struct EyeModel { + eye: usize, + l: f64, + r: f64, + t: f64, + b: f64, + roll: f64, + /// Panel -> render target, sampled on a grid, per channel. + forward: Vec<[[f64; 2]; 3]>, + /// Render target spots the compositor never draws. + hidden: Vec, + /// Render target spots that land somewhere on the panel. + covered: Vec, +} + +impl EyeModel { + fn new(vr: &OpenVr, eye: usize) -> Result { + let proj = vr.projection(eye); + let roll = vr.eye_roll(eye); + log::info!( + "eye {eye} projection: {proj:?}, render target {:?}, roll {:.2}°", + vr.render_target_size(), + roll.to_degrees() + ); + if proj.left >= 0.0 || proj.right <= 0.0 || proj.top >= 0.0 || proj.bottom <= 0.0 { + bail!("SteamVR reported a weird projection: {proj:?}"); + } + + let n = DISTORTION_GRID; + let mut forward = vec![[[f64::NAN; 2]; 3]; n * n]; + for j in 0..n { + for i in 0..n { + let (u, v) = (i as f32 / (n - 1) as f32, j as f32 / (n - 1) as f32); + if let Some(d) = vr.distortion(eye, u, v) { + forward[j * n + i] = d.map(|c| [c[0] as f64, c[1] as f64]); + } + } + } + + let triangles = vr.hidden_area(eye); + let hidden = rasterize(&triangles); + if log::log_enabled!(log::Level::Debug) { + let rows: Vec = (0..32) + .map(|y| (0..64).map(|x| if hidden[(y * 16) * MASK_SIZE + x * 8] { '#' } else { '.' }).collect()) + .collect(); + log::debug!("hidden area of eye {eye}:\n{}", rows.join("\n")); + } + log::info!( + "hidden area: {} triangles, {:.1}% of the render target", + triangles.len(), + hidden.iter().filter(|h| **h).count() as f64 * 100.0 / hidden.len() as f64 + ); + + let covered = rasterize(&panel_coverage(&forward, n)); + + Ok(Self { + eye, + l: proj.left as f64, + r: proj.right as f64, + t: proj.top as f64, + b: proj.bottom as f64, + roll, + forward, + hidden, + covered, + }) + } + + /// Render target UV for a view direction given as tangents, with the + /// eye's roll undone so the video stays level with the head. + fn target(&self, tx: f64, ty: f64) -> [f64; 2] { + let (s, c) = (-self.roll).sin_cos(); + let (rx, ry) = (tx * c - ty * s, tx * s + ty * c); + [(rx - self.l) / (self.r - self.l), (ry - self.t) / (self.b - self.t)] + } + + /// Whether a render target spot is on the panel and actually drawn. + fn shown(&self, uv: [f64; 2]) -> bool { + if !(0.0..1.0).contains(&uv[0]) || !(0.0..1.0).contains(&uv[1]) { + return false; + } + let i = (uv[1] * MASK_SIZE as f64) as usize * MASK_SIZE + (uv[0] * MASK_SIZE as f64) as usize; + self.covered[i] && !self.hidden[i] + } + + /// Panel position (eye local) that shows this render target spot in channel `c`. + fn solve(&self, c: usize, target: [f64; 2], guess: [f64; 2]) -> Option<[f64; 2]> { + let n = DISTORTION_GRID; + invert(&self.forward, n, c, target, guess).or_else(|| { + let start = nearest(&self.forward, n, c, target)?; + invert(&self.forward, n, c, target, start) + }) + } + + /// Widest view at this aspect ratio that's entirely drawn, as the + /// horizontal half-angle tangent plus a center offset (also tangents). + /// The center may drift a few degrees off straight ahead when that buys + /// a noticeably wider view, since the lens isn't symmetric. + fn widest(&self, aspect: f64) -> (f64, f64, f64) { + let fits = |tan_h: f64, cx: f64, cy: f64| { + let tan_v = tan_h * aspect; + (0..=EDGE_SAMPLES).all(|k| { + let f = k as f64 / EDGE_SAMPLES as f64 * 2.0 - 1.0; + [ + (cx + f * tan_h, cy - tan_v), + (cx + f * tan_h, cy + tan_v), + (cx - tan_h, cy + f * tan_v), + (cx + tan_h, cy + f * tan_v), + ] + .iter() + .all(|&(x, y)| self.shown(self.target(x, y))) + }) + }; + let limit = (-self.l).max(self.r).max((-self.t).max(self.b) / aspect); + let search = |cx: f64, cy: f64| { + let (mut lo, mut hi) = (0.0, limit); + if !fits(1e-3, cx, cy) { + return 0.0; + } + for _ in 0..30 { + let mid = (lo + hi) / 2.0; + if fits(mid, cx, cy) { + lo = mid; + } else { + hi = mid; + } + } + lo + }; + + let centered = search(0.0, 0.0); + let max_shift = MAX_CENTER_SHIFT_DEG.to_radians().tan(); + let mut best = (centered, 0.0, 0.0); + for i in -CENTER_STEPS..=CENTER_STEPS { + for j in -CENTER_STEPS..=CENTER_STEPS { + let cx = max_shift * i as f64 / CENTER_STEPS as f64; + let cy = max_shift * j as f64 / CENTER_STEPS as f64; + let w = search(cx, cy); + if w > best.0 { + best = (w, cx, cy); + } + } + } + // Only move off center when it's worth it. + if best.0 < centered * 1.02 { + best = (centered, 0.0, 0.0); + } + best + } +} + +/// The panel's outline in render target space, as triangles. +fn panel_coverage(forward: &[[[f64; 2]; 3]], n: usize) -> Vec<[[f32; 2]; 3]> { + let mut triangles = Vec::new(); + let at = |i: usize, j: usize| forward[j * n + i][1]; + for j in 0..n - 1 { + for i in 0..n - 1 { + let q = [at(i, j), at(i + 1, j), at(i + 1, j + 1), at(i, j + 1)]; + if q.iter().any(|p| !p[0].is_finite() || !p[1].is_finite()) { + continue; + } + let q = q.map(|p| [p[0] as f32, p[1] as f32]); + triangles.push([q[0], q[1], q[2]]); + triangles.push([q[0], q[2], q[3]]); + } + } + triangles +} + +/// Turns the hidden area triangles into a lookup mask. +fn rasterize(triangles: &[[[f32; 2]; 3]]) -> Vec { + let mut mask = vec![false; MASK_SIZE * MASK_SIZE]; + let size = MASK_SIZE as f64; + for tri in triangles { + let p = tri.map(|v| [v[0] as f64 * size, v[1] as f64 * size]); + let min_x = p.iter().map(|v| v[0]).fold(f64::INFINITY, f64::min).floor().max(0.0) as usize; + let max_x = p.iter().map(|v| v[0]).fold(f64::NEG_INFINITY, f64::max).ceil().min(size) as usize; + let min_y = p.iter().map(|v| v[1]).fold(f64::INFINITY, f64::min).floor().max(0.0) as usize; + let max_y = p.iter().map(|v| v[1]).fold(f64::NEG_INFINITY, f64::max).ceil().min(size) as usize; + let edge = |a: [f64; 2], b: [f64; 2], c: [f64; 2]| (b[0] - a[0]) * (c[1] - a[1]) - (b[1] - a[1]) * (c[0] - a[0]); + let area = edge(p[0], p[1], p[2]); + if area.abs() < 1e-12 { + continue; + } + for y in min_y..max_y { + for x in min_x..max_x { + let c = [x as f64 + 0.5, y as f64 + 0.5]; + let w = [edge(p[1], p[2], c), edge(p[2], p[0], c), edge(p[0], p[1], c)]; + if w.iter().all(|w| w * area.signum() >= 0.0) { + mask[y * MASK_SIZE + x] = true; + } + } + } + } + mask +} + +/// Fits an upright flat source (SteamVR's headset view) into the output, +/// letterboxed if the shapes differ. +pub fn flat(out_w: u32, out_h: u32, src_w: u32, src_h: u32) -> Lut { + let mut lut = raw(out_w, out_h, src_w, src_h); + lut.rotated_eyes = false; + lut +} + +/// One eye as a flat 2D picture: the lens distortion SteamVR applied gets +/// undone, cropped to the output's aspect ratio around straight ahead. With +/// no field of view given, it picks the widest one that has no black edges. +pub fn undistorted(vr: &OpenVr, eye: usize, fov_deg: Option, out_w: u32, out_h: u32, center_x: f64, center_y: f64) -> Result { + let model = EyeModel::new(vr, eye)?; + let aspect = out_h as f64 / out_w as f64; + + let (widest, wide_cx, wide_cy) = model.widest(aspect); + if widest <= 0.0 { + bail!("couldn't find any clean part of the view, SteamVR's lens data looks off"); + } + log::info!( + "widest clean view at this aspect ratio: {:.1}° x {:.1}°, centered {:.1}° right and {:.1}° down", + 2.0 * widest.atan().to_degrees(), + 2.0 * (widest * aspect).atan().to_degrees(), + wide_cx.atan().to_degrees(), + wide_cy.atan().to_degrees() + ); + let (tan_h, cx, cy) = match fov_deg { + Some(fov) => { + let wanted = (fov.to_radians() / 2.0).tan(); + if wanted > widest { + log::warn!("a {fov}° view reaches past what the lens shows, the edges will be black"); + } + (wanted, 0.0, 0.0) + } + None => (widest * AUTO_FOV_MARGIN, wide_cx, wide_cy), + }; + let cx = cx + center_x.to_radians().tan(); + let cy = cy + center_y.to_radians().tan(); + let tan_v = tan_h * aspect; + log::info!( + "recording {:.1}° x {:.1}° of the eye's view", + 2.0 * tan_h.atan().to_degrees(), + 2.0 * tan_v.atan().to_degrees() + ); + + let mut lut = Lut::new(out_w, out_h); + let eye_offset = model.eye as f64 * 0.5; + lut.view = Some((tan_h, cx, cy)); + let mut guess = [[0.5f64, 0.5]; 3]; + // Widest color fringe, measured in output pixels. + let mut max_split = 0.0f64; + let mut max_minify = 0.0f64; + let mut prev_green: Option<[f64; 2]> = None; + + let cells: Vec<_> = lut.grid().collect(); + for (idx, x, y) in cells { + if x == 0.0 { + prev_green = None; + } + let tx = cx + (x / out_w as f64 * 2.0 - 1.0) * tan_h; + let ty = cy + (y / out_h as f64 * 2.0 - 1.0) * tan_v; + let target = model.target(tx, ty); + + let mut solved = [None; 3]; + for c in 0..3 { + if let Some(d) = model.solve(c, target, guess[c]) { + guess[c] = d; + solved[c] = Some(d); + lut.points[idx + c] = [(eye_offset + d[0] * 0.5) as f32, d[1] as f32]; + } + } + if let (Some(rd), Some(gd), Some(bd), Some(prev)) = (solved[0], solved[1], solved[2], prev_green) { + // How far the panel moves per output pixel, to express the split in output pixels. + let panel_per_out = dist(gd, prev) / STEP as f64; + if panel_per_out > 0.0 { + max_split = max_split.max(dist(rd, gd).max(dist(bd, gd)) / panel_per_out); + max_minify = max_minify.max(panel_per_out * EYE_PX); + } + } + prev_green = solved[1]; + } + + // A fringe well under a pixel in the video isn't worth tripling the + // texture reads for. + lut.per_channel = max_split > MIN_FRINGE_PX; + lut.supersample = max_minify > SUPERSAMPLE_ABOVE; + log::info!( + "distortion grid ready: color fringe up to {:.2} px (correcting: {}), up to {:.2} panel px per output px (supersampling: {})", + max_split, + lut.per_channel, + max_minify, + lut.supersample + ); + Ok(lut) +} + +fn dist(a: [f64; 2], b: [f64; 2]) -> f64 { + ((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt() +} + +/// Bilinear read of the sampled forward mapping. None outside the panel. +fn eval(forward: &[[[f64; 2]; 3]], n: usize, c: usize, d: [f64; 2]) -> Option<[f64; 2]> { + if !(0.0..=1.0).contains(&d[0]) || !(0.0..=1.0).contains(&d[1]) { + return None; + } + let gx = d[0] * (n - 1) as f64; + let gy = d[1] * (n - 1) as f64; + let (i, j) = ((gx.floor() as usize).min(n - 2), (gy.floor() as usize).min(n - 2)); + let (fx, fy) = (gx - i as f64, gy - j as f64); + let p = |ii: usize, jj: usize| forward[jj * n + ii][c]; + let (a, b, e, f) = (p(i, j), p(i + 1, j), p(i, j + 1), p(i + 1, j + 1)); + let out = [0, 1].map(|k| { + let top = a[k] + (b[k] - a[k]) * fx; + let bottom = e[k] + (f[k] - e[k]) * fx; + top + (bottom - top) * fy + }); + out.iter().all(|v| v.is_finite()).then_some(out) +} + +/// Newton's method on the forward mapping. +fn invert(forward: &[[[f64; 2]; 3]], n: usize, c: usize, target: [f64; 2], start: [f64; 2]) -> Option<[f64; 2]> { + const H: f64 = 1e-4; + let mut d = start; + for _ in 0..24 { + let f = eval(forward, n, c, d)?; + let err = [f[0] - target[0], f[1] - target[1]]; + if err[0].abs() < 1e-7 && err[1].abs() < 1e-7 { + return Some(d); + } + let dx0 = eval(forward, n, c, [(d[0] - H).max(0.0), d[1]])?; + let dx1 = eval(forward, n, c, [(d[0] + H).min(1.0), d[1]])?; + let dy0 = eval(forward, n, c, [d[0], (d[1] - H).max(0.0)])?; + let dy1 = eval(forward, n, c, [d[0], (d[1] + H).min(1.0)])?; + let hx = (d[0] + H).min(1.0) - (d[0] - H).max(0.0); + let hy = (d[1] + H).min(1.0) - (d[1] - H).max(0.0); + let j = [ + [(dx1[0] - dx0[0]) / hx, (dy1[0] - dy0[0]) / hy], + [(dx1[1] - dx0[1]) / hx, (dy1[1] - dy0[1]) / hy], + ]; + let det = j[0][0] * j[1][1] - j[0][1] * j[1][0]; + if det.abs() < 1e-12 { + return None; + } + let step = [ + (j[1][1] * err[0] - j[0][1] * err[1]) / det, + (-j[1][0] * err[0] + j[0][0] * err[1]) / det, + ]; + d = [(d[0] - step[0]).clamp(0.0, 1.0), (d[1] - step[1]).clamp(0.0, 1.0)]; + } + let f = eval(forward, n, c, d)?; + (dist(f, target) < 1e-5).then_some(d) +} + +fn nearest(forward: &[[[f64; 2]; 3]], n: usize, c: usize, target: [f64; 2]) -> Option<[f64; 2]> { + let (best, _) = forward + .iter() + .enumerate() + .filter(|(_, p)| p[c][0].is_finite()) + .map(|(k, p)| (k, dist(p[c], target))) + .min_by(|a, b| a.1.total_cmp(&b.1))?; + Some([(best % n) as f64 / (n - 1) as f64, (best / n) as f64 / (n - 1) as f64]) +} diff --git a/media/capture/src/main.rs b/media/capture/src/main.rs new file mode 100644 index 0000000..f4ce3c6 --- /dev/null +++ b/media/capture/src/main.rs @@ -0,0 +1,85 @@ +//! Framely's panel capture worker. stdout is Annex-B H.264, stderr is logging. +mod clock; +mod encoder; +mod gpu; +mod kms; +mod lut; + +use anyhow::{Context, Result}; +use clap::Parser; +use framely_capture::openvr::OpenVr; +use std::{io::{self, Write}, os::fd::AsRawFd, path::Path, time::{Duration, Instant}}; + +#[derive(Parser)] +struct Args { + #[arg(long, default_value_t = 1920)] width: u32, + #[arg(long, default_value_t = 1080)] height: u32, + #[arg(long, default_value_t = 30)] fps: u32, + #[arg(long, default_value_t = 8)] bitrate: u32, + #[arg(long, default_value_t = 0)] eye: usize, + #[arg(long)] raw: bool, + #[arg(long)] fov: Option, + #[arg(long, default_value_t = 0.)] center_x: f64, + #[arg(long, default_value_t = 0.)] center_y: f64, +} +fn run(a: Args) -> Result<()> { + anyhow::ensure!(a.width >= 160 && a.width <= 4096 && a.width % 2 == 0 + && a.height >= 160 && a.height <= 4096 && a.height % 2 == 0 + && a.width as u64 * a.height as u64 <= 8_847_360 + && (1..=120).contains(&a.fps) && (1..=80).contains(&a.bitrate) && a.eye <= 1, + "Invalid output dimensions, rate or eye"); + anyhow::ensure!(a.fov.is_none_or(|v| v.is_finite() && (10.0..=150.0).contains(&v)) + && [a.center_x, a.center_y].into_iter().all(|v| v.is_finite() && (-60.0..=60.0).contains(&v)), "Invalid field of view"); + let kms = kms::Kms::open(Path::new("/dev/dri/card0"))?; + let height = if a.raw { ((a.width as f64 * kms.mode_size.1 as f64 / kms.mode_size.0 as f64).round() as u32 + 1) & !1 } else { a.height }; + anyhow::ensure!(height>=160&&height<=4096&&u64::from(a.width)*u64::from(height)<=8_847_360,"Raw output exceeds supported dimensions"); + let map = if a.raw { lut::raw(a.width, height, kms.mode_size.0, kms.mode_size.1) } + else { lut::undistorted(&OpenVr::connect()?, a.eye, a.fov, a.width, height, a.center_x, a.center_y)? }; + if let Some((tan_h, cx, cy)) = map.view { + eprintln!("{}", serde_json::json!({"event":"view","tanHalfHorizontal":tan_h,"centerTanX":cx,"centerTanY":cy,"aspect":a.width as f64/height as f64})); + } + let mut enc = encoder::Encoder::open(&encoder::find_device()?, encoder::Config { + codec: encoder::Codec::H264, width: a.width, height, fps: a.fps, + bitrate: a.bitrate * 1_000_000, qp: None, + }, 3)?; + let filter = if map.supersample { gpu::Filter::Supersample } else { gpu::Filter::Sharp }; + let mut gpu = gpu::Gpu::new(enc.layout(), 3, &map, filter, gpu::Priority::Low, None)?; + let mut output = io::BufWriter::new(io::stdout().lock()); + let mut origin = None; + let mut next = Duration::ZERO; + let interval = Duration::from_secs_f64(1. / a.fps as f64); + let tolerance = Duration::from_secs_f64(0.5 / kms.refresh_hz); + let mut frames = 0u64; + let mut last_status = Instant::now(); + loop { + let time = kms.wait_vblank().context("Headset display is unavailable")?; + let first = *origin.get_or_insert(time); + let elapsed = time.saturating_sub(first); + if elapsed + tolerance < next { continue; } + // Keep a stable cadence when output FPS is not a divisor of display refresh. + while next <= elapsed + tolerance { next += interval; } + let Some(fb) = kms.current_fb()? else { continue; }; + let buf = kms.export(fb)?; + kms::wait_idle(&buf.fd, Duration::from_millis(4)); + let id = kms::buffer_id(&buf.fd)?; + if !gpu.has_source(fb, id) { gpu.add_source(buf, id)?; } + if let Some(slot) = enc.free_slot()? { + gpu.convert(fb, id, slot)?; + enc.queue_frame(slot, gpu.targets()[slot].fd.as_raw_fd(), elapsed.as_micros() as u64)?; + frames += 1; + } + let mut error = None; + enc.poll_packets(|packet| { + if let Err(e) = output.write_all(packet.data).and_then(|_| output.flush()) { error = Some(e); } + })?; + if let Some(e) = error { return Err(e.into()); } + if frames == 1 || last_status.elapsed() > Duration::from_secs(5) { + eprintln!("{}", serde_json::json!({"event":"capture","width":a.width,"height":height,"frames":frames})); + last_status = Instant::now(); + } + } +} +fn main() { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + if let Err(e) = run(Args::parse()) { eprintln!("{e:#}"); std::process::exit(1); } +} diff --git a/media/capture/src/openvr.rs b/media/capture/src/openvr.rs new file mode 100644 index 0000000..0ee72ea --- /dev/null +++ b/media/capture/src/openvr.rs @@ -0,0 +1,264 @@ +//! Just enough OpenVR to ask SteamVR what its lens distortion looks like. +//! We connect as a background app, grab the numbers and disconnect again, so +//! nothing stays attached to the compositor while recording. + +use std::ffi::{c_char, c_void, CStr, CString}; + +use anyhow::{bail, Result}; + +const DEFAULT_LIB: &str = "/opt/steamvr/bin/linuxarm64/libopenvr_api.so"; + +/// How we present ourselves to SteamVR. +#[derive(Clone, Copy)] +pub enum AppType { + /// Doesn't show anything and doesn't start SteamVR. + Background = 3, + /// Draws overlays (the dashboard tab). Careful: connecting like this + /// starts SteamVR's server if it isn't running, and a server started + /// from outside SteamVR's own launcher leaves the headset without a + /// working compositor. Check `server_running` first. + Overlay = 2, +} + +/// Interface versions we know the function table layout of. The first four +/// slots haven't moved in any of them; the rest only get used on the newest. +const SYSTEM_VERSIONS: &[&str] = &[ + "IVRSystem_026", + "IVRSystem_023", + "IVRSystem_022", + "IVRSystem_021", + "IVRSystem_020", + "IVRSystem_019", +]; +const FULL_LAYOUT: &str = "IVRSystem_026"; + +// Slots in VR_IVRSystem_FnTable (openvr_capi.h). +const SLOT_RENDER_TARGET_SIZE: usize = 0; +const SLOT_PROJECTION_RAW: usize = 2; +const SLOT_COMPUTE_DISTORTION: usize = 3; +const SLOT_EYE_TO_HEAD: usize = 5; +const SLOT_HIDDEN_AREA_MESH: usize = 34; + +const HIDDEN_AREA_STANDARD: i32 = 0; + +#[repr(C)] +#[derive(Default, Clone, Copy)] +struct DistortionCoordinates { + red: [f32; 2], + green: [f32; 2], + blue: [f32; 2], +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct HmdMatrix34 { + m: [[f32; 4]; 3], +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct HiddenAreaMesh { + vertices: *const [f32; 2], + triangles: u32, +} + +type InitFn = unsafe extern "C" fn(*mut i32, i32, *const c_char) -> isize; +type ShutdownFn = unsafe extern "C" fn(); +type GetInterfaceFn = unsafe extern "C" fn(*const c_char, *mut i32) -> *mut c_void; +type ErrorNameFn = unsafe extern "C" fn(i32) -> *const c_char; + +/// Tangents of the half angles an eye's render target covers. Top is +/// negative, since OpenVR's y points down in texture space. +#[derive(Clone, Copy, Debug)] +pub struct Projection { + pub left: f32, + pub right: f32, + pub top: f32, + pub bottom: f32, +} + +pub struct OpenVr { + lib: *mut c_void, + table: *const *const c_void, + full_layout: bool, + shutdown: ShutdownFn, + get_interface: GetInterfaceFn, +} + +impl OpenVr { + pub fn connect() -> Result { + Self::connect_as(AppType::Background) + } + + /// Whether SteamVR's server is up. Connects as a background app, which + /// SteamVR never starts a server for, so this can't launch one by accident. + pub fn server_running() -> bool { + Self::connect_as(AppType::Background).is_ok() + } + + pub fn connect_as(app: AppType) -> Result { + let path = std::env::var("FRAMECORDER_OPENVR_LIB").unwrap_or_else(|_| DEFAULT_LIB.into()); + let cpath = CString::new(path.clone())?; + let lib = unsafe { libc::dlopen(cpath.as_ptr(), libc::RTLD_NOW | libc::RTLD_LOCAL) }; + if lib.is_null() { + bail!("couldn't load {path}: {}", dl_error()); + } + + let init: InitFn = unsafe { sym(lib, c"VR_InitInternal2")? }; + let shutdown: ShutdownFn = unsafe { sym(lib, c"VR_ShutdownInternal")? }; + let get_interface: GetInterfaceFn = unsafe { sym(lib, c"VR_GetGenericInterface")? }; + let error_name: Option = + unsafe { sym(lib, c"VR_GetVRInitErrorAsEnglishDescription").ok() }; + + let mut err = 0i32; + unsafe { init(&mut err, app as i32, std::ptr::null()) }; + if err != 0 { + let msg = error_name + .map(|f| { + unsafe { CStr::from_ptr(f(err)) } + .to_string_lossy() + .into_owned() + }) + .unwrap_or_default(); + unsafe { libc::dlclose(lib) }; + bail!("SteamVR isn't available ({err}: {msg})"); + } + + let found = SYSTEM_VERSIONS.iter().find_map(|v| { + let name = CString::new(format!("FnTable:{v}")).ok()?; + let mut err = 0i32; + let table = unsafe { get_interface(name.as_ptr(), &mut err) }; + (err == 0 && !table.is_null()).then_some((*v, table as *const *const c_void)) + }); + let Some((version, table)) = found else { + unsafe { + shutdown(); + libc::dlclose(lib); + } + bail!("this SteamVR doesn't offer a known IVRSystem version"); + }; + log::debug!("using {version}"); + if version != FULL_LAYOUT { + log::warn!( + "SteamVR only offers {version}, so the hidden area and eye roll can't be read" + ); + } + + Ok(Self { + lib, + table, + full_layout: version == FULL_LAYOUT, + shutdown, + get_interface, + }) + } + + /// Function table of another OpenVR interface, e.g. `IVROverlay_028`. + pub fn interface(&self, version: &str) -> Option<*const *const c_void> { + let name = CString::new(format!("FnTable:{version}")).ok()?; + let mut err = 0i32; + let table = unsafe { (self.get_interface)(name.as_ptr(), &mut err) }; + (err == 0 && !table.is_null()).then_some(table as *const *const c_void) + } + + unsafe fn slot(&self, index: usize) -> T { + std::mem::transmute_copy(&*self.table.add(index)) + } + + pub fn render_target_size(&self) -> (u32, u32) { + let (mut w, mut h) = (0, 0); + unsafe { + let f: unsafe extern "C" fn(*mut u32, *mut u32) = self.slot(SLOT_RENDER_TARGET_SIZE); + f(&mut w, &mut h); + } + (w, h) + } + + pub fn projection(&self, eye: usize) -> Projection { + let (mut l, mut r, mut t, mut b) = (0.0, 0.0, 0.0, 0.0); + unsafe { + let f: unsafe extern "C" fn(i32, *mut f32, *mut f32, *mut f32, *mut f32) = + self.slot(SLOT_PROJECTION_RAW); + f(eye as i32, &mut l, &mut r, &mut t, &mut b); + } + Projection { + left: l, + right: r, + top: t, + bottom: b, + } + } + + /// For a point on the panel (eye local, 0..1), where in the undistorted + /// render target each color channel comes from. + pub fn distortion(&self, eye: usize, u: f32, v: f32) -> Option<[[f32; 2]; 3]> { + let mut out = DistortionCoordinates::default(); + let ok = unsafe { + let f: unsafe extern "C" fn(i32, f32, f32, *mut DistortionCoordinates) -> bool = + self.slot(SLOT_COMPUTE_DISTORTION); + f(eye as i32, u, v, &mut out) + }; + ok.then_some([out.red, out.green, out.blue]) + } + + /// How far the eye's view is rolled relative to the head, in radians. + /// Canted displays show up here. + pub fn eye_roll(&self, eye: usize) -> f64 { + if !self.full_layout { + return 0.0; + } + let t = unsafe { + let f: unsafe extern "C" fn(i32) -> HmdMatrix34 = self.slot(SLOT_EYE_TO_HEAD); + f(eye as i32) + }; + log::debug!("eye {eye} to head: {:?}", t.m); + (t.m[1][0] as f64).atan2(t.m[0][0] as f64) + } + + /// Triangles, in render target UV, covering what the compositor never + /// draws for this eye. + pub fn hidden_area(&self, eye: usize) -> Vec<[[f32; 2]; 3]> { + if !self.full_layout { + return Vec::new(); + } + unsafe { + let f: unsafe extern "C" fn(i32, i32) -> HiddenAreaMesh = + self.slot(SLOT_HIDDEN_AREA_MESH); + let mesh = f(eye as i32, HIDDEN_AREA_STANDARD); + if mesh.vertices.is_null() || mesh.triangles == 0 || mesh.triangles > 1 << 20 { + return Vec::new(); + } + std::slice::from_raw_parts(mesh.vertices, mesh.triangles as usize * 3) + .as_chunks::<3>() + .0 + .to_vec() + } + } +} + +impl Drop for OpenVr { + fn drop(&mut self) { + unsafe { + (self.shutdown)(); + libc::dlclose(self.lib); + } + } +} + +unsafe fn sym(lib: *mut c_void, name: &CStr) -> Result { + let ptr = libc::dlsym(lib, name.as_ptr()); + if ptr.is_null() { + bail!("{} is missing from libopenvr_api", name.to_string_lossy()); + } + Ok(std::mem::transmute_copy(&ptr)) +} + +fn dl_error() -> String { + let err = unsafe { libc::dlerror() }; + if err.is_null() { + return "unknown error".into(); + } + unsafe { CStr::from_ptr(err) } + .to_string_lossy() + .into_owned() +} diff --git a/media/dependencies.json b/media/dependencies.json new file mode 100644 index 0000000..9a31bab --- /dev/null +++ b/media/dependencies.json @@ -0,0 +1,12 @@ +{ + "mediamtx": { + "version": "1.21.2", + "url": "https://github.com/bluenviron/mediamtx/releases/download/v1.21.2/mediamtx_v1.21.2_linux_arm64.tar.gz", + "sha256": "180a235374a548d9133cdd5e20f5dbd8b01317be1c86e0d7e66df585dc42ed6f" + }, + "uxplay": { + "version": "1.73.7", + "url": "https://github.com/FDH2/UxPlay/archive/refs/tags/v1.73.7.tar.gz", + "sha256": "65feb8732de666a7161a9e562aa603a7f5fe0ebb890c2a30c5e1a9c85c76309f" + } +} diff --git a/media/native/airplay_sink.cpp b/media/native/airplay_sink.cpp new file mode 100644 index 0000000..1ecac3b --- /dev/null +++ b/media/native/airplay_sink.cpp @@ -0,0 +1,30 @@ +#include "frame_sink.h" +#include +#include +struct FramelySink {GstBin parent;FrameSink* frame;GstElement* gate;GstElement* gain;guint timer;bool audio;}; +struct FramelySinkClass {GstBinClass parent;}; +typedef FramelySink FramelyAudio;typedef FramelySinkClass FramelyAudioClass; +G_DEFINE_TYPE(FramelySink,framely_sink,GST_TYPE_BIN) +G_DEFINE_TYPE(FramelyAudio,framely_audio,GST_TYPE_BIN) +static gboolean tick(gpointer p){auto* s=(FramelySink*)p;bool allowed=s->frame->accepted();g_object_set(s->gate,"drop",!allowed,nullptr); + std::ifstream volume(s->frame->directory+"/"+s->frame->id+".volume");double gain=1.;if(volume>>gain)g_object_set(s->gain,"volume",std::clamp(gain,0.,1.),nullptr);return G_SOURCE_CONTINUE;} +static void audio_frame(GstElement*,GstBuffer*,gpointer p){auto* s=(FramelySink*)p;tick(s);if(!s->frame->announced.exchange(true))s->frame->event("audio");} +static void setup(FramelySink* s,bool audio){ + s->audio=audio;GError* error=nullptr; + auto* bin=gst_parse_bin_from_description(audio?"identity name=tap signal-handoffs=true ! valve name=gate drop=true drop-mode=transform-to-gap ! volume name=gain ! pulsesink async=false":"videoflip video-direction=auto ! videoconvert ! videoscale ! video/x-raw,pixel-aspect-ratio=1/1,format=BGRA ! appsink name=frames emit-signals=true sync=true max-buffers=1 drop=true",true,&error); + if(!bin){g_warning("Framely sink: %s",error?error->message:"pipeline error");g_clear_error(&error);return;} + gst_bin_add(GST_BIN(s),bin);auto* pad=gst_element_get_static_pad(bin,"sink");gst_element_add_pad(GST_ELEMENT(s),gst_ghost_pad_new("sink",pad));gst_object_unref(pad); + if(audio){s->gate=gst_bin_get_by_name(GST_BIN(bin),"gate");s->gain=gst_bin_get_by_name(GST_BIN(bin),"gain");auto* tap=gst_bin_get_by_name(GST_BIN(bin),"tap");g_signal_connect(tap,"handoff",G_CALLBACK(audio_frame),s);gst_object_unref(tap);} + else {auto* sink=gst_bin_get_by_name(GST_BIN(bin),"frames");g_signal_connect(sink,"new-sample",G_CALLBACK(frame_sample),s->frame);g_signal_connect(sink,"new-preroll",G_CALLBACK(frame_preroll),s->frame);gst_object_unref(sink);} +} +static void finalize(GObject* p){auto* s=(FramelySink*)p;if(s->timer)g_source_remove(s->timer);if(s->gate)gst_object_unref(s->gate);if(s->gain)gst_object_unref(s->gain);delete s->frame;G_OBJECT_CLASS(framely_sink_parent_class)->finalize(p);} +static void framely_sink_class_init(FramelySinkClass* c){G_OBJECT_CLASS(c)->finalize=finalize;gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(c),"Framely video consent sink","Sink/Video","Gated AirPlay receiver","Framely");} +static void initialize(FramelySink* s,bool audio){const char* d=getenv("FRAMELY_CAST_DIR"),*i=getenv("FRAMELY_CAST_ID");s->frame=new FrameSink(d?d:"/tmp",i?i:"airplay","AirPlay");GST_OBJECT_FLAG_SET(s,GST_ELEMENT_FLAG_SINK);setup(s,audio);} +static void framely_sink_init(FramelySink* s){initialize(s,false);} +static void framely_audio_class_init(FramelyAudioClass* c){G_OBJECT_CLASS(c)->finalize=finalize;gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(c),"Framely audio consent sink","Sink/Audio","Gated AirPlay receiver","Framely");} +static void framely_audio_init(FramelyAudio* s){initialize(s,true);} +static gboolean plugin_init(GstPlugin* p){return gst_element_register(p,"framelyvideosink",GST_RANK_NONE,framely_sink_get_type()) && gst_element_register(p,"framelyaudiosink",GST_RANK_NONE,framely_audio_get_type());} +#ifndef PACKAGE +#define PACKAGE "framely" +#endif +GST_PLUGIN_DEFINE(GST_VERSION_MAJOR,GST_VERSION_MINOR,framely,"Framely casting",plugin_init,"1.0","AGPL","Framely","https://github.com/SteamFramelyHomebrew/framely") diff --git a/media/native/frame_sink.h b/media/native/frame_sink.h new file mode 100644 index 0000000..c8a355d --- /dev/null +++ b/media/native/frame_sink.h @@ -0,0 +1,47 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// A frame file is written under an advisory lock. Readers never see a partial frame. +struct FrameSink { + std::string directory,id,protocol; std::atomic announced{false}; std::atomic width{0},height{0}; + FrameSink(std::string d,std::string i,std::string p):directory(d),id(i),protocol(p){} + void event(const std::string& type,unsigned w=0,unsigned h=0){ + int fd=socket(AF_UNIX,SOCK_DGRAM|SOCK_CLOEXEC,0); if(fd<0)return; + sockaddr_un a{};a.sun_family=AF_UNIX; auto path=directory+"/events.sock";if(path.size()>=sizeof(a.sun_path)){close(fd);return;}strcpy(a.sun_path,path.c_str()); + auto msg="{\"event\":\""+type+"\",\"id\":\""+id+"\",\"protocol\":\""+protocol+"\",\"width\":"+std::to_string(w)+",\"height\":"+std::to_string(h)+"}"; + sendto(fd,msg.data(),msg.size(),MSG_DONTWAIT,(sockaddr*)&a,sizeof(a));close(fd); + } + bool accepted() const {return access((directory+"/"+id+".accept").c_str(),F_OK)==0;} + GstFlowReturn sample(GstAppSink* sink,bool preroll=false){ + auto* sample=preroll?gst_app_sink_pull_preroll(sink):gst_app_sink_pull_sample(sink);if(!sample)return GST_FLOW_EOS; + GstVideoInfo info{};GstVideoFrame frame{}; + if(gst_video_info_from_caps(&info,gst_sample_get_caps(sample))&&GST_VIDEO_INFO_FORMAT(&info)==GST_VIDEO_FORMAT_BGRA&&gst_video_frame_map(&frame,&info,gst_sample_get_buffer(sample),GST_MAP_READ)){ + unsigned w=GST_VIDEO_INFO_WIDTH(&info),h=GST_VIDEO_INFO_HEIGHT(&info); + if(w&&h&&w<=4096&&h<=4096&&uint64_t(w)*h<=16777216){ + bool resized=width.exchange(w)!=w;resized=(height.exchange(h)!=h)||resized;bool first=!announced.exchange(true);if(resized||first)event("video",w,h); + if(accepted()){ + std::string path=directory+"/"+id+".frame";int fd=open(path.c_str(),O_CREAT|O_WRONLY|O_CLOEXEC|O_NOFOLLOW,0600); + if(fd>=0){flock(fd,LOCK_EX);uint32_t header[]={0x46434153,w,h,w*4};bool ok=pwrite(fd,header,sizeof(header),0)==sizeof(header); + for(unsigned y=0;ok&&y(data)->sample(sink);} +inline GstFlowReturn frame_preroll(GstAppSink* sink,gpointer data){return static_cast(data)->sample(sink,true);} diff --git a/media/native/receiver.cpp b/media/native/receiver.cpp new file mode 100644 index 0000000..40f3d08 --- /dev/null +++ b/media/native/receiver.cpp @@ -0,0 +1,277 @@ +#include "frame_sink.h" +#include +#include "json.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; +static std::string directory, current, own_udn, xml_directory; +static GMainLoop* loop; +static std::mutex output_mutex; +static bool receive_enabled = false; +static constexpr const char* protocols = "http-get:*:video/mp4:*,http-get:*:audio/mpeg:*,http-get:*:audio/mp4:*,http-get:*:audio/flac:*,http-get:*:audio/wav:*,http-get:*:application/vnd.apple.mpegurl:*,http-get:*:application/x-mpegURL:*,http-get:*:video/mpeg:*"; +static void output(json j) { std::lock_guard lock(output_mutex); std::cout << j.dump() << std::endl; } +struct Media { + FrameSink sink; + GstElement* pipeline = nullptr; + std::string uri, metadata; + bool accepted = false, paused = false, muted = false; + double volume = 1.; + guint watch = 0; + Media(std::string id, std::string u, std::string m): sink(directory,id,"DLNA"), uri(u), metadata(m) {} + ~Media() { + if (watch) g_source_remove(watch); + if (pipeline) { gst_element_set_state(pipeline,GST_STATE_NULL); gst_object_unref(pipeline); } + } +}; +static std::map> sessions; +static std::map devices; +static std::vector transports, renderers; +static Media* active() { auto i = sessions.find(current); return i == sessions.end() ? nullptr : i->second.get(); } +static std::string state() { auto* m = active(); return !m ? "NO_MEDIA_PRESENT" : !m->accepted ? "STOPPED" : m->paused ? "PAUSED_PLAYBACK" : "PLAYING"; } +static gint64 position(Media* m, bool duration) { + gint64 value = 0; + if (m && m->pipeline) { + if (duration) gst_element_query_duration(m->pipeline,GST_FORMAT_TIME,&value); + else gst_element_query_position(m->pipeline,GST_FORMAT_TIME,&value); + } + return std::max(gint64(0),value); +} +static std::string time_string(gint64 ns) { + auto seconds = ns / GST_SECOND; + char text[48]; snprintf(text,sizeof(text),"%02lld:%02lld:%02lld",(long long)(seconds/3600),(long long)(seconds/60%60),(long long)(seconds%60)); + return text; +} +static std::string last_change(bool rendering) { + std::string values; + if (rendering) { + auto* m = active(); + values = "volume*100) : 100) + "\"/>muted ? "1" : "0") + "\"/>"; + } else values = ""; + return "" + values + ""; +} +static void notify() { + for (auto* service : transports) { auto value = last_change(false); gupnp_service_notify(service,"LastChange",G_TYPE_STRING,value.c_str(),nullptr); } + for (auto* service : renderers) { auto value = last_change(true); gupnp_service_notify(service,"LastChange",G_TYPE_STRING,value.c_str(),nullptr); } +} +static void erase_media(const std::string& id) { + sessions.erase(id); unlink((directory+"/"+id+".accept").c_str()); + if (current == id) current.clear(); + notify(); +} +static gboolean erase_later(gpointer data) { std::unique_ptr id((std::string*)data); erase_media(*id); return G_SOURCE_REMOVE; } +static gboolean bus(GstBus*, GstMessage* message, gpointer data) { + auto* media = (Media*)data; + if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR || GST_MESSAGE_TYPE(message) == GST_MESSAGE_EOS) { + if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) { + GError* error=nullptr; gchar* debug=nullptr; gst_message_parse_error(message,&error,&debug); + output({{"event","error"},{"id",media->sink.id},{"error",error ? error->message : "Media playback failed"}}); + g_clear_error(&error); g_free(debug); + } else output({{"event","ended"},{"id",media->sink.id}}); + // Destroy on the main loop after this callback has returned. NULL state joins + // streaming threads before their FrameSink and Media pointers are released. + media->watch=0; + g_idle_add(erase_later,new std::string(media->sink.id)); + return G_SOURCE_REMOVE; + } + return G_SOURCE_CONTINUE; +} +static void streams_changed(GstElement* p, gpointer data) { + auto* m=(Media*)data; gint video=0,audio=0; g_object_get(p,"n-video",&video,"n-audio",&audio,nullptr); + if (audio>0 && video==0 && !m->sink.announced.exchange(true)) m->sink.event("audio"); +} +static void play(Media& media) { + if (!media.pipeline) { + media.pipeline=gst_element_factory_make("playbin",nullptr); + if (!media.pipeline) throw std::runtime_error("GStreamer playbin unavailable"); + gst_object_ref_sink(media.pipeline); + g_signal_connect(media.pipeline,"audio-changed",G_CALLBACK(streams_changed),&media); + g_signal_connect(media.pipeline,"video-changed",G_CALLBACK(streams_changed),&media); + GError* error=nullptr; + auto* video=gst_parse_bin_from_description("videoflip video-direction=auto ! videoconvert ! videoscale ! video/x-raw,pixel-aspect-ratio=1/1,format=BGRA ! appsink name=frames emit-signals=true max-buffers=1 drop=true sync=true",true,&error); + if (!video) { std::string reason=error ? error->message : "GStreamer video sink unavailable"; g_clear_error(&error); throw std::runtime_error(reason); } + gst_object_ref_sink(video); + auto* sink=gst_bin_get_by_name(GST_BIN(video),"frames"); + g_signal_connect(sink,"new-sample",G_CALLBACK(frame_sample),&media.sink); + g_signal_connect(sink,"new-preroll",G_CALLBACK(frame_preroll),&media.sink); gst_object_unref(sink); + auto* audio=gst_element_factory_make("pulsesink",nullptr); + if (!audio) { gst_object_unref(video); throw std::runtime_error("GStreamer PulseAudio sink unavailable"); } + gst_object_ref_sink(audio); + g_object_set(media.pipeline,"uri",media.uri.c_str(),"video-sink",video,"audio-sink",audio,"volume",media.volume,"mute",media.muted,nullptr); + gst_object_unref(audio); gst_object_unref(video); + auto* b=gst_element_get_bus(media.pipeline); media.watch=gst_bus_add_watch(b,bus,&media); gst_object_unref(b); + } + if (gst_element_set_state(media.pipeline,media.paused ? GST_STATE_PAUSED : GST_STATE_PLAYING)==GST_STATE_CHANGE_FAILURE) + throw std::runtime_error("Media could not start playback"); +} +static bool seekable(Media& m) { + if(!m.pipeline)return false; + auto* query=gst_query_new_seeking(GST_FORMAT_TIME);gboolean allowed=false; + if(gst_element_query(m.pipeline,query))gst_query_parse_seeking(query,nullptr,&allowed,nullptr,nullptr); + gst_query_unref(query);return allowed; +} +static void seek(Media& m, double seconds) { + if (!seekable(m) || !std::isfinite(seconds) || seconds<0 || seconds>86400*365 || + !gst_element_seek_simple(m.pipeline,GST_FORMAT_TIME,(GstSeekFlags)(GST_SEEK_FLAG_FLUSH|GST_SEEK_FLAG_ACCURATE),gint64(seconds*GST_SECOND))) + throw std::runtime_error("This media cannot seek to the requested position"); +} +static void set_volume(Media& m, double volume) { + if (!std::isfinite(volume) || volume<0 || volume>1) throw std::runtime_error("Invalid volume"); + m.volume=volume; if (m.pipeline) g_object_set(m.pipeline,"volume",volume,nullptr); notify(); +} +static void invoke(GUPnPServiceProxy* proxy,GUPnPServiceProxyAction* action) { + GError* error=nullptr; + if (!gupnp_service_proxy_call_action(proxy,action,nullptr,&error)) { + std::string message=error ? error->message : "UPnP action failed"; g_clear_error(&error); gupnp_service_proxy_action_unref(action); throw std::runtime_error(message); + } + gupnp_service_proxy_action_unref(action); +} +static gboolean input(GIOChannel* channel,GIOCondition condition,gpointer) { + if (condition&G_IO_HUP) { g_main_loop_quit(loop); return G_SOURCE_REMOVE; } + gchar* line=nullptr; gsize length=0; + if (g_io_channel_read_line(channel,&line,&length,nullptr,nullptr)!=G_IO_STATUS_NORMAL) return G_SOURCE_CONTINUE; + std::unique_ptr owner(line,g_free); json request; + try { + request=json::parse(line); std::string kind=request.value("kind",""), id=request.value("id",""); json result=true; + if (kind=="devices") { + result=json::array(); + for (auto& [id,p]:devices) { + auto* info=GUPNP_DEVICE_INFO(g_object_get_data(G_OBJECT(p),"device")); auto* name=gupnp_device_info_get_friendly_name(info); + auto* context=gupnp_service_info_get_context(GUPNP_SERVICE_INFO(p)); + result.push_back({{"id",id},{"name",name ? name : id},{"localAddress",gssdp_client_get_host_ip(GSSDP_CLIENT(context))}}); g_free(name); + } + } else if (kind=="send" || kind=="send.stop") { + auto it=devices.find(id); if (it==devices.end()) throw std::runtime_error("Device is no longer available"); + if (kind=="send") { + invoke(it->second,gupnp_service_proxy_action_new("SetAVTransportURI","InstanceID",G_TYPE_UINT,0,"CurrentURI",G_TYPE_STRING,request.at("uri").get().c_str(),"CurrentURIMetaData",G_TYPE_STRING,"",nullptr)); + invoke(it->second,gupnp_service_proxy_action_new("Play","InstanceID",G_TYPE_UINT,0,"Speed",G_TYPE_STRING,"1",nullptr)); + } else invoke(it->second,gupnp_service_proxy_action_new("Stop","InstanceID",G_TYPE_UINT,0,nullptr)); + } else { + auto it=sessions.find(id); if (it==sessions.end()) throw std::runtime_error("Media request expired"); auto& m=*it->second; + if (kind=="accept") { + if (m.accepted) throw std::runtime_error("Media already accepted"); + std::ofstream gate(directory+"/"+id+".accept"); if (!gate) throw std::runtime_error("Cannot accept media"); gate.close(); + m.accepted=true; play(m); notify(); + } else if (kind=="stop" || kind=="reject") { erase_media(id); output({{"event","ended"},{"id",id}}); } + else { + if (!m.accepted) throw std::runtime_error("Media has not been accepted"); + if (kind=="pause") { m.paused=request.value("paused",true); gst_element_set_state(m.pipeline,m.paused ? GST_STATE_PAUSED : GST_STATE_PLAYING); notify(); } + else if (kind=="volume") set_volume(m,request.value("volume",1.)); + else if (kind=="seek") seek(m,request.value("seconds",0.)); + else if (kind=="status") result={{"position",position(&m,false)/double(GST_SECOND)},{"duration",position(&m,true)/double(GST_SECOND)},{"paused",m.paused},{"volume",m.volume},{"seekable",seekable(m)}}; + else throw std::runtime_error("Unknown media action"); + } + } + output({{"request",request.value("request",0)},{"result",result}}); + } catch (const std::exception& e) { output({{"request",request.value("request",0)},{"error",e.what()}}); } + return G_SOURCE_CONTINUE; +} +static void available(GUPnPControlPoint*,GUPnPDeviceProxy* device,gpointer) { + auto* service=gupnp_device_info_get_service(GUPNP_DEVICE_INFO(device),"urn:schemas-upnp-org:service:AVTransport:1"); if (!service) return; + std::string id=gupnp_device_info_get_udn(GUPNP_DEVICE_INFO(device)); + if (id==own_udn) { g_object_unref(service); return; } + if (devices.count(id)) g_object_unref(devices[id]); + g_object_set_data_full(G_OBJECT(service),"device",g_object_ref(device),g_object_unref); devices[id]=GUPNP_SERVICE_PROXY(service); +} +static void unavailable(GUPnPControlPoint*,GUPnPDeviceProxy* device,gpointer) { + std::string id=gupnp_device_info_get_udn(GUPNP_DEVICE_INFO(device)); auto it=devices.find(id); + if (it!=devices.end() && g_object_get_data(G_OBJECT(it->second),"device")==device) { g_object_unref(it->second); devices.erase(it); } +} +static void action(GUPnPService*,GUPnPServiceAction* a,gpointer) { + const char* name=gupnp_service_action_get_name(a); + try { + if (strcmp(name,"GetProtocolInfo") && strncmp(name,"GetCurrentConnection",20)) { + guint instance=0; gupnp_service_action_get(a,"InstanceID",G_TYPE_UINT,&instance,nullptr); + if (instance!=0) { gupnp_service_action_return_error(a,718,"Invalid InstanceID"); return; } + } + if (!strcmp(name,"SetAVTransportURI")) { + gchar *uri=nullptr,*metadata=nullptr; gupnp_service_action_get(a,"CurrentURI",G_TYPE_STRING,&uri,"CurrentURIMetaData",G_TYPE_STRING,&metadata,nullptr); + std::string u=uri ? uri : "", meta=metadata ? metadata : ""; g_free(uri); g_free(metadata); + if (u.size()>8192 || meta.size()>65536 || (u.rfind("http://",0)!=0 && u.rfind("https://",0)!=0)) throw std::runtime_error("Only HTTP media URLs are supported"); + if (sessions.size()>=16) throw std::runtime_error("Window limit reached"); + gchar* id=g_uuid_string_random(); current=id; g_free(id); + sessions[current]=std::make_unique(current,u,meta); notify(); + output({{"event","request"},{"id",current},{"protocol","DLNA"},{"uri",u}}); + } else if (!strcmp(name,"GetTransportInfo")) gupnp_service_action_set(a,"CurrentTransportState",G_TYPE_STRING,state().c_str(),"CurrentTransportStatus",G_TYPE_STRING,"OK","CurrentSpeed",G_TYPE_STRING,"1",nullptr); + else if (!strcmp(name,"GetMediaInfo")) { auto* m=active(); auto duration=time_string(position(m,true)); gupnp_service_action_set(a,"NrTracks",G_TYPE_UINT,m ? 1u : 0u,"MediaDuration",G_TYPE_STRING,duration.c_str(),"CurrentURI",G_TYPE_STRING,m ? m->uri.c_str() : "","CurrentURIMetaData",G_TYPE_STRING,m ? m->metadata.c_str() : "","NextURI",G_TYPE_STRING,"","NextURIMetaData",G_TYPE_STRING,"","PlayMedium",G_TYPE_STRING,"NETWORK","RecordMedium",G_TYPE_STRING,"NOT_IMPLEMENTED","WriteStatus",G_TYPE_STRING,"NOT_IMPLEMENTED",nullptr); } + else if (!strcmp(name,"GetPositionInfo")) { auto* m=active(); auto duration=time_string(position(m,true)), pos=time_string(position(m,false)); gupnp_service_action_set(a,"Track",G_TYPE_UINT,m ? 1u : 0u,"TrackDuration",G_TYPE_STRING,duration.c_str(),"TrackMetaData",G_TYPE_STRING,m ? m->metadata.c_str() : "","TrackURI",G_TYPE_STRING,m ? m->uri.c_str() : "","RelTime",G_TYPE_STRING,pos.c_str(),"AbsTime",G_TYPE_STRING,pos.c_str(),"RelCount",G_TYPE_INT,0,"AbsCount",G_TYPE_INT,0,nullptr); } + else if (!strcmp(name,"GetDeviceCapabilities")) gupnp_service_action_set(a,"PlayMedia",G_TYPE_STRING,"NETWORK","RecMedia",G_TYPE_STRING,"NOT_IMPLEMENTED","RecQualityModes",G_TYPE_STRING,"NOT_IMPLEMENTED",nullptr); + else if (!strcmp(name,"GetTransportSettings")) gupnp_service_action_set(a,"PlayMode",G_TYPE_STRING,"NORMAL","RecQualityMode",G_TYPE_STRING,"NOT_IMPLEMENTED",nullptr); + else if (!strcmp(name,"GetCurrentTransportActions")) gupnp_service_action_set(a,"Actions",G_TYPE_STRING,active() && active()->accepted ? "Play,Pause,Stop,Seek" : "Play,Stop",nullptr); + else if (!strcmp(name,"GetProtocolInfo")) gupnp_service_action_set(a,"Source",G_TYPE_STRING,"","Sink",G_TYPE_STRING,protocols,nullptr); + else if (!strcmp(name,"GetCurrentConnectionIDs")) gupnp_service_action_set(a,"ConnectionIDs",G_TYPE_STRING,"0",nullptr); + else if (!strcmp(name,"GetCurrentConnectionInfo")) gupnp_service_action_set(a,"RcsID",G_TYPE_INT,0,"AVTransportID",G_TYPE_INT,0,"ProtocolInfo",G_TYPE_STRING,"http-get:*:*:*","PeerConnectionManager",G_TYPE_STRING,"","PeerConnectionID",G_TYPE_INT,-1,"Direction",G_TYPE_STRING,"Input","Status",G_TYPE_STRING,"OK",nullptr); + else if (!strcmp(name,"GetVolume")) gupnp_service_action_set(a,"CurrentVolume",G_TYPE_UINT,active() ? unsigned(active()->volume*100) : 100u,nullptr); + else if (!strcmp(name,"GetMute")) gupnp_service_action_set(a,"CurrentMute",G_TYPE_BOOLEAN,active() && active()->muted,nullptr); + else { + auto* m=active(); if (!m) { gupnp_service_action_return_error(a,701,"No media present"); return; } + if (!strcmp(name,"Stop")) { auto id=current; erase_media(id); output({{"event","ended"},{"id",id}}); } + else if (!strcmp(name,"Pause")) { m->paused=true; if (m->pipeline) gst_element_set_state(m->pipeline,GST_STATE_PAUSED); notify(); } + else if (!strcmp(name,"Play")) { gchar* speed=nullptr; gupnp_service_action_get(a,"Speed",G_TYPE_STRING,&speed,nullptr); bool supported=speed && !strcmp(speed,"1"); g_free(speed); if (!supported) { gupnp_service_action_return_error(a,717,"Unsupported play speed"); return; } m->paused=false; if (m->accepted) play(*m); notify(); } + else if (!strcmp(name,"Seek")) { gchar *unit=nullptr,*target=nullptr; gupnp_service_action_get(a,"Unit",G_TYPE_STRING,&unit,"Target",G_TYPE_STRING,&target,nullptr); std::string u=unit ? unit : "",t=target ? target : ""; g_free(unit); g_free(target); unsigned h=0,min=0; double sec=0; int count=0; if (u!="REL_TIME" || sscanf(t.c_str(),"%u:%u:%lf%n",&h,&min,&sec,&count)!=3 || count!=int(t.size()) || min>=60 || sec<0 || sec>=60) throw std::runtime_error("Unsupported seek target"); seek(*m,h*3600. + min*60. + sec); } + else if (!strcmp(name,"SetVolume")) { guint value=100; gupnp_service_action_get(a,"DesiredVolume",G_TYPE_UINT,&value,nullptr); set_volume(*m,std::min(value,100u)/100.); } + else if (!strcmp(name,"SetMute")) { gboolean value=false; gupnp_service_action_get(a,"DesiredMute",G_TYPE_BOOLEAN,&value,nullptr); m->muted=value; if (m->pipeline) g_object_set(m->pipeline,"mute",value,nullptr); notify(); } + } + gupnp_service_action_return_success(a); + } catch (const std::exception& e) { gupnp_service_action_return_error(a,714,e.what()); } +} +static void query_variable(GUPnPService*,const char* variable,GValue* value,gpointer rendering) { + g_value_init(value,G_TYPE_STRING); + if (!strcmp(variable,"LastChange")) { auto text=last_change(GPOINTER_TO_INT(rendering)); g_value_set_string(value,text.c_str()); } + else if (!strcmp(variable,"SinkProtocolInfo")) g_value_set_string(value,protocols); + else if (!strcmp(variable,"CurrentConnectionIDs")) g_value_set_string(value,"0"); + else g_value_set_string(value,""); +} +static bool lan_context(GUPnPContext* context) { + ifaddrs* first=nullptr; if (getifaddrs(&first)) return false; + bool result=false; auto* name=gssdp_client_get_interface(GSSDP_CLIENT(context)); + for (auto* i=first;i;i=i->ifa_next) if (i->ifa_name && name && !strcmp(i->ifa_name,name) && (i->ifa_flags&IFF_UP) && (i->ifa_flags&IFF_RUNNING) && (i->ifa_flags&IFF_BROADCAST) && !(i->ifa_flags&(IFF_LOOPBACK|IFF_POINTOPOINT))) result=true; + freeifaddrs(first); return result; +} +static void context_available(GUPnPContextManager* manager,GUPnPContext* context,gpointer) { + if (!lan_context(context)) return; + try { + g_object_set(gupnp_context_get_session(context),"timeout",5u,nullptr); + auto* cp=gupnp_control_point_new(context,"urn:schemas-upnp-org:device:MediaRenderer:1"); + g_signal_connect(cp,"device-proxy-available",G_CALLBACK(available),nullptr); g_signal_connect(cp,"device-proxy-unavailable",G_CALLBACK(unavailable),nullptr); + gupnp_context_manager_manage_control_point(manager,cp); gssdp_resource_browser_set_active(GSSDP_RESOURCE_BROWSER(cp),true); g_object_unref(cp); + if (receive_enabled) { + GError* error=nullptr; auto* root=gupnp_root_device_new(context,"device.xml",xml_directory.c_str(),&error); + if (!root) { std::string reason=error ? error->message : "DLNA description error"; g_clear_error(&error); throw std::runtime_error(reason); } + own_udn=gupnp_device_info_get_udn(GUPNP_DEVICE_INFO(root)); + for (auto type:{"AVTransport","RenderingControl","ConnectionManager"}) { + std::string urn="urn:schemas-upnp-org:service:"+std::string(type)+":1"; auto* service=gupnp_device_info_get_service(GUPNP_DEVICE_INFO(root),urn.c_str()); + if (!service) throw std::runtime_error("Missing DLNA service"); + for (auto name:{"SetAVTransportURI","Play","Pause","Stop","Seek","GetTransportInfo","GetMediaInfo","GetPositionInfo","GetDeviceCapabilities","GetTransportSettings","GetCurrentTransportActions","GetProtocolInfo","GetCurrentConnectionIDs","GetCurrentConnectionInfo","GetVolume","GetMute","SetVolume","SetMute"}) { + std::string signal="action-invoked::"+std::string(name); g_signal_connect(service,signal.c_str(),G_CALLBACK(action),nullptr); + } + g_signal_connect(service,"query-variable",G_CALLBACK(query_variable),GINT_TO_POINTER(!strcmp(type,"RenderingControl"))); + if (!strcmp(type,"AVTransport")) transports.push_back(GUPNP_SERVICE(service)); + else if (!strcmp(type,"RenderingControl")) renderers.push_back(GUPNP_SERVICE(service)); + else g_object_unref(service); + } + gupnp_context_manager_manage_root_device(manager,root); gupnp_root_device_set_available(root,true); g_object_unref(root); + } + output({{"event","ready"},{"port",gupnp_context_get_port(context)},{"host",gssdp_client_get_host_ip(GSSDP_CLIENT(context))}}); + } catch (const std::exception& e) { output({{"event","fatal"},{"error",e.what()}}); } +} +int main(int argc,char** argv) { + try { + if (argc<4) throw std::runtime_error("receiver DIRECTORY XMLDIR RECEIVE"); + directory=argv[1]; xml_directory=argv[2]; receive_enabled=!strcmp(argv[3],"1"); gst_init(&argc,&argv); + auto* manager=gupnp_context_manager_create_full(GSSDP_UDA_VERSION_1_0,G_SOCKET_FAMILY_IPV4,0); + g_signal_connect(manager,"context-available",G_CALLBACK(context_available),nullptr); + loop=g_main_loop_new(nullptr,false); auto* channel=g_io_channel_unix_new(STDIN_FILENO); g_io_add_watch(channel,(GIOCondition)(G_IO_IN|G_IO_HUP),input,nullptr); + g_main_loop_run(loop); sessions.clear(); + for (auto& [_,p]:devices) g_object_unref(p); + for (auto* p:transports) g_object_unref(p); + for (auto* p:renderers) g_object_unref(p); + g_io_channel_unref(channel); g_object_unref(manager); g_main_loop_unref(loop); return 0; + } catch (const std::exception& e) { output({{"event","fatal"},{"error",e.what()}}); return 1; } +} diff --git a/media/patch-uxplay.py b/media/patch-uxplay.py new file mode 100644 index 0000000..56183f1 --- /dev/null +++ b/media/patch-uxplay.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Add only lifecycle events to UxPlay; protocol/decoder code remains upstream.""" +from pathlib import Path +import sys +p=Path(sys.argv[1])/'uxplay.cpp' +s=p.read_text() +if 'framely_client_ended' not in s: + at=s.index('extern "C" void conn_init') + s=s[:at]+'''#include "framely/frame_sink.h" +static void framely_client_ended() { + const char* directory=getenv("FRAMELY_CAST_DIR"),*id=getenv("FRAMELY_CAST_ID"); + if(directory&&id)FrameSink(directory,id,"AirPlay").event("ended"); +} + +'''+s[at:] + s=s.replace('if (open_connections == 0) {','if (open_connections == 0) {\n framely_client_ended();',1) + s=s.replace('extern "C" void conn_reset (void *cls, int reason) {','extern "C" void conn_reset (void *cls, int reason) {\n framely_client_ended();',1) + p.write_text(s) + from shutil import copyfile + dst=p.parent/'framely';dst.mkdir(exist_ok=True) + copyfile(Path(__file__).parent/'native/frame_sink.h',dst/'frame_sink.h') + +# Keep one client throughout a consented session; a second device must not +# inherit the first device's permission while UxPlay still has open sockets. +s=p.read_text() +if 'framely_client_device' not in s: + s=s.replace('#include "framely/frame_sink.h"','#include "framely/frame_sink.h"\n#include \nstatic std::mutex framely_client_mutex;\nstatic std::string framely_client_device;') + s=s.replace('static void framely_client_ended() {','static void framely_client_ended() {\n {std::lock_guard lock(framely_client_mutex);framely_client_device.clear();}') + marker=' // Pass device model to renderer for device frame display' + assert marker in s + s=s.replace(marker,''' if (*admit && deviceid) { + std::lock_guard lock(framely_client_mutex); + if (framely_client_device.empty()) framely_client_device=deviceid; + else if (framely_client_device!=deviceid) *admit=false; + } +'''+marker) + p.write_text(s) diff --git a/media/upnp/AVTransport.xml b/media/upnp/AVTransport.xml new file mode 100644 index 0000000..2237cd6 --- /dev/null +++ b/media/upnp/AVTransport.xml @@ -0,0 +1,2 @@ + +10SetAVTransportURIInstanceIDinA_ARG_TYPE_InstanceIDCurrentURIinA_ARG_TYPE_CurrentURICurrentURIMetaDatainA_ARG_TYPE_CurrentURIMetaDataPlayInstanceIDinA_ARG_TYPE_InstanceIDSpeedinA_ARG_TYPE_SpeedPauseInstanceIDinA_ARG_TYPE_InstanceIDStopInstanceIDinA_ARG_TYPE_InstanceIDGetTransportInfoInstanceIDinA_ARG_TYPE_InstanceIDCurrentTransportStateoutA_ARG_TYPE_CurrentTransportStateCurrentTransportStatusoutA_ARG_TYPE_CurrentTransportStatusCurrentSpeedoutA_ARG_TYPE_CurrentSpeedGetMediaInfoInstanceIDinA_ARG_TYPE_InstanceIDNrTracksoutA_ARG_TYPE_NrTracksMediaDurationoutA_ARG_TYPE_MediaDurationCurrentURIoutA_ARG_TYPE_CurrentURICurrentURIMetaDataoutA_ARG_TYPE_CurrentURIMetaDataNextURIoutA_ARG_TYPE_NextURINextURIMetaDataoutA_ARG_TYPE_NextURIMetaDataPlayMediumoutA_ARG_TYPE_PlayMediumRecordMediumoutA_ARG_TYPE_RecordMediumWriteStatusoutA_ARG_TYPE_WriteStatusSeekInstanceIDinA_ARG_TYPE_InstanceIDUnitinA_ARG_TYPE_UnitTargetinA_ARG_TYPE_TargetGetDeviceCapabilitiesInstanceIDinA_ARG_TYPE_InstanceIDPlayMediaoutA_ARG_TYPE_PlayMediaRecMediaoutA_ARG_TYPE_RecMediaRecQualityModesoutA_ARG_TYPE_RecQualityModesGetTransportSettingsInstanceIDinA_ARG_TYPE_InstanceIDPlayModeoutA_ARG_TYPE_PlayModeRecQualityModeoutA_ARG_TYPE_RecQualityModeGetCurrentTransportActionsInstanceIDinA_ARG_TYPE_InstanceIDActionsoutA_ARG_TYPE_ActionsGetPositionInfoInstanceIDinA_ARG_TYPE_InstanceIDTrackoutA_ARG_TYPE_TrackTrackDurationoutA_ARG_TYPE_TrackDurationTrackMetaDataoutA_ARG_TYPE_TrackMetaDataTrackURIoutA_ARG_TYPE_TrackURIRelTimeoutA_ARG_TYPE_RelTimeAbsTimeoutA_ARG_TYPE_AbsTimeRelCountoutA_ARG_TYPE_RelCountAbsCountoutA_ARG_TYPE_AbsCountA_ARG_TYPE_InstanceIDui4A_ARG_TYPE_CurrentURIstringA_ARG_TYPE_CurrentURIMetaDatastringA_ARG_TYPE_SpeedstringA_ARG_TYPE_CurrentTransportStatestringA_ARG_TYPE_CurrentTransportStatusstringA_ARG_TYPE_CurrentSpeedstringA_ARG_TYPE_NrTracksui4A_ARG_TYPE_MediaDurationstringA_ARG_TYPE_NextURIstringA_ARG_TYPE_NextURIMetaDatastringA_ARG_TYPE_PlayMediumstringA_ARG_TYPE_RecordMediumstringA_ARG_TYPE_WriteStatusstringA_ARG_TYPE_UnitstringA_ARG_TYPE_TargetstringA_ARG_TYPE_PlayMediastringA_ARG_TYPE_RecMediastringA_ARG_TYPE_RecQualityModesstringA_ARG_TYPE_PlayModestringA_ARG_TYPE_RecQualityModestringA_ARG_TYPE_ActionsstringA_ARG_TYPE_Trackui4A_ARG_TYPE_TrackDurationstringA_ARG_TYPE_TrackMetaDatastringA_ARG_TYPE_TrackURIstringA_ARG_TYPE_RelTimestringA_ARG_TYPE_AbsTimestringA_ARG_TYPE_RelCounti4A_ARG_TYPE_AbsCounti4LastChangestring \ No newline at end of file diff --git a/media/upnp/ConnectionManager.xml b/media/upnp/ConnectionManager.xml new file mode 100644 index 0000000..d479002 --- /dev/null +++ b/media/upnp/ConnectionManager.xml @@ -0,0 +1,2 @@ + +10GetProtocolInfoSourceoutA_ARG_TYPE_SourceSinkoutA_ARG_TYPE_SinkGetCurrentConnectionIDsConnectionIDsoutA_ARG_TYPE_ConnectionIDsGetCurrentConnectionInfoConnectionIDinA_ARG_TYPE_ConnectionIDRcsIDoutA_ARG_TYPE_RcsIDAVTransportIDoutA_ARG_TYPE_AVTransportIDProtocolInfooutA_ARG_TYPE_ProtocolInfoPeerConnectionManageroutA_ARG_TYPE_PeerConnectionManagerPeerConnectionIDoutA_ARG_TYPE_PeerConnectionIDDirectionoutA_ARG_TYPE_DirectionStatusoutA_ARG_TYPE_StatusA_ARG_TYPE_SourcestringA_ARG_TYPE_SinkstringA_ARG_TYPE_ConnectionIDsstringA_ARG_TYPE_ConnectionIDi4A_ARG_TYPE_RcsIDi4A_ARG_TYPE_AVTransportIDi4A_ARG_TYPE_ProtocolInfostringA_ARG_TYPE_PeerConnectionManagerstringA_ARG_TYPE_PeerConnectionIDi4A_ARG_TYPE_DirectionstringA_ARG_TYPE_StatusstringSourceProtocolInfostringSinkProtocolInfostringCurrentConnectionIDsstring \ No newline at end of file diff --git a/media/upnp/RenderingControl.xml b/media/upnp/RenderingControl.xml new file mode 100644 index 0000000..aeb243c --- /dev/null +++ b/media/upnp/RenderingControl.xml @@ -0,0 +1,2 @@ + +10GetVolumeInstanceIDinA_ARG_TYPE_InstanceIDChannelinA_ARG_TYPE_ChannelCurrentVolumeoutA_ARG_TYPE_CurrentVolumeSetVolumeInstanceIDinA_ARG_TYPE_InstanceIDChannelinA_ARG_TYPE_ChannelDesiredVolumeinA_ARG_TYPE_DesiredVolumeGetMuteInstanceIDinA_ARG_TYPE_InstanceIDChannelinA_ARG_TYPE_ChannelCurrentMuteoutA_ARG_TYPE_CurrentMuteSetMuteInstanceIDinA_ARG_TYPE_InstanceIDChannelinA_ARG_TYPE_ChannelDesiredMuteinA_ARG_TYPE_DesiredMuteA_ARG_TYPE_InstanceIDui4A_ARG_TYPE_ChannelstringA_ARG_TYPE_CurrentVolumeui2A_ARG_TYPE_DesiredVolumeui2A_ARG_TYPE_CurrentMutebooleanA_ARG_TYPE_DesiredMutebooleanLastChangestring \ No newline at end of file diff --git a/media/upnp/generate.py b/media/upnp/generate.py new file mode 100644 index 0000000..dec1cff --- /dev/null +++ b/media/upnp/generate.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Generate the standard UPnP AV service descriptions shipped with Framely.""" +from pathlib import Path +from xml.etree.ElementTree import Element,SubElement,tostring +root=Path(__file__).parent +services={ +'AVTransport':[ + ('SetAVTransportURI',[('InstanceID','in','ui4'),('CurrentURI','in','string'),('CurrentURIMetaData','in','string')]), + ('Play',[('InstanceID','in','ui4'),('Speed','in','string')]),('Pause',[('InstanceID','in','ui4')]),('Stop',[('InstanceID','in','ui4')]), + ('GetTransportInfo',[('InstanceID','in','ui4'),('CurrentTransportState','out','string'),('CurrentTransportStatus','out','string'),('CurrentSpeed','out','string')]), + ('GetMediaInfo',[('InstanceID','in','ui4')]+[(v,'out',t) for v,t in [('NrTracks','ui4'),('MediaDuration','string'),('CurrentURI','string'),('CurrentURIMetaData','string'),('NextURI','string'),('NextURIMetaData','string'),('PlayMedium','string'),('RecordMedium','string'),('WriteStatus','string')]]), + ('Seek',[('InstanceID','in','ui4'),('Unit','in','string'),('Target','in','string')]), + ('GetDeviceCapabilities',[('InstanceID','in','ui4'),('PlayMedia','out','string'),('RecMedia','out','string'),('RecQualityModes','out','string')]), + ('GetTransportSettings',[('InstanceID','in','ui4'),('PlayMode','out','string'),('RecQualityMode','out','string')]), + ('GetCurrentTransportActions',[('InstanceID','in','ui4'),('Actions','out','string')]), + ('GetPositionInfo',[('InstanceID','in','ui4')]+[(v,'out',t) for v,t in [('Track','ui4'),('TrackDuration','string'),('TrackMetaData','string'),('TrackURI','string'),('RelTime','string'),('AbsTime','string'),('RelCount','i4'),('AbsCount','i4')]])], +'RenderingControl':[(k,[('InstanceID','in','ui4'),('Channel','in','string'),(v,d,t)]) for k,v,d,t in [('GetVolume','CurrentVolume','out','ui2'),('SetVolume','DesiredVolume','in','ui2'),('GetMute','CurrentMute','out','boolean'),('SetMute','DesiredMute','in','boolean')]], +'ConnectionManager':[('GetProtocolInfo',[('Source','out','string'),('Sink','out','string')]),('GetCurrentConnectionIDs',[('ConnectionIDs','out','string')]),('GetCurrentConnectionInfo',[('ConnectionID','in','i4')]+[(v,'out',t) for v,t in [('RcsID','i4'),('AVTransportID','i4'),('ProtocolInfo','string'),('PeerConnectionManager','string'),('PeerConnectionID','i4'),('Direction','string'),('Status','string')]])], +} +for service,actions in services.items(): + doc=Element('scpd',xmlns='urn:schemas-upnp-org:service-1-0');version=SubElement(doc,'specVersion');SubElement(version,'major').text='1';SubElement(version,'minor').text='0';actionlist=SubElement(doc,'actionList');variables={} + for name,args in actions: + action=SubElement(actionlist,'action');SubElement(action,'name').text=name;arglist=SubElement(action,'argumentList') + for v,d,t in args: + arg=SubElement(arglist,'argument');SubElement(arg,'name').text=v;SubElement(arg,'direction').text=d;SubElement(arg,'relatedStateVariable').text='A_ARG_TYPE_'+v;variables[v]=t + table=SubElement(doc,'serviceStateTable') + for v,t in variables.items(): + var=SubElement(table,'stateVariable',sendEvents='no');SubElement(var,'name').text='A_ARG_TYPE_'+v;SubElement(var,'dataType').text=t + for name in (['LastChange'] if service!='ConnectionManager' else ['SourceProtocolInfo','SinkProtocolInfo','CurrentConnectionIDs']): + var=SubElement(table,'stateVariable',sendEvents='yes');SubElement(var,'name').text=name;SubElement(var,'dataType').text='string' + (root/(service+'.xml')).write_bytes(tostring(doc,encoding='utf-8',xml_declaration=True)) diff --git a/native/cast_frame.h b/native/cast_frame.h new file mode 100644 index 0000000..746790b --- /dev/null +++ b/native/cast_frame.h @@ -0,0 +1,31 @@ +#pragma once +#include +#include +#include +#include +// Raw decoded video is uploaded once into a GL texture, then scaled on the GPU. +struct CastFrame { + std::string path;GLuint texture=0,composite=0;int composite_width=0,composite_height=0;uint32_t width=0,height=0;timespec stamp{};std::vector data; + bool update(){if(path.empty())return false;int fd=open(path.c_str(),O_RDONLY|O_CLOEXEC|O_NOFOLLOW);if(fd<0)return false; + struct stat st{};bool changed=fstat(fd,&st)==0&&(st.st_mtim.tv_sec!=stamp.tv_sec||st.st_mtim.tv_nsec!=stamp.tv_nsec); + if(!changed||flock(fd,LOCK_SH|LOCK_NB)){close(fd);return false;} + uint32_t header[4]{};bool valid=pread(fd,header,16,0)==16&&header[0]==0x46434153&&header[1]>0&&header[2]>0&&header[1]<=4096&&header[2]<=4096&&header[3]==header[1]*4&&st.st_size==16+int64_t(header[1])*header[2]*4; + if(valid){data.resize(size_t(header[1])*header[2]*4);valid=pread(fd,data.data(),data.size(),16)==ssize_t(data.size());} + flock(fd,LOCK_UN);close(fd);if(!valid)return false; + stamp=st.st_mtim;width=header[1];height=header[2];if(!texture)glGenTextures(1,&texture);glBindTexture(GL_TEXTURE_2D,texture);glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);glPixelStorei(GL_UNPACK_ROW_LENGTH,0);glPixelStorei(GL_UNPACK_ALIGNMENT,4);glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA8,width,height,0,GL_BGRA,GL_UNSIGNED_BYTE,data.data());return true; + } + GLuint draw(GLuint target,int target_width,int target_height){if(!texture||target_height<=0)return target; + static auto gen=(PFNGLGENFRAMEBUFFERSPROC)glXGetProcAddressARB((const GLubyte*)"glGenFramebuffers"); + static auto bind=(PFNGLBINDFRAMEBUFFERPROC)glXGetProcAddressARB((const GLubyte*)"glBindFramebuffer"); + static auto attach=(PFNGLFRAMEBUFFERTEXTURE2DPROC)glXGetProcAddressARB((const GLubyte*)"glFramebufferTexture2D"); + static auto blit=(PFNGLBLITFRAMEBUFFERPROC)glXGetProcAddressARB((const GLubyte*)"glBlitFramebuffer"); + static auto del=(PFNGLDELETEFRAMEBUFFERSPROC)glXGetProcAddressARB((const GLubyte*)"glDeleteFramebuffers"); + if(!gen||!bind||!attach||!blit||!del)return target; + if(!composite)glGenTextures(1,&composite);glBindTexture(GL_TEXTURE_2D,composite);if(composite_width!=target_width||composite_height!=target_height){glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA8,target_width,target_height,0,GL_BGRA,GL_UNSIGNED_BYTE,nullptr);composite_width=target_width;composite_height=target_height;}GLuint fbo[2];gen(2,fbo);bind(GL_READ_FRAMEBUFFER,fbo[0]);attach(GL_READ_FRAMEBUFFER,GL_COLOR_ATTACHMENT0,GL_TEXTURE_2D,texture,0);bind(GL_DRAW_FRAMEBUFFER,fbo[1]);attach(GL_DRAW_FRAMEBUFFER,GL_COLOR_ATTACHMENT0,GL_TEXTURE_2D,composite,0);blit(0,0,width,height,0,0,target_width,target_height,GL_COLOR_BUFFER_BIT,GL_LINEAR); + // CEF paints transparent, premultiplied controls over the full video plane. + // Keep its texture intact so partial CEF damage never contains stale video. + bind(GL_FRAMEBUFFER,fbo[1]);glPushAttrib(GL_ALL_ATTRIB_BITS);glViewport(0,0,target_width,target_height);glDisable(GL_DEPTH_TEST);glDisable(GL_SCISSOR_TEST);glEnable(GL_TEXTURE_2D);glEnable(GL_BLEND);glBlendFunc(GL_ONE,GL_ONE_MINUS_SRC_ALPHA);glColor4f(1,1,1,1);glBindTexture(GL_TEXTURE_2D,target); + glMatrixMode(GL_PROJECTION);glPushMatrix();glLoadIdentity();glOrtho(0,1,0,1,-1,1);glMatrixMode(GL_MODELVIEW);glPushMatrix();glLoadIdentity();glBegin(GL_QUADS);glTexCoord2f(0,0);glVertex2f(0,0);glTexCoord2f(1,0);glVertex2f(1,0);glTexCoord2f(1,1);glVertex2f(1,1);glTexCoord2f(0,1);glVertex2f(0,1);glEnd();glPopMatrix();glMatrixMode(GL_PROJECTION);glPopMatrix();glPopAttrib();bind(GL_FRAMEBUFFER,0);del(2,fbo);return composite; + } + ~CastFrame(){if(texture)glDeleteTextures(1,&texture);if(composite)glDeleteTextures(1,&composite);} +}; diff --git a/native/host.cpp b/native/host.cpp index 6217ea2..9bb784e 100644 --- a/native/host.cpp +++ b/native/host.cpp @@ -122,7 +122,9 @@ template struct Handler { Handler(){api.base.size=sizeof(T);api.base.add_ref=[](cef_base_ref_counted_t* p){++reinterpret_cast(p)->refs;};api.base.release=[](cef_base_ref_counted_t* p){--reinterpret_cast(p)->refs;return 0;};api.base.has_one_ref=[](cef_base_ref_counted_t* p){return int(reinterpret_cast(p)->refs==1);};api.base.has_at_least_one_ref=[](cef_base_ref_counted_t* p){return int(reinterpret_cast(p)->refs>0);};} T* acquire(){api.base.add_ref(&api.base);return &api;} }; +#include "cast_frame.h" struct View { + CastFrame cast_frame; std::string key,plugin,title;int width=600,height=840;float render_scale=1.f;bool dashboard=false,shown=false,dirty=false,closing=false,closed=false; OverlayPlacement placement;NotificationClick notification_click;LauncherPointer launcher_pointer;std::chrono::steady_clock::time_point haptic_at{},activation_at{}; cef_mouse_event_t pointer{};bool pointer_valid=false,stick_scrolling=false;vr::TrackedDeviceIndex_t pointer_device=vr::k_unTrackedDeviceIndexInvalid;std::chrono::steady_clock::time_point pointer_at{}; @@ -162,7 +164,8 @@ static std::vector icon(bool hover,bool active){ return p; } -static json poll_http(int port){int fd=socket(AF_INET,SOCK_STREAM,0);if(fd<0)throw std::runtime_error("HTTP socket failed");timeval timeout{1,0};setsockopt(fd,SOL_SOCKET,SO_RCVTIMEO,&timeout,sizeof(timeout));setsockopt(fd,SOL_SOCKET,SO_SNDTIMEO,&timeout,sizeof(timeout));sockaddr_in addr{};addr.sin_family=AF_INET;addr.sin_port=htons(port);inet_pton(AF_INET,"127.0.0.1",&addr.sin_addr);if(connect(fd,reinterpret_cast(&addr),sizeof(addr))){close(fd);throw std::runtime_error("UI agent unavailable");}std::string payload;{std::lock_guard lock(visibility_mutex);payload=visible_views.dump();}std::string req="POST /host/poll HTTP/1.1\r\nHost: 127.0.0.1:"+std::to_string(port)+"\r\nX-Framely-Native: "+token+"\r\nContent-Type: application/json\r\nContent-Length: "+std::to_string(payload.size())+"\r\nConnection: close\r\n\r\n"+payload;size_t sent=0;while(sent0){response.append(buffer,n);if(response.size()>4*1024*1024){close(fd);throw std::runtime_error("HTTP body limit");}}close(fd);auto body=response.find("\r\n\r\n");if(body==std::string::npos||response.find("200 OK")==std::string::npos)throw std::runtime_error("Invalid agent response");return json::parse(response.substr(body+4));} +static std::vector closed_casts; +static json poll_http(int port){int fd=socket(AF_INET,SOCK_STREAM,0);if(fd<0)throw std::runtime_error("HTTP socket failed");timeval timeout{1,0};setsockopt(fd,SOL_SOCKET,SO_RCVTIMEO,&timeout,sizeof(timeout));setsockopt(fd,SOL_SOCKET,SO_SNDTIMEO,&timeout,sizeof(timeout));sockaddr_in addr{};addr.sin_family=AF_INET;addr.sin_port=htons(port);inet_pton(AF_INET,"127.0.0.1",&addr.sin_addr);if(connect(fd,reinterpret_cast(&addr),sizeof(addr))){close(fd);throw std::runtime_error("UI agent unavailable");}std::string payload;size_t closed_count=0;{std::lock_guard lock(visibility_mutex);closed_count=closed_casts.size();payload=json{{"views",visible_views},{"closedCasts",closed_casts}}.dump();}std::string req="POST /host/poll HTTP/1.1\r\nHost: 127.0.0.1:"+std::to_string(port)+"\r\nX-Framely-Native: "+token+"\r\nContent-Type: application/json\r\nContent-Length: "+std::to_string(payload.size())+"\r\nConnection: close\r\n\r\n"+payload;size_t sent=0;while(sent0){response.append(buffer,n);if(response.size()>4*1024*1024){close(fd);throw std::runtime_error("HTTP body limit");}}close(fd);auto body=response.find("\r\n\r\n");if(body==std::string::npos||response.find("200 OK")==std::string::npos)throw std::runtime_error("Invalid agent response");auto parsed=json::parse(response.substr(body+4));{std::lock_guard lock(visibility_mutex);closed_casts.erase(closed_casts.begin(),closed_casts.begin()+std::min(closed_count,closed_casts.size()));}return parsed;} // Frame renders the native laser cursor in the default overlay sort group. // Raising a panel above that group can obscure the runtime cursor. // OpenVR owns and renders the laser cursor. Coordinates must remain in the @@ -265,7 +268,7 @@ static bool keyboard_event(const vr::VREvent_t& e,vr::VROverlayHandle_t source_o }else {keyboard_done=keyboard_done||e.eventType==vr::VREvent_KeyboardDone;keyboard_close_pending=true;} return true; } -static void close_view(View& v){if(v.closing)return;cancel_file_dialog(v);if(keyboard_view==v.key)close_keyboard();v.closing=true;if(v.key=="menu")overlays->SetOverlayFlag(v.overlay,vr::VROverlayFlags_WantsModalBehavior,false);overlays->ClearOverlayCursorPositionOverride(v.overlay);overlays->HideOverlay(v.overlay);v.host->close_browser(v.host,1);} +static void close_view(View& v){if(v.closing)return;if(v.key.rfind("framely.cast.",0)==0){std::lock_guard lock(visibility_mutex);closed_casts.push_back(v.key.substr(13));}cancel_file_dialog(v);if(keyboard_view==v.key)close_keyboard();v.closing=true;if(v.key=="menu")overlays->SetOverlayFlag(v.overlay,vr::VROverlayFlags_WantsModalBehavior,false);overlays->ClearOverlayCursorPositionOverride(v.overlay);overlays->HideOverlay(v.overlay);v.host->close_browser(v.host,1);} static void anchor_to_head(View& v,float width,float x=0,float y=0,float z=-1.35f){vr::TrackedDevicePose_t poses[vr::k_unMaxTrackedDeviceCount]{};vr_system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding,0,poses,vr::k_unMaxTrackedDeviceCount);if(!poses[0].bPoseIsValid)return;auto m=poses[0].mDeviceToAbsoluteTracking;for(int r=0;r<3;r++)m.m[r][3]+=m.m[r][0]*x+m.m[r][1]*y+m.m[r][2]*z;overlays->SetOverlayWidthInMeters(v.overlay,width);overlays->SetOverlayTransformAbsolute(v.overlay,vr::TrackingUniverseStanding,&m);} static void anchor_notification(View& v){ // Let the compositor follow the HMD's render pose; a captured standing-space @@ -298,7 +301,7 @@ static View& create_view(const std::string& key,const std::string& title,const s } } if(dock){auto pixels=icon(false,true);overlays->SetOverlayRaw(v->thumbnail,pixels.data(),128,128,4);overlays->SetOverlayWidthInMeters(v->overlay,physical_width>0?physical_width:(key=="framely.manager"?3.2f:key=="launcher"?3.6f:1.2f));}else if(key!="menu"&&key!="notifications"&&key!="launcher"){anchor_to_head(*v,physical_width>0?physical_width:1.0f);} - cef_window_info_t info{};info.size=sizeof(info);info.windowless_rendering_enabled=1;cef_browser_settings_t bs{};bs.size=sizeof(bs);bs.windowless_frame_rate=browser_frame_rate();bs.background_color=0;std::cout<<"Browser frame rate "<DestroyOverlay(v->overlay);if(v->thumbnail)overlays->DestroyOverlay(v->thumbnail);throw std::runtime_error("Browser creation failed");}v->browser=browser;v->browser_id=browser->get_identifier(browser);v->host=browser->get_host(browser);glGenTextures(2,v->textures);auto* ptr=v.get();views.emplace(key,std::move(v));ptr->host->notify_screen_info_changed(ptr->host);ptr->host->was_resized(ptr->host);ptr->host->was_hidden(ptr->host,key!="notifications");if(dock&&key!="launcher")overlays->ShowDashboard(key.c_str());else if(key!="menu"&&key!="notifications"&&key!="launcher")show(*ptr,true);return *ptr; } static void plugin_thumbnail(View& view,const std::string& encoded){ @@ -547,9 +550,9 @@ static void entry_actions(vr::VROverlayHandle_t button,bool available){ if(gaze_allowed&&gaze_active)launcher_outside_at={}; if(launcher_open&&launcher_outside_at.time_since_epoch().count()&&now>=launcher_outside_at){launcher_outside_at={};if(!keyboard_open&&!launcher_dock_interaction())launcher_back();} } -static void commands(){std::deque commands;{std::lock_guard lock(queue_mutex);commands.swap(queue);}for(const auto& c:commands){try{auto kind=c.value("kind","");if(kind=="feedback.settings"){ui_haptics=c.value("haptics",true);ui_sounds=c.value("sounds",false);if(!ui_sounds)ui_audio.silence();}else if(kind=="file.dialog.complete"){for(auto& [key,view]:views){auto& v=*view;if(!v.file_dialog||v.file_dialog_id!=c.value("id",std::string{}))continue;auto paths=c.value("paths",json::array());if(paths.empty()){cancel_file_dialog(v);break;}if((!v.file_multiple&&paths.size()!=1)||paths.size()>64)break;auto* callback=v.file_dialog;v.file_dialog=nullptr;v.file_dialog_id.clear();auto list=cef_string_list_alloc();for(const auto& path:paths){cef_string_t value{};str(value,path.get());cef_string_list_append(list,&value);cef_string_utf16_clear(&value);}callback->cont(callback,list);cef_string_list_free(list);callback->base.release(&callback->base);break;}}else if(kind=="gaze.correction"){gaze_correction=c.value("calibration",json(nullptr));}else if(kind=="gaze.target"){gaze_target_id=c.value("id",std::string{});gaze_target_u=c.value("x",.5f);gaze_target_v=1-c.value("y",.5f);gaze_preview=c.value("preview",json(nullptr));gaze_target_cached=false;}else if(kind=="launcher.calibrate"){gaze_target_id.clear();gaze_target_cached=false;gaze_preview=nullptr;launcher_calibrating=c.value("active",true);if(launcher_calibrating&&!launcher_open){if(!views.count("launcher"))create_view("launcher","Framely launcher",origin+"/launcher#calibrate",true,1440,800);toggle_entry(true);}if(views.count("launcher")){auto& v=*views.at("launcher");auto* frame=v.browser->get_main_frame(v.browser);execute(frame,std::string("location.hash=")+(launcher_calibrating?"'#calibrate'":"''")+";window.dispatchEvent(new Event('hashchange'))");frame->base.release(&frame->base);}}else if(kind=="launcher.close"){if(c.contains("launcherSession")&&!c["launcherSession"].is_null()&&c["launcherSession"]!=launcher_session)continue;bool handoff=c.value("handoff",false);hide_launcher(!handoff,handoff);}else if(kind=="menu.open"){hide_launcher(false);auto& menu=create_view("menu","Framely",origin+"/",false,600,840);navigate(menu,"#plugin:"+c.at("plugin").get());menu_open=true;launcher_open=false;menu_button.opened(std::chrono::steady_clock::now());}else if(kind=="haptic"){auto key=c.value("view","");if(views.count(key)&&views.at(key)->shown&&!views.at(key)->closing){auto& view=*views.at(key);bool activation=c.value("feedback","hover")=="activate";if(activation&&key=="launcher"&&gaze_active&&launcher_gaze_allowed()){auto now=std::chrono::steady_clock::now();if(now-view.activation_at>=std::chrono::milliseconds(80)){view.activation_at=now;if(ui_sounds)ui_audio.play(true);}}else hover_haptic(view.overlay,activation?view.activation_at:view.haptic_at,activation);}}else if(kind=="manager.open"){hide_launcher(false);if(views.count("framely.manager")&&views.at("framely.manager")->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto page=c.value("page",std::string{});auto& manager=create_view("framely.manager","Framely",origin+"/manager"+((manager_navigation_page(page)||page.rfind("plugin:",0)==0)?"#"+page:""),true,1440,810);if(page.rfind("plugin:",0)==0)navigate(manager,"#"+page);if(manager_navigation_page(page)){auto* frame=manager.browser->get_main_frame(manager.browser);execute(frame,"location.hash="+json(page).dump()+";window.dispatchEvent(new Event(\"hashchange\"))");frame->base.release(&frame->base);}menu_open=false;launcher_open=false;}else if(kind=="agreement.decline"){auto key=c.value("view","");if(key=="menu"){menu_open=false;if(views.count("menu")){show(*views.at("menu"),false);views.at("menu")->browser->reload(views.at("menu")->browser);}}else if(views.count(key))close_view(*views.at(key));}else if(kind=="manager.close"){if(views.count("framely.manager"))close_view(*views.at("framely.manager"));}else if(kind=="menu.close"){menu_open=false;}else if(kind=="window.open"){hide_launcher(false);std::string plugin=c.at("plugin"),win=c.at("window");auto key="framely.window."+plugin+"."+win;if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto spec=c.at("spec");auto& opened=create_view("framely.window."+plugin+"."+win,spec.at("title"),origin+"/window/"+plugin+"/"+win,spec.value("dockIcon",false),spec.value("width",1600),spec.value("height",900),plugin,spec.value("widthMeters",3.f));plugin_thumbnail(opened,spec.value("iconData",std::string{}));menu_open=false;launcher_open=false;}else if(kind=="window.close"){auto key="framely.window."+c.at("plugin").get()+"."+c.at("window").get();if(views.count(key))close_view(*views.at(key));}else if(kind=="plugin.disabled"){for(auto&[key,v]:views)if(v->plugin==c.at("plugin"))close_view(*v);}else if(kind=="notification.changed"){auto& v=create_view("notifications","Framely notifications",origin+"/notifications",false,600,360);anchor_notification(v);v.host->was_hidden(v.host,0);v.host->invalidate(v.host,PET_VIEW);}else if(kind=="keyboard"){auto key=c.value("view","menu");if(!views.count(key))continue;auto& v=*views.at(key);if(v.closing)continue;close_keyboard();v.host->set_focus(v.host,1);keyboard_frame=v.browser->get_focused_frame(v.browser);keyboard_view=key;keyboard_target=v.overlay;++keyboard_token;auto existing=c.value("existing","");if(key=="menu")overlays->SetOverlayFlag(v.overlay,vr::VROverlayFlags_WantsModalBehavior,false);auto e=overlays->ShowKeyboardForOverlay(v.overlay,c.value("password",false)?vr::k_EGamepadTextInputModePassword:vr::k_EGamepadTextInputModeNormal,c.value("multiline",false)?vr::k_EGamepadTextInputLineModeMultipleLines:vr::k_EGamepadTextInputLineModeSingleLine,vr::KeyboardFlag_Modal|vr::KeyboardFlag_Minimal|vr::KeyboardFlag_ShowArrowKeys,"Framely",4096,existing.c_str(),keyboard_token);keyboard_open=e==vr::VROverlayError_None;keyboard_menu.begin(keyboard_open&&key=="menu");keyboard_state(keyboard_open);if(!keyboard_open)close_keyboard();}else if(kind=="notifications.empty"){if(views.count("notifications")){views.at("notifications")->notification_content=false;show(*views.at("notifications"),false);}}}catch(const std::exception& e){std::cerr<<"Native command error: "< commands;{std::lock_guard lock(queue_mutex);commands.swap(queue);}for(const auto& c:commands){try{auto kind=c.value("kind","");if(kind=="feedback.settings"){ui_haptics=c.value("haptics",true);ui_sounds=c.value("sounds",false);if(!ui_sounds)ui_audio.silence();}else if(kind=="file.dialog.complete"){for(auto& [key,view]:views){auto& v=*view;if(!v.file_dialog||v.file_dialog_id!=c.value("id",std::string{}))continue;auto paths=c.value("paths",json::array());if(paths.empty()){cancel_file_dialog(v);break;}if((!v.file_multiple&&paths.size()!=1)||paths.size()>64)break;auto* callback=v.file_dialog;v.file_dialog=nullptr;v.file_dialog_id.clear();auto list=cef_string_list_alloc();for(const auto& path:paths){cef_string_t value{};str(value,path.get());cef_string_list_append(list,&value);cef_string_utf16_clear(&value);}callback->cont(callback,list);cef_string_list_free(list);callback->base.release(&callback->base);break;}}else if(kind=="gaze.correction"){gaze_correction=c.value("calibration",json(nullptr));}else if(kind=="gaze.target"){gaze_target_id=c.value("id",std::string{});gaze_target_u=c.value("x",.5f);gaze_target_v=1-c.value("y",.5f);gaze_preview=c.value("preview",json(nullptr));gaze_target_cached=false;}else if(kind=="launcher.calibrate"){gaze_target_id.clear();gaze_target_cached=false;gaze_preview=nullptr;launcher_calibrating=c.value("active",true);if(launcher_calibrating&&!launcher_open){if(!views.count("launcher"))create_view("launcher","Framely launcher",origin+"/launcher#calibrate",true,1440,800);toggle_entry(true);}if(views.count("launcher")){auto& v=*views.at("launcher");auto* frame=v.browser->get_main_frame(v.browser);execute(frame,std::string("location.hash=")+(launcher_calibrating?"'#calibrate'":"''")+";window.dispatchEvent(new Event('hashchange'))");frame->base.release(&frame->base);}}else if(kind=="launcher.close"){if(c.contains("launcherSession")&&!c["launcherSession"].is_null()&&c["launcherSession"]!=launcher_session)continue;bool handoff=c.value("handoff",false);hide_launcher(!handoff,handoff);}else if(kind=="menu.open"){hide_launcher(false);auto& menu=create_view("menu","Framely",origin+"/",false,600,840);navigate(menu,"#plugin:"+c.at("plugin").get());menu_open=true;launcher_open=false;menu_button.opened(std::chrono::steady_clock::now());}else if(kind=="haptic"){auto key=c.value("view","");if(views.count(key)&&views.at(key)->shown&&!views.at(key)->closing){auto& view=*views.at(key);bool activation=c.value("feedback","hover")=="activate";if(activation&&key=="launcher"&&gaze_active&&launcher_gaze_allowed()){auto now=std::chrono::steady_clock::now();if(now-view.activation_at>=std::chrono::milliseconds(80)){view.activation_at=now;if(ui_sounds)ui_audio.play(true);}}else hover_haptic(view.overlay,activation?view.activation_at:view.haptic_at,activation);}}else if(kind=="manager.open"){hide_launcher(false);if(views.count("framely.manager")&&views.at("framely.manager")->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto page=c.value("page",std::string{});auto& manager=create_view("framely.manager","Framely",origin+"/manager"+((manager_navigation_page(page)||page.rfind("plugin:",0)==0)?"#"+page:""),true,1440,810);if(page.rfind("plugin:",0)==0)navigate(manager,"#"+page);if(manager_navigation_page(page)){auto* frame=manager.browser->get_main_frame(manager.browser);execute(frame,"location.hash="+json(page).dump()+";window.dispatchEvent(new Event(\"hashchange\"))");frame->base.release(&frame->base);}menu_open=false;launcher_open=false;}else if(kind=="agreement.decline"){auto key=c.value("view","");if(key=="menu"){menu_open=false;if(views.count("menu")){show(*views.at("menu"),false);views.at("menu")->browser->reload(views.at("menu")->browser);}}else if(views.count(key))close_view(*views.at(key));}else if(kind=="manager.close"){if(views.count("framely.manager"))close_view(*views.at("framely.manager"));}else if(kind=="menu.close"){menu_open=false;}else if(kind=="cast.window"){auto id=c.at("id").get();auto key="framely.cast."+id;if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}int width=c.value("width",640),height=c.value("height",240);if(width<160||width>1920||height<112||height>8192)continue;auto& v=create_view(key,"Framely casting",origin+"/cast/window/"+id,false,width,height,"",2.f);v.cast_frame.path=c.at("frame").get();if(v.width!=width||v.height!=height){v.width=width;v.height=height;v.pixels.clear();v.texture_ready[0]=v.texture_ready[1]=false;v.host->was_resized(v.host);vr::HmdVector2_t scale{{float(width),float(height)}};overlays->SetOverlayMouseScale(v.overlay,&scale);}menu_open=false;launcher_open=false;}else if(kind=="cast.close"){auto key="framely.cast."+c.at("id").get();if(views.count(key))close_view(*views.at(key));}else if(kind=="window.open"){hide_launcher(false);std::string plugin=c.at("plugin"),win=c.at("window");auto key="framely.window."+plugin+"."+win;if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto spec=c.at("spec");auto& opened=create_view("framely.window."+plugin+"."+win,spec.at("title"),origin+"/window/"+plugin+"/"+win,spec.value("dockIcon",false),spec.value("width",1600),spec.value("height",900),plugin,spec.value("widthMeters",3.f));plugin_thumbnail(opened,spec.value("iconData",std::string{}));menu_open=false;launcher_open=false;}else if(kind=="window.close"){auto key="framely.window."+c.at("plugin").get()+"."+c.at("window").get();if(views.count(key))close_view(*views.at(key));}else if(kind=="plugin.disabled"){for(auto&[key,v]:views)if(v->plugin==c.at("plugin"))close_view(*v);}else if(kind=="notification.changed"){auto& v=create_view("notifications","Framely notifications",origin+"/notifications",false,600,360);anchor_notification(v);v.host->was_hidden(v.host,0);v.host->invalidate(v.host,PET_VIEW);}else if(kind=="keyboard"){auto key=c.value("view","menu");if(!views.count(key))continue;auto& v=*views.at(key);if(v.closing)continue;close_keyboard();v.host->set_focus(v.host,1);keyboard_frame=v.browser->get_focused_frame(v.browser);keyboard_view=key;keyboard_target=v.overlay;++keyboard_token;auto existing=c.value("existing","");if(key=="menu")overlays->SetOverlayFlag(v.overlay,vr::VROverlayFlags_WantsModalBehavior,false);auto e=overlays->ShowKeyboardForOverlay(v.overlay,c.value("password",false)?vr::k_EGamepadTextInputModePassword:vr::k_EGamepadTextInputModeNormal,c.value("multiline",false)?vr::k_EGamepadTextInputLineModeMultipleLines:vr::k_EGamepadTextInputLineModeSingleLine,vr::KeyboardFlag_Modal|vr::KeyboardFlag_Minimal|vr::KeyboardFlag_ShowArrowKeys,"Framely",4096,existing.c_str(),keyboard_token);keyboard_open=e==vr::VROverlayError_None;keyboard_menu.begin(keyboard_open&&key=="menu");keyboard_state(keyboard_open);if(!keyboard_open)close_keyboard();}else if(kind=="notifications.empty"){if(views.count("notifications")){views.at("notifications")->notification_content=false;show(*views.at("notifications"),false);}}}catch(const std::exception& e){std::cerr<<"Native command error: "<IsHoverTargetOverlay(v.overlay))launcher_back();else if(v.key=="menu"&&!keyboard_open&&!keyboard_menu.holds(std::chrono::steady_clock::now())&&menu_button.can_cancel(std::chrono::steady_clock::now())){menu_open=false;std::cout<<"Menu closed: outside click"<get_main_frame(v.browser);execute(frame,"window.dispatchEvent(new Event('framely.back'))");frame->base.release(&frame->base);}break;case vr::VREvent_FocusLeave:if(v.key=="launcher"&&v.launcher_pointer.holding(std::chrono::steady_clock::now()))break;if(v.key!="notifications"||!v.notification_click.active)v.host->send_mouse_move_event(v.host,&mouse,1);break;case vr::VREvent_MouseMove:if(v.key=="launcher")v.launcher_pointer.constrain(mouse.x,mouse.y,std::chrono::steady_clock::now());if(v.key!="notifications"||!v.notification_click.hold(mouse.x,mouse.y,std::chrono::steady_clock::now()))v.host->send_mouse_move_event(v.host,&mouse,0);break;case vr::VREvent_MouseButtonDown:case vr::VREvent_MouseButtonUp:{v.host->set_focus(v.host,1);auto button=e.data.mouse.button==vr::VRMouseButton_Right?MBT_RIGHT:MBT_LEFT;if(v.key=="launcher"&&button==MBT_LEFT){auto now=std::chrono::steady_clock::now();if(e.eventType==vr::VREvent_MouseButtonDown)v.launcher_pointer.begin(mouse.x,mouse.y,now);else{v.launcher_pointer.constrain(mouse.x,mouse.y,now);v.launcher_pointer.clear();}}if(v.key=="notifications"&&button==MBT_LEFT){auto now=std::chrono::steady_clock::now();if(e.eventType==vr::VREvent_MouseButtonDown)v.notification_click.begin(mouse.x,mouse.y,now);else v.notification_click.release(mouse.x,mouse.y,now);}mouse.modifiers=e.eventType==vr::VREvent_MouseButtonDown?EVENTFLAG_LEFT_MOUSE_BUTTON:0;v.host->send_mouse_click_event(v.host,&mouse,button,e.eventType==vr::VREvent_MouseButtonUp,1);break;}case vr::VREvent_ScrollSmooth:case vr::VREvent_ScrollDiscrete:if(v.stick_scrolling||keyboard_open||!v.pointer_valid)break;mouse=v.pointer;mouse.modifiers=0;v.host->send_mouse_wheel_event(v.host,&mouse,int(e.data.scroll.xdelta*120),int(e.data.scroll.ydelta*120));break;case vr::VREvent_OverlayGamepadFocusLost:if(!keyboard_open&&!v.dashboard&&v.key!="menu"&&v.key!="notifications"&&v.key!="launcher")close_view(v);break;default:break;}} +static void overlay_input(View& v,const vr::VREvent_t& e){if(v.closing)return;if((v.key=="notifications"||v.key=="launcher"||v.key=="menu")&&!v.shown&&e.eventType!=vr::VREvent_OverlayClosed)return;if(v.key=="launcher"&&!launcher_open&&e.eventType!=vr::VREvent_OverlayClosed)return;if(v.key=="launcher"&&!launcher_accepts(e)&&(e.eventType==vr::VREvent_Modal_Cancel||e.eventType==vr::VREvent_ButtonPress||e.eventType==vr::VREvent_MouseButtonDown||e.eventType==vr::VREvent_MouseButtonUp))return;if(keyboard_event(e,v.overlay))return;laser_cursor(v.overlay,e);cef_mouse_event_t mouse{};mouse.x=int(e.data.mouse.x);mouse.y=v.height-int(e.data.mouse.y);if(e.eventType==vr::VREvent_MouseMove||e.eventType==vr::VREvent_MouseButtonDown||e.eventType==vr::VREvent_MouseButtonUp){v.pointer=mouse;v.pointer_valid=true;v.pointer_device=e.trackedDeviceIndex;v.pointer_at=std::chrono::steady_clock::now();}switch(e.eventType){case vr::VREvent_Modal_Cancel:if(v.key=="launcher"&&!keyboard_open&&!gaze_active&&!launcher_dock_interaction()&&!overlays->IsHoverTargetOverlay(v.overlay))launcher_back();else if(v.key=="menu"&&!keyboard_open&&!keyboard_menu.holds(std::chrono::steady_clock::now())&&menu_button.can_cancel(std::chrono::steady_clock::now())){menu_open=false;std::cout<<"Menu closed: outside click"<get_main_frame(v.browser);execute(frame,"window.dispatchEvent(new Event('framely.back'))");frame->base.release(&frame->base);}break;case vr::VREvent_FocusLeave:if(v.key=="launcher"&&v.launcher_pointer.holding(std::chrono::steady_clock::now()))break;if(v.key!="notifications"||!v.notification_click.active)v.host->send_mouse_move_event(v.host,&mouse,1);break;case vr::VREvent_MouseMove:if(v.key=="launcher")v.launcher_pointer.constrain(mouse.x,mouse.y,std::chrono::steady_clock::now());if(v.key!="notifications"||!v.notification_click.hold(mouse.x,mouse.y,std::chrono::steady_clock::now()))v.host->send_mouse_move_event(v.host,&mouse,0);break;case vr::VREvent_MouseButtonDown:case vr::VREvent_MouseButtonUp:{v.host->set_focus(v.host,1);auto button=e.data.mouse.button==vr::VRMouseButton_Right?MBT_RIGHT:MBT_LEFT;if(v.key=="launcher"&&button==MBT_LEFT){auto now=std::chrono::steady_clock::now();if(e.eventType==vr::VREvent_MouseButtonDown)v.launcher_pointer.begin(mouse.x,mouse.y,now);else{v.launcher_pointer.constrain(mouse.x,mouse.y,now);v.launcher_pointer.clear();}}if(v.key=="notifications"&&button==MBT_LEFT){auto now=std::chrono::steady_clock::now();if(e.eventType==vr::VREvent_MouseButtonDown)v.notification_click.begin(mouse.x,mouse.y,now);else v.notification_click.release(mouse.x,mouse.y,now);}mouse.modifiers=e.eventType==vr::VREvent_MouseButtonDown?EVENTFLAG_LEFT_MOUSE_BUTTON:0;v.host->send_mouse_click_event(v.host,&mouse,button,e.eventType==vr::VREvent_MouseButtonUp,1);break;}case vr::VREvent_ScrollSmooth:case vr::VREvent_ScrollDiscrete:if(v.stick_scrolling||keyboard_open||!v.pointer_valid)break;mouse=v.pointer;mouse.modifiers=0;v.host->send_mouse_wheel_event(v.host,&mouse,int(e.data.scroll.xdelta*120),int(e.data.scroll.ydelta*120));break;case vr::VREvent_OverlayGamepadFocusLost:if(!keyboard_open&&!v.dashboard&&v.key!="menu"&&v.key!="notifications"&&v.key!="launcher"&&v.key.rfind("framely.cast.",0)!=0)close_view(v);break;default:break;}} int main(int argc,char** argv){ std::cout.setf(std::ios::unitbuf);std::string initial;int port=0;for(int i=1;i+1GetOverlayMouseScale(main_overlay,&main_scale)&&!overlays->GetTransformForOverlayCoordinates(main_overlay,vr::TrackingUniverseStanding,{{main_scale.v[0]*.5f,main_scale.v[1]*.5f}},&main_pose))main=&main_pose; if(evidence.radius>0?dock_menu_placement(anchor,size,evidence.radius,evidence.pre_curve_pitch,popup):menu_placement(anchor,size,popup,0,main)){v.placement.apply(overlays,v.overlay,popup.transform,popup.width);v.placement.curve(overlays,v.overlay,popup.width,popup.radius);show(v,menu_open);}else show(v,false);}}else if(!keyboard_menu.holds(now)){if(overlays->IsOverlayVisible(button))overlays->ClearOverlayCursorPositionOverride(button);overlays->HideOverlay(button);if(views.count("menu"))show(*views.at("menu"),false);} - if(was_visible&&!visible&&!keyboard_open&&!keyboard_menu.holds(now)){menu_button.pressed=false;menu_open=false;if(!overlays->IsDashboardVisible())hide_launcher(false,true);for(auto&[key,v]:views)if(!v->dashboard&&key!="menu"&&key!="notifications"&&key!="launcher")close_view(*v);}was_visible=visible; + if(was_visible&&!visible&&!keyboard_open&&!keyboard_menu.holds(now)){menu_button.pressed=false;menu_open=false;if(!overlays->IsDashboardVisible())hide_launcher(false,true);for(auto&[key,v]:views)if(!v->dashboard&&key!="menu"&&key!="notifications"&&key!="launcher"&&key.rfind("framely.cast.",0)!=0)close_view(*v);}was_visible=visible; if(launcher_close_at.time_since_epoch().count()&&now>=launcher_close_at){launcher_close_at={};if(!launcher_open)finish_launcher_close();} // The runtime owns dashboard placement, curvature, visibility and switching. // Never move the launcher into standing space or hide another app's overlay. @@ -656,9 +659,10 @@ int main(int argc,char** argv){ for(auto&[key,v]:views)v->stick_scrolling=false; entry_actions(button,anchor_ok); if(launcher_open){bool held=false;for(auto& hand:entry_buttons)for(auto& trigger:hand)held|=trigger.held;launcher_dismiss.observe(held);} - for(auto&[key,v]:views){if(v->closing)continue;if(v->dashboard){bool active=overlays->IsDashboardVisible()&&overlays->IsActiveDashboardOverlay(v->overlay)&&overlays->IsOverlayVisible(v->overlay);if(active!=v->shown){if(!active&&key=="launcher"){if(v->launcher_pointer.active){cef_mouse_event_t outside{};outside.modifiers=EVENTFLAG_LEFT_MOUSE_BUTTON;v->host->send_mouse_move_event(v->host,&outside,0);outside.modifiers=0;v->host->send_mouse_click_event(v->host,&outside,MBT_LEFT,1,1);}v->launcher_pointer.clear();auto* frame=v->browser->get_main_frame(v->browser);execute(frame,"window.dispatchEvent(new Event('framely.launcher.cancelInput'))");frame->base.release(&frame->base);}v->host->was_hidden(v->host,!active);v->shown=active;}if(!active)v->active_seen=false;if(active&&!v->active_seen){v->active_seen=true;if(key!="launcher")for(auto&[other,t]:views)if(!t->dashboard&&other!="menu"&&other!="notifications"&&other!="launcher"&&!keyboard_open)close_view(*t);}} + for(auto&[key,v]:views){if(v->closing)continue;if(v->dashboard){bool active=overlays->IsDashboardVisible()&&overlays->IsActiveDashboardOverlay(v->overlay)&&overlays->IsOverlayVisible(v->overlay);if(active!=v->shown){if(!active&&key=="launcher"){if(v->launcher_pointer.active){cef_mouse_event_t outside{};outside.modifiers=EVENTFLAG_LEFT_MOUSE_BUTTON;v->host->send_mouse_move_event(v->host,&outside,0);outside.modifiers=0;v->host->send_mouse_click_event(v->host,&outside,MBT_LEFT,1,1);}v->launcher_pointer.clear();auto* frame=v->browser->get_main_frame(v->browser);execute(frame,"window.dispatchEvent(new Event('framely.launcher.cancelInput'))");frame->base.release(&frame->base);}v->host->was_hidden(v->host,!active);v->shown=active;}if(!active)v->active_seen=false;if(active&&!v->active_seen){v->active_seen=true;if(key!="launcher")for(auto&[other,t]:views)if(!t->dashboard&&other!="menu"&&other!="notifications"&&other!="launcher"&&other.rfind("framely.cast.",0)!=0&&!keyboard_open)close_view(*t);}} while(overlays->PollNextOverlayEvent(v->overlay,&event,sizeof(event)))overlay_input(*v,event); - if(v->dirty&&!v->closing){ + if(!v->cast_frame.path.empty()){glXMakeContextCurrent(xdisplay,glsurface,glsurface,glcontext);if(v->cast_frame.update())v->dirty=true;} + if(v->dirty&&!v->closing&&v->pixels.size()==size_t(v->width*v->render_scale)*size_t(v->height*v->render_scale)*4){ glXMakeContextCurrent(xdisplay,glsurface,glsurface,glcontext); // Upload to the other texture; never redefine storage submitted this frame. const int slot=1-v->texture_slot;const GLuint id=v->textures[slot]; @@ -681,8 +685,9 @@ int main(int argc,char** argv){ glPixelStorei(GL_UNPACK_ROW_LENGTH,0);damage.clear(); if(mipmapped)generate_mipmap(GL_TEXTURE_2D); } + GLuint submitted=v->cast_frame.draw(id,pixel_width,pixel_height); // glFlush queues the upload; glFinish makes the full image ready for sharing. - glFinish();vr::Texture_t texture{reinterpret_cast(uintptr_t(id)),vr::TextureType_OpenGL,vr::ColorSpace_Auto}; + glFinish();vr::Texture_t texture{reinterpret_cast(uintptr_t(submitted)),vr::TextureType_OpenGL,vr::ColorSpace_Auto}; auto e=overlays->SetOverlayTexture(v->overlay,&texture);glFlush(); if(e)std::cerr<<"Texture submission failed "<texture_slot=slot;v->dirty=false; diff --git a/native/plugin_navigation.h b/native/plugin_navigation.h index cd77cfe..133aa8e 100644 --- a/native/plugin_navigation.h +++ b/native/plugin_navigation.h @@ -21,5 +21,5 @@ inline bool allow_plugin_navigation(const std::string& url, const std::string& o } inline bool manager_navigation_page(const std::string& page) { - return page=="installed"||page=="catalog"||page=="sources"||page=="notification-settings"||page=="settings"||page=="launcher-settings"||page=="updates"||page=="about"||page=="terminal"||page=="files"||page=="apk"||page=="apk-containers"||page=="apk-cleanup"||page=="apk-settings"; + return page=="casting"||page=="casting-picture"||page=="casting-receive"||page=="casting-devices"||page=="installed"||page=="catalog"||page=="sources"||page=="notification-settings"||page=="settings"||page=="launcher-settings"||page=="updates"||page=="about"||page=="terminal"||page=="files"||page=="apk"||page=="apk-containers"||page=="apk-cleanup"||page=="apk-settings"; } diff --git a/packaging/install.sh b/packaging/install.sh index 34ac5a4..3493fbd 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -13,10 +13,31 @@ steam_uid=$(id -u "$steam_user") steam_home=$(getent passwd "$steam_user" | cut -d: -f6) [[ $steam_uid -ge 1000 && $steam_home == /* && $steam_home != *' '* ]] || { echo 'Invalid Steam user/home.' >&2; exit 1; } [[ -w /etc/systemd/system && -w /etc && -w /var/lib ]] || { echo 'System configuration is not writable. Framely did not change the read-only setting; installation stopped.' >&2; exit 1; } +# A terminal opened by Framely belongs to its service cgroup. Stop requests +# would kill this installer along with the UI, before it can switch/restart. +# Journal output avoids retaining a pipe to the terminal being shut down. +installer_cgroup=$(cat /proc/self/cgroup) +if [[ $installer_cgroup =~ /framely(-session|-ui-recovery|-backend-[^/]+|-plugin-[^/]+|-hook-[^/]+)?\.service(/|$|[[:space:]]) ]]; then + worker_args=("$steam_user") + if $repair; then worker_args=(--repair "$steam_user"); fi + install_unit="framely-install-$$-$(date +%s)" + systemd-run --quiet --collect --no-block --unit="$install_unit" \ + --property=Type=exec --property=StandardInput=null \ + --property=StandardOutput=journal --property=StandardError=journal \ + /usr/bin/bash "$base/install.sh" "${worker_args[@]}" + echo "Installation continues independently. Log: journalctl -u $install_unit" + exit 0 +fi cd "$base" sha256sum --quiet -c SHA256SUMS -for binary in bin/framely; do - dependencies=$(ldd "$binary" 2>&1) +casting_binaries=() +if [[ -d lib/media ]]; then + command -v ffmpeg >/dev/null || { echo "Missing system FFmpeg for casting." >&2; exit 1; } + casting_binaries=(bin/framely-panel-grab lib/media/framely-capture lib/media/framely-receiver lib/media/uxplay lib/media/libgstframely.so lib/media/mediamtx) +fi +for binary in bin/framely "${casting_binaries[@]}"; do + dependencies=$(ldd "$binary" 2>&1 || true) + [[ -f $binary ]] || { echo "Missing binary: $binary" >&2; exit 1; } [[ $dependencies != *'not found'* ]] || { echo "$dependencies" >&2; exit 1; } done version=$(cat VERSION) @@ -41,6 +62,9 @@ for unit in framely.service framely-session.service; do previous_active[$unit]=$(systemctl is-active "$unit" || true) previous_enabled[$unit]=$(systemctl is-enabled "$unit" 2>/dev/null || true) done +if systemctl --user --machine="$steam_user@" is-active --quiet framely-ui-recovery.service 2>/dev/null; then + previous_active[framely-session.service]=active +fi if [[ -d $root && ! -L $root ]]; then # Stop writers before moving state across filesystems. Restore running services # if migration or subsequent preflight fails; the compatibility path survives. @@ -145,6 +169,8 @@ rollback_on_failure() { exit "$code" } trap rollback_on_failure ERR +# Retire the temporary user UI used to recover an interrupted installation. +systemctl --user --machine="$steam_user@" stop framely-ui-recovery.service 2>/dev/null || true systemctl stop framely-session.service framely.service 2>/dev/null || true # Clear the uninstall guard retained by older releases only for a clean reinstall. # Existing installations, repairs and incomplete plugin cleanup retain safe mode. diff --git a/src/cast/mod.rs b/src/cast/mod.rs new file mode 100644 index 0000000..9005cc8 --- /dev/null +++ b/src/cast/mod.rs @@ -0,0 +1,6 @@ +//! Built-in casting. Settings live in the core database; all media workers +//! run as the Steam session user, apart from the framebuffer export helper. +pub mod settings; +pub mod runtime; + +pub mod receiver; diff --git a/src/cast/receiver.rs b/src/cast/receiver.rs new file mode 100644 index 0000000..01fe7b0 --- /dev/null +++ b/src/cast/receiver.rs @@ -0,0 +1,541 @@ +use super::settings::Settings; +use anyhow::{bail, ensure, Context, Result}; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, VecDeque}, + fs, + io::{BufRead, BufReader, Write}, + os::unix::{fs::PermissionsExt, net::UnixDatagram, process::CommandExt}, + path::PathBuf, + process::{Child, ChildStdin, Command, Stdio}, + sync::mpsc, + time::{Duration, Instant}, +}; +struct Worker { + child: Child, + input: ChildStdin, + output: mpsc::Receiver, +} +impl Worker { + fn new(mut command: Command, log: PathBuf) -> Result { + super::runtime::bind_to_session(&mut command); + let mut child = command + .process_group(0) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::from( + fs::OpenOptions::new().create(true).append(true).open(log)?, + )) + .spawn()?; + let input = child.stdin.take().unwrap(); + let output = child.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + let Ok(line) = line else { + break; + }; + if let Ok(v) = serde_json::from_str(&line) { + if tx.send(v).is_err() { + break; + } + } + } + }); + Ok(Self { + child, + input, + output: rx, + }) + } + fn stop(&mut self) { + if self.child.try_wait().ok().flatten().is_none() { + unsafe { + libc::kill(-(self.child.id() as i32), libc::SIGTERM); + } + let end = Instant::now() + Duration::from_secs(2); + while self.child.try_wait().ok().flatten().is_none() && Instant::now() < end { + std::thread::sleep(Duration::from_millis(20)); + } + if self.child.try_wait().ok().flatten().is_none() { + unsafe { + libc::kill(-(self.child.id() as i32), libc::SIGKILL); + } + } + } + let _ = self.child.wait(); + } +} +impl Drop for Worker { + fn drop(&mut self) { + self.stop(); + } +} +pub struct Receivers { + pub directory: PathBuf, + binaries: PathBuf, + configuration: Option<(bool, bool, String)>, + socket: Option, + worker: Option, + airplay: Option, + airplay_id: String, + sequence: u64, + pub sessions: BTreeMap, + events: VecDeque, + pub error: Option, +} +impl Receivers { + pub fn new(directory: PathBuf, binaries: PathBuf) -> Self { + Self { + directory, + binaries, + configuration: None, + socket: None, + worker: None, + airplay: None, + airplay_id: String::new(), + sequence: 0, + sessions: BTreeMap::new(), + events: VecDeque::new(), + error: None, + } + } + pub fn configure(&mut self, s: &Settings) -> Result<()> { + let configuration = (s.airplay, s.dlna, s.receiver_name.clone()); + if self.configuration.as_ref() == Some(&configuration) { + return Ok(()); + } + self.stop(); + self.configuration = Some(configuration); + self.error = None; + let result = (|| { + fs::create_dir_all(&self.directory)?; + fs::set_permissions(&self.directory, fs::Permissions::from_mode(0o700))?; + let path = self.directory.join("events.sock"); + let _ = fs::remove_file(&path); + let socket = UnixDatagram::bind(path)?; + socket.set_nonblocking(true)?; + self.socket = Some(socket); + self.write_description(&s.receiver_name)?; + let executable = self.binaries.join("framely-receiver"); + ensure!(executable.is_file(), "缺少内置投屏接收程序"); + let mut cmd = Command::new(executable); + cmd.arg(&self.directory) + .arg(self.directory.join("upnp")) + .arg(if s.dlna { "1" } else { "0" }); + self.worker = Some(Worker::new(cmd, self.directory.join("receiver.log"))?); + if s.airplay { + self.start_airplay(&s.receiver_name)?; + } + Ok(()) + })(); + if let Err(e) = &result { + self.error = Some(format!("{e:#}")); + } + result + } + fn write_description(&self, name: &str) -> Result<()> { + let xml = self.directory.join("upnp"); + fs::create_dir_all(&xml)?; + let resources = self.binaries.join("upnp"); + for file in [ + "AVTransport.xml", + "RenderingControl.xml", + "ConnectionManager.xml", + ] { + fs::copy(resources.join(file), xml.join(file))?; + } + let name = name + .replace('&', "&") + .replace('<', "<") + .replace('>', ">"); + let services=["AVTransport","RenderingControl","ConnectionManager"].map(|s|format!("urn:schemas-upnp-org:service:{s}:1urn:upnp-org:serviceId:{s}/{s}.xml/{s}/control/{s}/event")).join(""); + // The advertised renderer has a persistent identity for this Steam user. + use sha2::{Digest, Sha256}; + let mut identity = fs::read("/etc/machine-id").context("无法读取设备身份")?; + identity.extend_from_slice(crate::steam::home()?.as_os_str().as_encoded_bytes()); + let hash = hex::encode(Sha256::digest(identity)); + let uuid = format!( + "{}-{}-{}-{}-{}", + &hash[..8], + &hash[8..12], + &hash[12..16], + &hash[16..20], + &hash[20..32] + ); + fs::write(xml.join("device.xml"),format!("10urn:schemas-upnp-org:device:MediaRenderer:1{name}FramelyFramelyuuid:{uuid}{services}"))?; + Ok(()) + } + fn start_airplay(&mut self, name: &str) -> Result<()> { + let executable = self.binaries.join("uxplay"); + ensure!(executable.is_file(), "缺少内置 AirPlay 接收程序"); + self.airplay_id = crate::session::cast_random_key()[..32].into(); + let mut cmd = Command::new(executable); + cmd.process_group(0) + .args([ + "-n", + name, + "-vs", + "framelyvideosink", + "-as", + "framelyaudiosink", + ]) + .env("GST_PLUGIN_PATH", &self.binaries) + .env("FRAMELY_CAST_DIR", &self.directory) + .env("FRAMELY_CAST_ID", &self.airplay_id) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::from( + fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.directory.join("airplay.log"))?, + )); + super::runtime::bind_to_session(&mut cmd); + self.airplay = Some(cmd.spawn()?); + Ok(()) + } + fn event(&mut self, mut e: Value) { + let Some(id) = e["id"].as_str().map(str::to_owned) else { + if e["event"] == "fatal" { + self.error = e["error"].as_str().map(str::to_owned); + } + return; + }; + if !id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-') || id.len() > 64 { + return; + } + if e["protocol"] == "AirPlay" && id != self.airplay_id { + return; + } + let kind = e["event"].as_str().unwrap_or(""); + if matches!(kind, "request" | "video" | "audio") { + let fresh = !self.sessions.contains_key(&id); + if fresh { + e["accepted"] = json!(false); + e["createdAt"] = json!(crate::service::now_ms()); + self.sessions.insert(id.clone(), e.clone()); + self.events + .push_back(json!({"kind":"cast.request","session":e})); + } else if matches!(kind, "video" | "audio") { + let session = self.sessions.get_mut(&id).unwrap(); + if kind == "video" { + session["width"] = e["width"].clone(); + session["height"] = e["height"].clone(); + } + if session["accepted"] == true { + self.events + .push_back(json!({"kind":"cast.window","session":session.clone()})); + } + } + } else if matches!(kind, "ended" | "error") { + if let Some(mut s) = self.sessions.remove(&id) { + s["error"] = e["error"].clone(); + self.events + .push_back(json!({"kind":"cast.ended","session":s})); + } + self.clean_session(&id); + } + } + pub fn poll(&mut self) -> Vec { + let mut values = Vec::new(); + if let Some(w) = &self.worker { + values.extend(w.output.try_iter()); + } + if let Some(socket) = &self.socket { + loop { + let mut bytes = [0; 16384]; + let Ok(n) = socket.recv(&mut bytes) else { + break; + }; + if let Ok(v) = serde_json::from_slice(&bytes[..n]) { + values.push(v); + } + } + } + let mut disconnected = false; + for e in values { + if e["protocol"] == "AirPlay" + && e["event"] == "ended" + && e["id"] == self.airplay_id + && self.sessions.contains_key(&self.airplay_id) + { + disconnected = true; + } + self.event(e); + } + if disconnected { + if let Some(mut p) = self.airplay.take() { + stop_child(&mut p); + } + if let Some(config) = &self.configuration { + let name = config.2.clone(); + if config.0 { + if let Err(e) = self.start_airplay(&name) { + self.error = Some(e.to_string()); + } + } + } + } + + let mut retained = VecDeque::new(); + while let Some(e) = self.events.pop_front() { + if e["kind"] == "cast.engine.event" { + self.event(e["data"].clone()); + } else { + retained.push_back(e); + } + } + self.events = retained; + if let Some(code) = self + .worker + .as_mut() + .and_then(|w| w.child.try_wait().ok().flatten()) + { + self.error = Some(format!("DLNA 接收程序已退出：{code}")); + self.worker = None; + self.end_protocol("DLNA"); + } + if let Some(code) = self + .airplay + .as_mut() + .and_then(|w| w.try_wait().ok().flatten()) + { + self.error = Some(format!("AirPlay 接收程序已退出：{code}")); + self.airplay = None; + self.end_protocol("AirPlay"); + } + // Pending requests expire; a delayed notification must never open an old cast. + let expired: Vec<_> = self + .sessions + .iter() + .filter(|(_, s)| { + s["accepted"] != true + && crate::service::now_ms().saturating_sub(s["createdAt"].as_u64().unwrap_or(0)) + > 60000 + }) + .map(|(id, _)| id.clone()) + .collect(); + for id in expired { + let _ = self.control(&id, "reject", json!({})); + } + self.events.drain(..).collect() + } + fn request(&mut self, mut value: Value) -> Result { + self.sequence += 1; + value["request"] = json!(self.sequence); + let sequence = self.sequence; + let w = self.worker.as_mut().context("投屏服务尚未启动")?; + writeln!(w.input, "{value}")?; + w.input.flush()?; + let deadline = Instant::now() + Duration::from_secs(8); + loop { + let reply = w + .output + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .context("投屏服务未响应")?; + if reply["request"].as_u64() == Some(sequence) { + if let Some(e) = reply["error"].as_str() { + bail!("{e}"); + } + return Ok(reply["result"].clone()); + } + self.events + .push_back(json!({"kind":"cast.engine.event","data":reply})); + } + } + pub fn devices(&mut self) -> Result { + self.request(json!({"kind":"devices"})) + } + pub fn send(&mut self, id: &str, uri: &str) -> Result { + self.request(json!({"kind":"send","id":id,"uri":uri})) + } + pub fn stop_send(&mut self, id: &str) -> Result { + self.request(json!({"kind":"send.stop","id":id})) + } + pub fn control(&mut self, id: &str, kind: &str, params: Value) -> Result { + let session = self.sessions.get(id).context("投屏请求已过期")?.clone(); + ensure!( + matches!( + kind, + "accept" | "reject" | "stop" | "pause" | "volume" | "seek" | "status" + ), + "Invalid media action" + ); + ensure!( + matches!(kind, "accept" | "reject" | "stop") || session["accepted"] == true, + "尚未接受投屏" + ); + ensure!( + !matches!(kind, "accept" | "reject") || session["accepted"] != true, + "投屏已接受" + ); + if session["protocol"] == "AirPlay" { + ensure!(id == self.airplay_id, "投屏请求已过期"); + let path = self.directory.join(format!("{id}.accept")); + match kind { + "accept" => { + fs::write(&path, [])?; + } + "pause" => { + if params["paused"].as_bool().unwrap_or(true) { + let _ = fs::remove_file(path); + } else { + fs::write(path, [])?; + } + } + "reject" | "stop" => { + let _ = fs::remove_file(path); + if let Some(mut child) = self.airplay.take() { + stop_child(&mut child); + } + self.clean_session(id); + self.sessions.remove(id); + self.events + .push_back(json!({"kind":"cast.ended","session":session})); + let name = self.configuration.as_ref().context("服务已关闭")?.2.clone(); + self.start_airplay(&name)?; + return Ok(json!(true)); + } + "volume" => { + let volume = params["volume"].as_f64().context("Invalid volume")?; + ensure!( + volume.is_finite() && (0.0..=1.0).contains(&volume), + "Invalid volume" + ); + fs::write( + self.directory.join(format!("{id}.volume")), + volume.to_string(), + )?; + } + "status" => return Ok(json!({"paused":!path.exists(),"position":0,"duration":0})), + _ => bail!("AirPlay 不支持跳转"), + }; + } else { + let mut p = params; + p["kind"] = json!(kind); + p["id"] = json!(id); + let result = self.request(p)?; + if kind == "status" { + return Ok(result); + } + } + if kind == "accept" { + self.sessions.get_mut(id).unwrap()["accepted"] = json!(true); + if session["protocol"] == "AirPlay" { + self.events + .push_back(json!({"kind":"cast.window","session":self.sessions[id]})); + } + } + if matches!(kind, "stop" | "reject") { + self.clean_session(id); + self.sessions.remove(id); + self.events + .push_back(json!({"kind":"cast.ended","session":session})); + } + Ok(json!(true)) + } + pub fn window(&self, id: &str) -> Result { + let s = self.sessions.get(id).context("投屏已经结束")?; + ensure!(s["accepted"] == true, "尚未接受投屏"); + Ok(s.clone()) + } + fn end_protocol(&mut self, protocol: &str) { + let ids: Vec<_> = self + .sessions + .iter() + .filter(|(_, s)| s["protocol"] == protocol) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + self.event(json!({"event":"ended","id":id})); + } + } + fn clean_session(&self, id: &str) { + for suffix in ["accept", "frame", "volume"] { + let _ = fs::remove_file(self.directory.join(format!("{id}.{suffix}"))); + } + } + fn stop(&mut self) { + self.worker = None; + if let Some(mut p) = self.airplay.take() { + stop_child(&mut p); + } + for (id, s) in &self.sessions { + self.clean_session(id); + self.events + .push_back(json!({"kind":"cast.ended","session":s})); + } + self.sessions.clear(); + self.socket = None; + } +} +impl Drop for Receivers { + fn drop(&mut self) { + self.stop(); + } +} + +fn stop_child(child: &mut Child) { + if child.try_wait().ok().flatten().is_some() { + return; + } + unsafe { + libc::kill(-(child.id() as i32), libc::SIGTERM); + } + let deadline = Instant::now() + Duration::from_secs(2); + while child.try_wait().ok().flatten().is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + if child.try_wait().ok().flatten().is_none() { + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + } + let _ = child.wait(); +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn consent_blocks_playback_controls_and_rejects_stale_airplay_sessions() { + let root = tempfile::tempdir().unwrap(); + let mut r = Receivers::new(root.path().to_owned(), root.path().join("bin")); + r.airplay_id = "0123".into(); + r.event(json!({"event":"video","id":"0123","protocol":"AirPlay","width":320,"height":240})); + assert!(r.window("0123").is_err()); + assert!(r.control("0123", "pause", json!({})).is_err()); + r.control("0123", "accept", json!({})).unwrap(); + assert_eq!(r.window("0123").unwrap()["width"], 320); + assert!(r.control("0123", "accept", json!({})).is_err()); + r.airplay_id = "4567".into(); + assert!(r.control("0123", "volume", json!({"volume":0.5})).is_err()); + } + #[test] + fn session_updates_preserve_consent_and_disconnect_cleans_only_its_protocol() { + let root = tempfile::tempdir().unwrap(); + let mut r = Receivers::new(root.path().to_owned(), root.path().join("bin")); + r.airplay_id = "4567".into(); + for (id, protocol) in [("0123", "DLNA"), ("4567", "AirPlay")] { + r.event(json!({"event":"request","id":id,"protocol":protocol})); + r.sessions.get_mut(id).unwrap()["accepted"] = json!(true); + for suffix in ["frame", "accept", "volume"] { + fs::write(root.path().join(format!("{id}.{suffix}")), []).unwrap(); + } + } + r.event(json!({"event":"video","id":"0123","protocol":"DLNA","width":240,"height":320})); + assert_eq!(r.window("0123").unwrap()["height"], 320); + assert_eq!(r.sessions.len(), 2); + r.end_protocol("DLNA"); + assert!(r.window("0123").is_err()); + assert!(r.window("4567").is_ok()); + for suffix in ["frame", "accept", "volume"] { + assert!(!root.path().join(format!("0123.{suffix}")).exists()); + } + assert!(r + .events + .iter() + .any(|e| e["kind"] == "cast.ended" && e["session"]["id"] == "0123")); + } +} diff --git a/src/cast/runtime.rs b/src/cast/runtime.rs new file mode 100644 index 0000000..797fced --- /dev/null +++ b/src/cast/runtime.rs @@ -0,0 +1,660 @@ +use super::settings::{Eye, Output, Settings, Source}; +use anyhow::{bail, ensure, Context, Result}; +use serde_json::{json, Value}; +use std::{ + fs, + io::Read, + os::unix::{fs::PermissionsExt, process::CommandExt}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +pub struct Casting { + pub directory: PathBuf, + pub binaries: PathBuf, + workers: Vec, + started: Option, + pub whep_port: u16, + rtsp_port: u16, + api_port: u16, + dlna_started: bool, + vp8: Option<(Child, Instant)>, + compatibility_poll: Instant, + active_settings: Option, + pub hls_port: u16, + pub stream_key: String, + error: Option, + preview_restore: Option>, +} +fn port() -> Result { + Ok(std::net::TcpListener::bind("127.0.0.1:0")? + .local_addr()? + .port()) +} +impl Casting { + pub fn new(directory: PathBuf, binaries: PathBuf) -> Self { + Self { + directory, + binaries, + workers: Vec::new(), + started: None, + whep_port: 0, + rtsp_port: 0, + api_port: 0, + dlna_started: false, + vp8: None, + compatibility_poll: Instant::now(), + active_settings: None, + hls_port: 0, + stream_key: String::new(), + error: None, + preview_restore: None, + } + } + pub fn status(&mut self) -> Value { + let exited = self + .workers + .iter_mut() + .find_map(|p| p.try_wait().ok().flatten()); + if let Some(code) = exited { + self.error = Some(format!("串流进程已退出：{code}")); + self.stop_workers(); + } + let view = fs::File::open(self.directory.join("stream.log")).ok().and_then(|file| { let mut log = String::new(); file.take(65536).read_to_string(&mut log).ok()?; Some(log) }).and_then(|log| log.lines().filter_map(|line|serde_json::from_str::(line).ok()).find(|v|v["event"]=="view")); + json!({"running":self.started.is_some(),"error":self.error,"preview":self.preview_restore.is_some(),"view":view, + "watchPath":if self.started.is_some(){Some("/cast/watch")}else{None}, + "elapsedSeconds":self.started.map(|t|t.elapsed().as_secs()), + "dominantEye":dominant_eye().ok()}) + } + fn stop_workers(&mut self) { + for child in &mut self.workers { + if child.try_wait().ok().flatten().is_some() { + continue; + } + // Each worker owns a process group, including its media children. + unsafe { + libc::kill(-(child.id() as i32), libc::SIGTERM); + } + } + let deadline = Instant::now() + Duration::from_secs(2); + for child in &mut self.workers { + while child.try_wait().ok().flatten().is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + if child.try_wait().ok().flatten().is_none() { + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + } + let _ = child.wait(); + } + self.workers.clear(); + self.dlna_started = false; + self.stop_vp8(); + self.active_settings = None; + self.started = None; + self.stream_key.clear(); + } + /// Negotiate with browsers that omit H.264 (including the bundled CEF). + /// All VP8 viewers share one on-demand compatibility encoder. + pub fn ensure_vp8(&mut self) -> Result<()> { + ensure!(self.status()["running"] == true, "请先开始头显串流"); + if self.vp8.is_some() { + return Ok(()); + } + let s = self + .active_settings + .clone() + .context("Missing capture settings")?; + let source = format!("rtsp://127.0.0.1:{}/headset", self.rtsp_port); + let target = format!("rtsp://127.0.0.1:{}/headset_vp8", self.rtsp_port); + self.spawn(Command::new("ffmpeg").args([ + "-hide_banner", + "-loglevel", + "warning", + "-nostdin", + "-rtsp_transport", + "tcp", + "-i", + &source, + "-map", + "0:v:0", + "-map", + "0:a?", + "-c:v", + "libvpx", + "-deadline", + "realtime", + "-cpu-used", + "8", + "-lag-in-frames", + "0", + "-error-resilient", + "1", + "-threads", + "2", + "-g", + &s.fps.to_string(), + "-b:v", + &format!("{}M", s.bitrate_mbps), + "-c:a", + "copy", + "-f", + "rtsp", + "-rtsp_transport", + "tcp", + &target, + ]))?; + self.vp8 = Some((self.workers.pop().unwrap(), Instant::now())); + let ready = (|| { + let end = Instant::now() + Duration::from_secs(12); + while !self.path_ready("headset_vp8") { + ensure!(Instant::now() < end, "兼容编码启动超时"); + ensure!( + self.vp8.as_mut().unwrap().0.try_wait()?.is_none(), + "兼容编码启动失败" + ); + std::thread::sleep(Duration::from_millis(50)); + } + Ok(()) + })(); + if ready.is_err() { + self.stop_vp8(); + } + ready + } + fn stop_vp8(&mut self) { + if let Some((mut child, _)) = self.vp8.take() { + if child.try_wait().ok().flatten().is_none() { + unsafe { + libc::kill(-(child.id() as i32), libc::SIGTERM); + } + let end = Instant::now() + Duration::from_secs(2); + while child.try_wait().ok().flatten().is_none() && Instant::now() < end { + std::thread::sleep(Duration::from_millis(20)); + } + if child.try_wait().ok().flatten().is_none() { + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + } + } + let _ = child.wait(); + } + } + pub fn maintain(&mut self) { + if self.compatibility_poll.elapsed() < Duration::from_secs(2) { + return; + } + self.compatibility_poll = Instant::now(); + let Some((child, started)) = &mut self.vp8 else { + return; + }; + if child.try_wait().ok().flatten().is_some() { + self.stop_vp8(); + return; + } + // Allow ICE establishment before deciding the compatibility encoder is idle. + if started.elapsed() < Duration::from_secs(10) { + return; + } + let idle = ureq::get(&format!( + "http://127.0.0.1:{}/v3/paths/get/headset_vp8", + self.api_port + )) + .timeout(Duration::from_millis(300)) + .call() + .ok() + .and_then(|r| r.into_json::().ok()) + .is_some_and(|v| v["readers"].as_array().is_some_and(Vec::is_empty)); + if idle { + self.stop_vp8(); + } + } + fn path_ready(&self, path: &str) -> bool { + ureq::get(&format!( + "http://127.0.0.1:{}/v3/paths/get/{path}", + self.api_port + )) + .timeout(Duration::from_millis(300)) + .call() + .ok() + .and_then(|r| r.into_json::().ok()) + .is_some_and(|v| v["ready"] == true) + } + pub fn ensure_dlna(&mut self) -> Result<()> { + ensure!(self.status()["running"] == true, "请先开始头显串流"); + if self.dlna_started { + return Ok(()); + } + let source = format!("rtsp://127.0.0.1:{}/headset", self.rtsp_port); + let target = format!("rtsp://127.0.0.1:{}/dlna", self.rtsp_port); + self.spawn(Command::new("ffmpeg").args([ + "-hide_banner", + "-loglevel", + "warning", + "-nostdin", + "-rtsp_transport", + "tcp", + "-i", + &source, + "-map", + "0:v:0", + "-map", + "0:a?", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-f", + "rtsp", + "-rtsp_transport", + "tcp", + &target, + ]))?; + self.dlna_started = true; + Ok(()) + } + pub fn stop(&mut self) -> Value { + self.preview_restore = None; + self.stop_workers(); + self.error = None; + json!(true) + } + pub fn start_preview(&mut self, settings: &Settings, socket: &Path) -> Result { + ensure!(self.preview_restore.is_none(), "取景预览已在运行"); + let restore = self.active_settings.clone(); + self.stop_workers(); + let mut reference = settings.clone(); + reference.source = Source::Screen; + reference.output = Output::Eye; + reference.width = 960; + reference.height = 960; + reference.fps = 30; + reference.bitrate_mbps = 4; + reference.horizontal_fov = None; + reference.center_x = 0.; + reference.center_y = 0.; + reference.system_audio = false; + reference.microphone = false; + self.start(&reference, Some(socket))?; + self.preview_restore = Some(restore); + Ok(self.status()) + } + pub fn end_preview(&mut self) -> Result> { + let restore = self.preview_restore.take().context("没有正在运行的取景预览")?; + self.stop_workers(); + self.error = None; + Ok(restore) + } + fn spawn(&mut self, cmd: &mut Command) -> Result<()> { + bind_to_session(cmd); + cmd.process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::null()); + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.directory.join("stream.log"))?; + cmd.stderr(Stdio::from(log)); + self.workers.push(cmd.spawn().context("启动串流程序失败")?); + Ok(()) + } + pub fn start(&mut self, settings: &Settings, panel_socket: Option<&Path>) -> Result { + settings.validate()?; + ensure!(self.started.is_none(), "串流已在运行"); + fs::create_dir_all(&self.directory)?; + fs::set_permissions(&self.directory, fs::Permissions::from_mode(0o700))?; + // A new session does not accumulate unbounded log history. + fs::write(self.directory.join("stream.log"), [])?; + let result = self.start_inner(settings, panel_socket); + if let Err(e) = &result { + use std::io::{Read, Seek, SeekFrom}; + let mut detail = String::new(); + if let Ok(mut log) = fs::File::open(self.directory.join("stream.log")) { + if let Ok(length) = log.metadata().map(|m| m.len()) { + let _ = log.seek(SeekFrom::Start(length.saturating_sub(2048))); + let mut bytes = Vec::new(); + let _ = log.take(2048).read_to_end(&mut bytes); + detail = String::from_utf8_lossy(&bytes).into_owned(); + } + } + self.error = Some(format!("{e:#}\n{}", detail.trim())); + self.stop_workers(); + return Err(anyhow::anyhow!(self.error.clone().unwrap())); + } + result + } + fn start_inner(&mut self, s: &Settings, panel_socket: Option<&Path>) -> Result { + let mediamtx = self.binaries.join("mediamtx"); + ensure!( + mediamtx.is_file(), + "缺少内置媒体服务，请重新安装完整 Framely 运行包" + ); + let rtsp = port()?; + self.rtsp_port = rtsp; + self.api_port = port()?; + self.whep_port = port()?; + self.hls_port = port()?; + self.stream_key = crate::session::cast_random_key(); + let configuration = format!("logLevel: warn\nrtspAddress: 127.0.0.1:{rtsp}\nrtspTransports: [tcp]\nrtmp: no\nmoq: no\nsrt: no\napi: yes\napiAddress: 127.0.0.1:{}\nmetrics: no\nplayback: no\nwebrtc: yes\nwebrtcAddress: 127.0.0.1:{}\nwebrtcLocalUDPAddress: :0\nwebrtcLocalTCPAddress: :0\nhls: yes\nhlsAddress: 127.0.0.1:{}\nhlsVariant: mpegts\nhlsSegmentCount: 3\nhlsSegmentDuration: 1s\npaths:\n headset:\n source: publisher\n dlna:\n source: publisher\n headset_vp8:\n source: publisher\n", self.api_port, self.whep_port, self.hls_port); + let config_path = self.directory.join("mediamtx.yml"); + fs::write(&config_path, configuration)?; + self.spawn(Command::new(mediamtx).arg(config_path))?; + // Wait for readiness instead of hiding startup failures behind a fixed sleep. + let deadline = Instant::now() + Duration::from_secs(5); + while std::net::TcpStream::connect(("127.0.0.1", rtsp)).is_err() { + ensure!(Instant::now() < deadline, "媒体服务启动超时"); + if let Some(code) = self.workers[0].try_wait()? { + bail!("媒体服务启动失败：{code}"); + } + std::thread::sleep(Duration::from_millis(30)); + } + let mut ffmpeg = Command::new("ffmpeg"); + ffmpeg.args(["-hide_banner", "-loglevel", "warning", "-nostdin"]); + let mut captured = None; + if s.source == Source::Screen { + let executable = self.binaries.join("framely-capture"); + ensure!(executable.is_file(), "缺少屏幕采集程序"); + let socket = panel_socket.context("缺少屏幕采集助手连接")?; + let stream = std::os::unix::net::UnixStream::connect(socket)?; + use std::os::fd::AsRawFd; + let fd = stream.as_raw_fd(); + let eye = if s.output == Output::Raw { + 0 + } else { + match s.eye { + Eye::Left => 0, + Eye::Right => 1, + Eye::SteamVR => { + dominant_eye().context("无法读取 SteamVR 主力眼，请手动选择左眼或右眼")? + } + } + }; + let mut command = Command::new(executable); + command.args([ + "--width", + &s.width.to_string(), + "--height", + &s.height.to_string(), + "--fps", + &s.fps.to_string(), + "--bitrate", + &s.bitrate_mbps.to_string(), + "--eye", + &eye.to_string(), + ]); + if s.output == Output::Raw { + command.arg("--raw"); + } else { + if let Some(fov) = s.horizontal_fov { + command.args(["--fov", &fov.to_string()]); + } + command.args([ + "--center-x", + &s.center_x.to_string(), + "--center-y", + &s.center_y.to_string(), + ]); + } + bind_to_session(&mut command); + command + .process_group(0) + .env("FRAMELY_GRAB_FD", "3") + .stdout(Stdio::piped()) + .stdin(Stdio::null()); + let log = fs::OpenOptions::new() + .append(true) + .open(self.directory.join("stream.log"))?; + command.stderr(Stdio::from(log)); + unsafe { + command.pre_exec(move || { + if fd != 3 && libc::dup2(fd, 3) < 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::fcntl(3, libc::F_SETFD, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut child = command.spawn()?; + drop(stream); + captured = child.stdout.take(); + self.workers.push(child); + ffmpeg.args([ + "-fflags", + "+genpts", + "-use_wallclock_as_timestamps", + "1", + "-framerate", + &s.fps.to_string(), + "-f", + "h264", + "-i", + "pipe:0", + ]); + } else { + ffmpeg.args(["-f", "v4l2", "-i", "/dev/video99"]); + } + let audio = s.system_audio || s.microphone; + if s.system_audio { + ffmpeg.args(["-f", "pulse", "-i", "@DEFAULT_MONITOR@"]); + } + if s.microphone { + ffmpeg.args(["-f", "pulse", "-i", "default"]); + } + ffmpeg.args(["-map", "0:v:0"]); + if s.system_audio && s.microphone { + ffmpeg.args([ + "-filter_complex", + "[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[a]", + "-map", + "[a]", + ]); + } else if audio { + ffmpeg.args(["-map", "1:a:0"]); + } + if s.source == Source::Screen { + ffmpeg.args(["-c:v", "copy"]); + } else { + ffmpeg.args(["-vf", &format!("fps={},scale={}:{}:force_original_aspect_ratio=decrease,pad={}:{}:(ow-iw)/2:(oh-ih)/2,format=yuv420p",s.fps,s.width,s.height,s.width,s.height), + "-c:v", "libx264", "-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "baseline", "-bf", "0", "-g", &s.fps.to_string(), "-b:v", &format!("{}M",s.bitrate_mbps)]); + } + if audio { + ffmpeg.args(["-c:a", "libopus", "-b:a", "96k", "-ar", "48000"]); + } else { + ffmpeg.arg("-an"); + } + ffmpeg.args([ + "-f", + "rtsp", + "-rtsp_transport", + "tcp", + &format!("rtsp://127.0.0.1:{rtsp}/headset"), + ]); + bind_to_session(&mut ffmpeg); + ffmpeg.process_group(0).stdout(Stdio::null()); + ffmpeg.stdin(captured.map(Stdio::from).unwrap_or_else(Stdio::null)); + ffmpeg.stderr(Stdio::from( + fs::OpenOptions::new() + .append(true) + .open(self.directory.join("stream.log"))?, + )); + self.workers + .push(ffmpeg.spawn().context("启动 FFmpeg 失败")?); + let deadline = Instant::now() + Duration::from_secs(12); + loop { + if self + .workers + .iter_mut() + .any(|child| child.try_wait().ok().flatten().is_some()) + { + bail!("采集或编码程序启动失败，请查看串流日志"); + } + if ureq::get(&format!( + "http://127.0.0.1:{}/v3/paths/get/headset", + self.api_port + )) + .timeout(Duration::from_millis(300)) + .call() + .ok() + .and_then(|r| r.into_json::().ok()) + .is_some_and(|v| v["ready"] == true) + { + break; + } + ensure!( + Instant::now() < deadline, + "采集未输出画面，请确认头显已唤醒" + ); + std::thread::sleep(Duration::from_millis(100)); + } + self.started = Some(Instant::now()); + self.active_settings = Some(s.clone()); + self.error = None; + Ok(self.status()) + } +} +impl Drop for Casting { + fn drop(&mut self) { + self.stop_workers(); + } +} + +pub fn dominant_eye() -> Result { + let home = crate::steam::home()?; + let custom = std::env::var_os("XDG_CONFIG_HOME") + .map(PathBuf::from) + .unwrap_or_else(|| home.join(".config")); + let files = [ + custom.join("openvr/config/steamvr.vrsettings"), + home.join(".local/share/Steam/config/steamvr.vrsettings"), + PathBuf::from("/opt/steamvr/resources/settings/default.vrsettings"), + ]; + for path in files { + let Ok(bytes) = fs::read(path) else { + continue; + }; + let Ok(value) = serde_json::from_slice::(&bytes) else { + continue; + }; + if let Some(eye) = value["steamvr"]["dominantEye"].as_u64().filter(|v| *v <= 1) { + return Ok(eye as usize); + } + } + bail!("未找到 SteamVR 主力眼设置") +} + +pub fn binary_directory(assets: &Path) -> PathBuf { + let root = assets + .parent() + .and_then(Path::parent) + .unwrap_or(Path::new(".")); + let installed = root.join("lib/media"); + if installed.is_dir() { + installed + } else { + root.join("media/bin") + } +} + +pub fn bind_to_session(command: &mut Command) { + // Linux PDEATHSIG follows the spawning thread, not the session process. + // HTTP request threads end immediately after returning a response. Worker + // cleanup belongs to the owning runtime and the session's systemd cgroup. + command.process_group(0); +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn media_worker_survives_the_http_request_thread() { + let mut child = std::thread::spawn(|| { + let mut command = Command::new("sleep"); + command.arg("30"); + bind_to_session(&mut command); + command.spawn().unwrap() + }).join().unwrap(); + std::thread::sleep(Duration::from_millis(100)); + let exited = child.try_wait().unwrap(); + let _ = child.kill(); + let _ = child.wait(); + assert!(exited.is_none(), "media ended with its request thread: {exited:?}"); + } + #[test] + #[ignore = "requires Steam Frame, FFmpeg and the packaged media workers"] + fn steamvr_stream_has_requested_output_and_stops_all_workers() { + let directory = tempfile::tempdir().unwrap(); + let binaries = PathBuf::from(std::env::var("FRAMELY_MEDIA_BIN").unwrap()); + let mut cast = Casting::new(directory.path().to_owned(), binaries); + let settings = Settings { + source: Source::SteamVR, + width: 640, + height: 360, + fps: 30, + system_audio: true, + ..Settings::default() + }; + cast.start(&settings, None).unwrap_or_else(|e| { + panic!( + "{e:#}\n{}", + fs::read_to_string(directory.path().join("stream.log")).unwrap_or_default() + ) + }); + let address = format!("rtsp://127.0.0.1:{}/headset", cast.rtsp_port); + let output = Command::new("ffprobe") + .args([ + "-v", + "error", + "-rtsp_transport", + "tcp", + "-show_entries", + "stream=codec_name,width,height,r_frame_rate", + "-of", + "json", + &address, + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let streams: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(streams["streams"][0]["width"], 640); + assert_eq!(streams["streams"][0]["height"], 360); + assert_eq!(streams["streams"][0]["codec_name"], "h264"); + assert!(streams["streams"] + .as_array() + .unwrap() + .iter() + .any(|v| v["codec_name"] == "opus")); + cast.ensure_dlna().unwrap(); + let end = Instant::now() + Duration::from_secs(10); + while ureq::get(&format!( + "http://127.0.0.1:{}/dlna/index.m3u8", + cast.hls_port + )) + .timeout(Duration::from_millis(500)) + .call() + .is_err() + { + assert!( + Instant::now() < end, + "{}", + fs::read_to_string(directory.path().join("stream.log")).unwrap_or_default() + ); + } + let rtsp = cast.rtsp_port; + cast.stop(); + assert!(std::net::TcpStream::connect(("127.0.0.1", rtsp)).is_err()); + } +} diff --git a/src/cast/settings.rs b/src/cast/settings.rs new file mode 100644 index 0000000..6d1800e --- /dev/null +++ b/src/cast/settings.rs @@ -0,0 +1,124 @@ +use anyhow::{ensure, Result}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum Source { + #[default] + Screen, + SteamVR, +} +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum Output { + #[default] + Eye, + Raw, +} +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum Eye { + #[default] + SteamVR, + Left, + Right, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields, default)] +pub struct Settings { + pub source: Source, + pub output: Output, + pub eye: Eye, + pub width: u32, + pub height: u32, + pub fps: u32, + pub bitrate_mbps: u32, + /// None selects the widest fully covered field of view for this aspect. + pub horizontal_fov: Option, + pub center_x: f64, + pub center_y: f64, + pub system_audio: bool, + pub microphone: bool, + pub airplay: bool, + pub dlna: bool, + pub receiver_name: String, +} +impl Default for Settings { + fn default() -> Self { + Self { + source: Source::Screen, + output: Output::Eye, + eye: Eye::SteamVR, + width: 1920, + height: 1080, + fps: 30, + bitrate_mbps: 8, + horizontal_fov: None, + center_x: 0., + center_y: 0., + system_audio: true, + microphone: false, + airplay: false, + dlna: false, + receiver_name: "Framely".into(), + } + } +} +impl Settings { + pub fn validate(&self) -> Result<()> { + ensure!( + (160..=4096).contains(&self.width) + && self.width % 2 == 0 + && (160..=4096).contains(&self.height) + && self.height % 2 == 0 + && u64::from(self.width) * u64::from(self.height) <= 8_847_360, + "输出宽高必须是 160–4096 之间的偶数，且不超过 8847360 像素" + ); + ensure!((1..=120).contains(&self.fps), "输出帧率必须是 1–120 fps"); + ensure!( + (1..=80).contains(&self.bitrate_mbps), + "视频码率必须是 1–80 Mbps" + ); + ensure!( + self.horizontal_fov + .is_none_or(|v| v.is_finite() && (10.0..=150.0).contains(&v)), + "水平视野必须是 10–150 度" + ); + ensure!( + [self.center_x, self.center_y] + .into_iter() + .all(|v| v.is_finite() && (-60.0..=60.0).contains(&v)), + "取景中心必须在 -60–60 度之间" + ); + ensure!( + !self.receiver_name.trim().is_empty() + && self.receiver_name.len() <= 80 + && !self.receiver_name.chars().any(char::is_control), + "接收名称必须是 1–80 字节且不能包含控制字符" + ); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn settings_reject_unsafe_media_sizes_and_invalid_ranges() { + let mut s = Settings::default(); + s.validate().unwrap(); + s.width = 4096; + s.height = 4096; + assert!(s.validate().is_err()); + s = Settings::default(); + s.horizontal_fov = Some(f64::NAN); + assert!(s.validate().is_err()); + s = Settings::default(); + s.receiver_name = "Framely\nInjected".into(); + assert!(s.validate().is_err()); + assert!( + serde_json::from_value::(serde_json::json!({"source":"automatic"})).is_err() + ); + } +} diff --git a/src/main.rs b/src/main.rs index a3e409e..3e48769 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod apk; mod apk_metadata; mod auth; +mod cast; mod desktop; mod diagnostics; mod file_browser; diff --git a/src/model.rs b/src/model.rs index 0780074..78701d9 100644 --- a/src/model.rs +++ b/src/model.rs @@ -814,6 +814,8 @@ pub struct AgreementAcceptance { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Database { + #[serde(default)] + pub casting: crate::cast::settings::Settings, #[serde(default = "default_language")] pub language: String, #[serde(default)] diff --git a/src/service.rs b/src/service.rs index 40bc231..2d7bc83 100644 --- a/src/service.rs +++ b/src/service.rs @@ -101,6 +101,7 @@ impl Service { }; db.update_check.validate()?; db.launcher.validate()?; + db.casting.validate()?; for p in db.plugins.values_mut() { if let Some(e) = &p.manifest.engines { if let Err(error) = e.check() { @@ -1080,6 +1081,48 @@ impl Service { self.save()?; Ok(json!(true)) } + "cast.settings.save" => { + let settings: crate::cast::settings::Settings = serde_json::from_value(p)?; + settings.validate()?; + self.db.casting = settings; + self.save()?; + Ok(json!(true)) + } + "cast.panel.start" => { + let executable = self.root.join("current/bin/framely-panel-grab"); + ensure!(executable.is_file(), "缺少内置屏幕采集助手"); + let directory = self.root.join("cast-runtime"); + fs::create_dir_all(&directory)?; + fs::set_permissions(&directory, fs::Permissions::from_mode(0o755))?; + let path = directory.join(format!("panel-{}.sock", &crate::session::cast_random_key()[..16])); + let listener = UnixListener::bind(&path)?; + fs::set_permissions(&path, fs::Permissions::from_mode(0o600))?; + let name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes())?; + ensure!(unsafe { libc::chown(name.as_ptr(), self.manager, u32::MAX) } == 0, "无法设置采集助手连接身份"); + listener.set_nonblocking(true)?; + let manager = self.manager; + let cleanup = path.clone(); + std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + match listener.accept() { + Ok((stream, _)) => { + let mut credentials: libc::ucred = unsafe { std::mem::zeroed() }; + let mut length = std::mem::size_of::() as libc::socklen_t; + let valid = unsafe { libc::getsockopt(stream.as_raw_fd(), libc::SOL_SOCKET, libc::SO_PEERCRED, (&mut credentials as *mut libc::ucred).cast(), &mut length) } == 0 && credentials.uid == manager; + if !valid { continue; } + let _ = fs::remove_file(&cleanup); + let _ = Command::new(&executable).stdin(std::process::Stdio::from(std::os::fd::OwnedFd::from(stream))).stdout(std::process::Stdio::null()).status(); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => std::thread::sleep(Duration::from_millis(20)), + Err(_) => break, + } + } + let _ = fs::remove_file(cleanup); + }); + Ok(json!({"socket":path})) + } "launcher.order.save" => { let order: Vec = serde_json::from_value(p["order"].clone())?; ensure!(order.len() <= 4096, "Too many launcher entries"); @@ -1345,6 +1388,14 @@ impl Service { } Ok(json!(true)) } + "cast.notification" => { + ensure!(id.is_empty(), "Core notification only"); + let notification: Notification=serde_json::from_value(p["notification"].clone())?; + notification.validate()?; + ensure!(notification.id.starts_with("cast."), "Invalid cast notification"); + self.insert_notification("", "Framely", notification, None)?; + Ok(json!(true)) + }, "notification.send" => self.notify(&id, p["notification"].clone()), "notification.remove" | "notification.dismiss" => { if !id.is_empty() { @@ -1752,7 +1803,9 @@ impl Service { .iter() .find(|a| a.id == action) .context("Action not declared")?; - let result = if let Some(update) = &entry.update { + let result = if id.is_empty() && key.starts_with("cast.") { + json!(true) + } else if let Some(update) = &entry.update { ensure!( self.db.update_source.as_ref() == Some(&update.source) && self.db.update_channel == update.channel, diff --git a/src/session.rs b/src/session.rs index 861371a..3e66678 100644 --- a/src/session.rs +++ b/src/session.rs @@ -35,6 +35,8 @@ struct Agent { terminals: crate::terminal::Terminals, files: Arc, catalog_cache: Arc>>, + casting: Mutex, + receivers: Mutex, } #[derive(Default)] struct EventLog { @@ -63,6 +65,7 @@ fn random_key() -> String { rand::rngs::OsRng.fill_bytes(&mut b); hex::encode(b) } +pub(crate) fn cast_random_key() -> String { random_key() } fn push(q: &Mutex>, v: Value) { let mut q = q.lock().unwrap(); if q.len() >= 256 { @@ -130,6 +133,16 @@ impl Agent { } else { self.origin.clone() }; + if let Some(resource)=path.strip_prefix("/cast/media/") { + ensure!(r.method()==&Method::Get||r.method()==&Method::Head,"Invalid media request"); + let (key,file)=resource.split_once('/').context("Invalid media resource")?; + let port={let mut cast=self.casting.lock().unwrap();ensure!(!cast.stream_key.is_empty()&&key==cast.stream_key&&cast.status()["running"]==true,"Media session expired");cast.hls_port}; + ensure!(file.len()<128&&file.bytes().all(|b|b.is_ascii_alphanumeric()||b==b'.'||b==b'_'||b==b'-')&&(file.ends_with(".m3u8")||file.ends_with(".ts")),"Invalid media resource"); + let target=format!("http://127.0.0.1:{port}/dlna/{file}"); + let response=match ureq::AgentBuilder::new().timeout(Duration::from_secs(15)).redirects(0).build().request(r.method().as_str(),&target).call(){Ok(v)=>v,Err(ureq::Error::Status(_,v))=>v,Err(e)=>return Err(e.into())}; + let code=response.status();let mime=response.header("Content-Type").unwrap_or("application/octet-stream").to_owned(); + r.respond(Response::new(StatusCode(code),vec![header("Content-Type",&mime),header("Cache-Control","no-store"),header("X-Content-Type-Options","nosniff")],response.into_reader(),None,None))?;return Ok(()); + } if remote && (path.starts_with("/boot/") || path.starts_with("/host/")) { r.respond(Response::empty(StatusCode(403)))?; return Ok(()); @@ -224,11 +237,11 @@ impl Agent { if configured && !protected && r.method() == &Method::Get - && (path == "/" || path == "/manager") + && (path == "/" || path == "/manager" || path == "/cast/watch") { r.respond( Response::empty(StatusCode(302)) - .with_header(header("Location", "/manager")) + .with_header(header("Location", if path == "/cast/watch" { "/cast/watch" } else { "/manager" })) .with_header(header( "Set-Cookie", &format!( @@ -284,6 +297,7 @@ impl Agent { let mut body = String::new(); r.as_reader().take(8193).read_to_string(&mut body)?; ensure!(body.len() <= 8192, "Login request too large"); + let destination = url::form_urlencoded::parse(body.as_bytes()).any(|(k,v)| k == "next" && v == "/cast/watch"); let code = url::form_urlencoded::parse(body.as_bytes()) .find(|(k, _)| k == "password") .map(|(_, v)| v.into_owned()); @@ -310,7 +324,7 @@ impl Agent { if verified && key_before == *self.network_key.lock().unwrap() { r.respond( Response::empty(StatusCode(302)) - .with_header(header("Location", "/manager")) + .with_header(header("Location", if destination { "/cast/watch" } else { "/manager" })) .with_header(header( "Set-Cookie", &format!( @@ -328,8 +342,8 @@ impl Agent { } return Ok(()); } - if r.method() == &Method::Get && (path == "/" || path == "/manager") { - r.respond(Response::from_string(if configured { NETWORK_LOGIN } else { NETWORK_SETUP }).with_header(header("Content-Type","text/html; charset=utf-8")).with_header(header("Content-Security-Policy","default-src 'none'; img-src 'self'; style-src 'unsafe-inline'; form-action 'self'; frame-ancestors 'none'")))?; + if r.method() == &Method::Get && (path == "/" || path == "/manager" || path == "/cast/watch") { + r.respond(Response::from_string(if configured { if path == "/cast/watch" { NETWORK_LOGIN.replace("
", "") } else { NETWORK_LOGIN.into() } } else { NETWORK_SETUP.into() }).with_header(header("Content-Type","text/html; charset=utf-8")).with_header(header("Content-Security-Policy","default-src 'none'; img-src 'self'; style-src 'unsafe-inline'; form-action 'self'; frame-ancestors 'none'")))?; } else { r.respond( Response::empty(StatusCode(401)) @@ -351,7 +365,10 @@ impl Agent { r.as_reader().take(32769).read_to_end(&mut bytes)?; ensure!(bytes.len() <= 32768, "Visibility report too large"); let views: Value = serde_json::from_slice(&bytes)?; - if views.is_object() { + if views["views"].is_object() { + self.core("host.ui.visibility",views["views"].clone())?; + if let Some(ids)=views["closedCasts"].as_array(){ensure!(ids.len()<=32,"Too many closed casts");for id in ids{if let Some(id)=id.as_str(){let _=self.receivers.lock().unwrap().control(id,"stop",json!({}));}}} + } else if views.is_object() { self.core("host.ui.visibility", views)?; } } @@ -772,6 +789,49 @@ impl Agent { }, ); } + if path == "/cast/whep" || path.starts_with("/cast/whep/") { + ensure!(matches!(r.method(), Method::Post | Method::Patch | Method::Delete | Method::Options), "Invalid WHEP method"); + let origin = r.headers().iter().find(|h| h.field.equiv("Origin")).map(|h|h.value.as_str()); + ensure!(origin == Some(expected_origin.as_str()), "Cross-origin playback request denied"); + if r.method()==&Method::Options {r.respond(Response::empty(StatusCode(204)).with_header(header("Accept-Post","application/sdp")))?;return Ok(());} + let suffix = path.strip_prefix("/cast/whep").unwrap(); + let mut body = Vec::new(); + r.as_reader().take(65537).read_to_end(&mut body)?; + ensure!(body.len() <= 65536, "SDP too large"); + let (codec,resource)=if suffix.is_empty() { + ensure!(r.method()==&Method::Post||r.method()==&Method::Options,"Missing playback session"); + let offer=std::str::from_utf8(&body)?; + let supports=|codec:&str|offer.lines().any(|line|line.starts_with("a=rtpmap:")&&line.split_whitespace().nth(1).is_some_and(|v|v.eq_ignore_ascii_case(codec))); + if supports("H264/90000") {("h264","")} else {ensure!(supports("VP8/90000"),"浏览器没有可用的视频编码");("vp8","")} + } else { + let (codec,resource)=suffix.strip_prefix('/').and_then(|s|s.split_once('/')).context("Invalid playback session")?; + ensure!(matches!(codec,"h264"|"vp8")&&resource.len()==36&&resource.bytes().all(|b|b.is_ascii_hexdigit()||b==b'-'),"Invalid playback session"); + (codec,resource) + }; + let port = { let mut cast=self.casting.lock().unwrap();ensure!(cast.status()["running"]==true,"串流尚未开始");if codec=="vp8"&&suffix.is_empty(){cast.ensure_vp8()?;}cast.whep_port }; + let media_path=if codec=="vp8"{"headset_vp8"}else{"headset"}; + let target=format!("http://127.0.0.1:{port}/{media_path}/whep{}",if resource.is_empty(){String::new()}else{format!("/{resource}")}); + let mut request = ureq::AgentBuilder::new().redirects(0).timeout(Duration::from_secs(10)).build().request(r.method().as_str(), &target); + for name in ["Content-Type", "If-Match"] { + if let Some(value) = r.headers().iter().find(|h| h.field.equiv(name)) { request = request.set(name, value.value.as_str()); } + } + let response = match request.send_bytes(&body) { Ok(v) => v, Err(ureq::Error::Status(_,v)) => v, Err(e) => return Err(e.into()) }; + let code = response.status(); + let mut headers = vec![header("Cache-Control", "no-store")]; + for name in ["Content-Type", "ETag", "Accept-Patch"] { + if let Some(value) = response.header(name) { headers.push(header(name,value)); } + } + if let Some(location) = response.header("Location") { + let parsed = url::Url::parse(&target)?.join(location)?; + let prefix=format!("/{media_path}/whep/"); + ensure!(parsed.path().starts_with(&prefix), "Invalid media session location"); + headers.push(header("Location", &format!("/cast/whep/{codec}/{}",parsed.path().strip_prefix(&prefix).unwrap()))); + } + let mut bytes = Vec::new(); response.into_reader().take(65537).read_to_end(&mut bytes)?; + ensure!(bytes.len() <= 65536, "Media response too large"); + r.respond(Response::new(StatusCode(code), headers, std::io::Cursor::new(bytes), None, None))?; + return Ok(()); + } if path == "/api" && r.method() == &Method::Post { let origin = r .headers() @@ -803,6 +863,8 @@ impl Agent { return send_json(r, v); } if path == "/" + || path == "/cast/watch" + || path.starts_with("/cast/window/") || path == "/manager" || path == "/notifications" || path == "/launcher" @@ -827,6 +889,26 @@ impl Agent { r.respond(Response::from_string("Not found").with_status_code(404))?; Ok(()) } + fn maintain_cast(&self) { + self.casting.lock().unwrap().maintain(); + let events=self.receivers.lock().unwrap().poll(); + for event in events { + let session=&event["session"]; + let Some(id)=session["id"].as_str() else {continue;}; + match event["kind"].as_str().unwrap_or("") { + "cast.request" => {let _=self.core("cast.notification",json!({"notification":{"id":format!("cast.{id}"),"title":"收到投屏","body":format!("{} 请求投屏，是否打开？",session["protocol"].as_str().unwrap_or("设备")),"durationMs":60000,"inbox":true,"actions":[{"id":"open","label":"打开投屏"},{"id":"reject","label":"拒绝"}]}}));}, + "cast.window" => { + let _=self.core("notification.remove",json!({"plugin":"","id":format!("cast.{id}")})); + let w=session["width"].as_u64().unwrap_or(0);let h=session["height"].as_u64().unwrap_or(0); + let width=if w>0 { w.min(1920) } else {640};let height=if w>0&&h>0 {width*h/w}else{240}; + let frame=self.receivers.lock().unwrap().directory.join(format!("{id}.frame")); + push(&self.commands,json!({"kind":"cast.window","id":id,"width":width,"height":height,"frame":frame})); + }, + "cast.ended" => {push(&self.commands,json!({"kind":"cast.close","id":id}));let _=self.core("notification.remove",json!({"plugin":"","id":format!("cast.{id}")}));}, + _=>{}, + } + } + } fn api(&self, v: Value) -> Result { let method = v["method"].as_str().context("Missing method")?; let p = v["params"].clone(); @@ -853,6 +935,76 @@ impl Agent { ); } match method { + "cast.status" => { + let mut value = self.casting.lock().unwrap().status(); + let receivers = self.receivers.lock().unwrap(); + value["sessions"] = json!(receivers.sessions.values().collect::>()); + value["receiverError"] = json!(receivers.error); + let network=self.network_status.lock().unwrap(); + value["watchAddresses"]=if network["enabled"]==true{json!(network_ipv4_addresses().into_iter().map(|ip|format!("http://{ip}:{}/cast/watch",network["port"])).collect::>())}else{json!([])}; + Ok(value) + }, + "cast.settings.save" => { + let settings: crate::cast::settings::Settings = serde_json::from_value(p.clone())?; + settings.validate()?; + let result = self.core(method,p)?; + self.receivers.lock().unwrap().configure(&settings)?; + Ok(result) + }, + "cast.devices" => self.receivers.lock().unwrap().devices(), + "cast.send" => { + let network=self.network_status.lock().unwrap().clone();ensure!(network["enabled"]==true,"请先开启网络管理面板，以便设备获取投屏画面"); + let id=p["id"].as_str().context("Missing device")?; + let devices=self.receivers.lock().unwrap().devices()?; + let host=devices.as_array().into_iter().flatten().find(|device|device["id"]==id).and_then(|d|d["localAddress"].as_str()).context("设备已离线，请重新查找")?.to_owned(); + ensure!(host.parse::().is_ok(),"Invalid device interface"); + let mut cast=self.casting.lock().unwrap();cast.ensure_dlna()?; + let uri=format!("http://{host}:{}/cast/media/{}/index.m3u8",network["port"],cast.stream_key); + self.receivers.lock().unwrap().send(p["id"].as_str().context("Missing device")?,&uri) + }, + "cast.send.stop" => self.receivers.lock().unwrap().stop_send(p["id"].as_str().context("Missing device")?), + "cast.window" => self.receivers.lock().unwrap().window(p["id"].as_str().context("Missing session")?), + "cast.control" => self.receivers.lock().unwrap().control(p["id"].as_str().context("Missing session")?,p["action"].as_str().context("Missing action")?,p.clone()), + "notification.action" if p["plugin"] == "" && p["id"].as_str().is_some_and(|id|id.starts_with("cast.")) => { + let id=p["id"].as_str().unwrap().strip_prefix("cast.").unwrap(); + let notices=self.core("status",json!({}))?; + ensure!(notices["notifications"].as_array().into_iter().flatten().chain(notices["inbox"].as_array().into_iter().flatten()).any(|n|n["plugin"]==""&&n["notification"]["id"]==p["id"]&&n["createdAt"]==p["createdAt"]),"投屏通知已过期或已改变"); + let action=p["action"].as_str().context("Missing action")?; + self.receivers.lock().unwrap().control(id,if action=="open"{"accept"}else if action=="reject"{"reject"}else{anyhow::bail!("Invalid cast action")},json!({}))?; + self.core(method,p) + }, + "cast.preview.start" => { + let settings: crate::cast::settings::Settings = serde_json::from_value(p.clone())?; + settings.validate()?; + ensure!(settings.source == crate::cast::settings::Source::Screen && settings.output == crate::cast::settings::Output::Eye, "当前采集方式不支持取景调整"); + let response = self.core("cast.panel.start", json!({}))?; + let socket = PathBuf::from(response["socket"].as_str().context("Missing panel socket")?); + self.casting.lock().unwrap().start_preview(&settings, &socket) + }, + "cast.preview.end" => { + let mut cast = self.casting.lock().unwrap(); + if let Some(mut settings) = cast.end_preview()? { + if p["apply"] == true { + settings = serde_json::from_value(self.core("status",json!({}))?["database"]["casting"].clone())?; + } + let panel = if settings.source == crate::cast::settings::Source::Screen { + let response = self.core("cast.panel.start",json!({}))?; + Some(PathBuf::from(response["socket"].as_str().context("Missing panel socket")?)) + } else { None }; + cast.start(&settings, panel.as_deref())?; + } + Ok(cast.status()) + }, + "cast.start" => { + let status = self.core("status", json!({}))?; + let settings: crate::cast::settings::Settings = serde_json::from_value(status["database"]["casting"].clone())?; + let panel = if settings.source == crate::cast::settings::Source::Screen { + let response = self.core("cast.panel.start", json!({}))?; + Some(PathBuf::from(response["socket"].as_str().context("Missing panel socket")?)) + } else { None }; + self.casting.lock().unwrap().start(&settings, panel.as_deref()) + } + "cast.stop" => Ok(self.casting.lock().unwrap().stop()), "launcher.search.index" => { let names: Vec = serde_json::from_value(p["names"].clone())?; ensure!( @@ -1160,6 +1312,10 @@ impl Agent { | "launcher-settings" | "apk" | "apk-settings" + | "casting" + | "casting-picture" + | "casting-receive" + | "casting-devices" ) ) { @@ -1699,6 +1855,8 @@ pub fn serve( use std::os::unix::fs::PermissionsExt; fs::set_permissions(&runtime, fs::Permissions::from_mode(0o700))?; let agent = Arc::new(Agent { + casting: Mutex::new(crate::cast::runtime::Casting::new(runtime.join("cast"), crate::cast::runtime::binary_directory(&assets))), + receivers: Mutex::new(crate::cast::receiver::Receivers::new(runtime.join("receive"), crate::cast::runtime::binary_directory(&assets))), socket, state, assets, @@ -1716,6 +1874,14 @@ pub fn serve( files: Arc::default(), catalog_cache: Arc::default(), }); + let cast_agent=agent.clone(); + std::thread::spawn(move || { let mut configured=std::time::Instant::now()-Duration::from_secs(10);loop { + if configured.elapsed()>Duration::from_secs(2) { + if let Ok(status)=cast_agent.core("status",json!({})) {if status["agreement"]["accepted"]==true {if let Ok(settings)=serde_json::from_value(status["database"]["casting"].clone()){let _=cast_agent.receivers.lock().unwrap().configure(&settings);}}} + configured=std::time::Instant::now(); + } + cast_agent.maintain_cast();std::thread::sleep(Duration::from_millis(100)); + }}); let file_agent = agent.clone(); std::thread::spawn(move || loop { if let Ok(home) = crate::steam::home() { @@ -1831,31 +1997,42 @@ pub fn serve( } fn network_ipv4_addresses() -> Vec { - let mut addresses = std::collections::BTreeSet::new(); + let mut addresses = Vec::new(); unsafe { let mut first: *mut libc::ifaddrs = std::ptr::null_mut(); - if libc::getifaddrs(&mut first) != 0 { - return Vec::new(); - } + if libc::getifaddrs(&mut first) != 0 { return Vec::new(); } let mut current = first; while !current.is_null() { let interface = &*current; + // LAN addresses belong to broadcast interfaces, not VPN tunnels or loopback. if !interface.ifa_addr.is_null() && interface.ifa_flags & libc::IFF_UP as u32 != 0 - && interface.ifa_flags & libc::IFF_LOOPBACK as u32 == 0 + && interface.ifa_flags & libc::IFF_RUNNING as u32 != 0 + && interface.ifa_flags & libc::IFF_BROADCAST as u32 != 0 + && interface.ifa_flags & (libc::IFF_LOOPBACK | libc::IFF_POINTOPOINT) as u32 == 0 && (*interface.ifa_addr).sa_family as i32 == libc::AF_INET { let socket = &*(interface.ifa_addr as *const libc::sockaddr_in); let ip = std::net::Ipv4Addr::from(socket.sin_addr.s_addr.to_ne_bytes()); - if !ip.is_loopback() && !ip.is_unspecified() { - addresses.insert(ip.to_string()); + if !ip.is_loopback() && !ip.is_unspecified() && !interface.ifa_name.is_null() { + let name=std::ffi::CStr::from_ptr(interface.ifa_name).to_string_lossy().into_owned(); + addresses.push((name,ip.to_string())); } } current = interface.ifa_next; } libc::freeifaddrs(first); } - addresses.into_iter().collect() + // The main routing table identifies the Wi-Fi/Ethernet uplink even when a VPN + // policy routes ordinary connections through a tunnel. + let route=std::fs::read_to_string("/proc/net/route").unwrap_or_default(); + let preferred=route.lines().skip(1).filter_map(|line| { + let fields:Vec<_>=line.split_whitespace().collect(); + (fields.len()>6 && fields[1]=="00000000" && u32::from_str_radix(fields[3],16).unwrap_or(0)&1!=0) + .then(||(fields[6].parse::().unwrap_or(u32::MAX),fields[0])) + }).min_by_key(|(metric,_)|*metric).map(|(_,name)|name); + addresses.sort_by_key(|(name,ip)|(Some(name.as_str())!=preferred, name.clone(),ip.clone())); + addresses.into_iter().map(|(_,ip)|ip).collect() } const NETWORK_SETUP: &str = r#"Framely · 设置访问密码
Framely

设置访问密码

首次访问网络管理面板，请设置至少 8 个字符的密码。保存后使用该密码登录。

"#; @@ -1932,6 +2109,46 @@ fn wait_for_steam_session() { } #[cfg(test)] mod tests { + #[test] + #[ignore = "manual browser and Steam Frame integration fixture"] + fn casting_browser_fixture() { + let root=tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink(std::env::var("FRAMELY_MEDIA_BIN").unwrap(),root.path().join("media")).unwrap(); + let port=std::env::var("FRAMELY_CAST_WEB_PORT").unwrap().parse::().unwrap(); + let server=Server::http(("0.0.0.0",port)).unwrap();let agent=agent(&server,root.path()); + let mut core=crate::tests::accepted_service(&agent.state,1000).unwrap(); + core.handle("network.save",json!({"enabled":true,"port":port,"passwordEnabled":false,"password":"fixture password"})).unwrap(); + core.handle("cast.settings.save",json!({"source":std::env::var("FRAMELY_CAST_TEST_SOURCE").unwrap_or_else(|_|"steamVR".into()),"width":640,"height":360,"fps":30,"systemAudio":true,"dlna":true})).unwrap(); + let socket=UnixListener::bind(&agent.socket).unwrap();let running=Arc::new(AtomicBool::new(true)); + socket.set_nonblocking(true).unwrap();let alive=running.clone(); + let worker=std::thread::spawn(move||{while alive.load(Ordering::Relaxed){if let Ok((mut stream,_))=socket.accept(){let request=ipc::read(&mut stream).unwrap();let result=if request["method"]=="cast.panel.start" {ipc::call(Path::new("/run/framely/control.sock"),"cast.panel.start",json!({}))}else{core.handle(request["method"].as_str().unwrap(),request["params"].clone())};let response=ipc::response(result);let _=ipc::write(&mut stream,&response);}else{std::thread::sleep(Duration::from_millis(5));}}}); + *agent.network_status.lock().unwrap()=json!({"enabled":true,"port":port}); + let settings:crate::cast::settings::Settings=serde_json::from_value(agent.core("status",json!({})).unwrap()["database"]["casting"].clone()).unwrap(); + agent.receivers.lock().unwrap().configure(&settings).unwrap(); + let origin=agent.origin.clone();let cookie=format!("framely-network={}",agent.network_key.lock().unwrap()); + let start=std::thread::spawn(move||ureq::post(&format!("{origin}/api")).set("Origin",&origin).set("Cookie",&cookie).send_json(json!({"method":"cast.start","params":{}})).unwrap().into_json::().unwrap()); + while !start.is_finished() {if let Ok(Some(request))=server.recv_timeout(Duration::from_millis(50)){let a=agent.clone();std::thread::spawn(move||{a.handle_http(request,true).unwrap();});}} + let response=start.join().unwrap();assert!(response.get("error").is_none(),"HTTP stream startup failed: {response}"); + std::thread::sleep(Duration::from_millis(200));assert_eq!(agent.casting.lock().unwrap().status()["running"],true); + println!("Fixture viewer: http://{}:{port}/cast/watch",network_ipv4_addresses().first().unwrap()); + let probe=std::env::var("FRAMELY_CAST_BROWSER_PROBE").ok(); + let mut browser=probe.map(|executable|Command::new(executable) + .arg(format!("http://127.0.0.1:{port}/cast/watch")) + .arg(root.path().join("browser")) + .args(["--no-sandbox","--disable-gpu","--change-stack-guard-on-fork=disable"]) + .spawn().unwrap()); + let end=std::time::Instant::now()+Duration::from_secs(if browser.is_some(){60}else{300}); + let mut result=None; + while std::time::Instant::now() Arc { Arc::new(Agent { + casting: Mutex::new(crate::cast::runtime::Casting::new(root.join("cast"), root.join("media"))), + receivers: Mutex::new(crate::cast::receiver::Receivers::new(root.join("receive"), root.join("media"))), socket: root.join("control.sock"), state: root.join("state"), assets: std::env::var_os("FRAMELY_TEST_ASSETS") diff --git a/src/visibility.rs b/src/visibility.rs index 418266f..a05846c 100644 --- a/src/visibility.rs +++ b/src/visibility.rs @@ -36,7 +36,8 @@ impl Tracker { && (k == "menu" || k == "launcher" || k == "framely.manager" - || k.starts_with("framely.window."))), + || k.starts_with("framely.window.") + || k.starts_with("framely.cast."))), "Invalid native visibility views" ); self.seen = Some(now); @@ -74,6 +75,12 @@ impl Tracker { mod tests { use super::*; #[test] + fn cast_windows_are_native_visibility_sources() { + let mut tracker=Tracker::default(); + tracker.report(BTreeMap::from([("framely.cast.0123".into(),true)]),Instant::now()).unwrap(); + assert!(tracker.state.capture_obscured); + } + #[test] fn aggregate_close_and_lease() { let now = Instant::now(); let mut t = Tracker::default(); diff --git a/tests/cast_frame.cpp b/tests/cast_frame.cpp new file mode 100644 index 0000000..89ef8e2 --- /dev/null +++ b/tests/cast_frame.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cast_frame.h" +int main(){ + auto* display=XOpenDisplay(nullptr);assert(display);int attrs[]={GLX_RGBA,GLX_RED_SIZE,8,GLX_GREEN_SIZE,8,GLX_BLUE_SIZE,8,GLX_ALPHA_SIZE,8,None};auto* visual=glXChooseVisual(display,DefaultScreen(display),attrs);assert(visual);auto context=glXCreateContext(display,visual,nullptr,True);assert(context);auto pixmap=XCreatePixmap(display,RootWindow(display,visual->screen),4,4,visual->depth);auto surface=glXCreateGLXPixmap(display,visual,pixmap);assert(glXMakeCurrent(display,surface,context)); + { + CastFrame frame;frame.width=frame.height=2;glGenTextures(1,&frame.texture);glBindTexture(GL_TEXTURE_2D,frame.texture);glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);std::vector blue(16);for(size_t n=0;n controls(64);for(size_t n=0;n pixels(64);glGetTexImage(GL_TEXTURE_2D,0,GL_RGBA,GL_UNSIGNED_BYTE,pixels.data());for(size_t n=0;n=126&&pixels[n+2]<=128);assert(pixels[n+3]==255);}glBindTexture(GL_TEXTURE_2D,ui);glGetTexImage(GL_TEXTURE_2D,0,GL_RGBA,GL_UNSIGNED_BYTE,pixels.data());assert(pixels==controls); + std::fill(controls.begin(),controls.end(),0);glTexSubImage2D(GL_TEXTURE_2D,0,0,0,4,4,GL_RGBA,GL_UNSIGNED_BYTE,controls.data());output=frame.draw(ui,4,4);glBindTexture(GL_TEXTURE_2D,output);glGetTexImage(GL_TEXTURE_2D,0,GL_RGBA,GL_UNSIGNED_BYTE,pixels.data());assert(pixels[0]==0&&pixels[2]==255);assert(glGetError()==GL_NO_ERROR);glDeleteTextures(1,&ui); + } + glXMakeCurrent(display,None,nullptr);glXDestroyGLXPixmap(display,surface);XFreePixmap(display,pixmap);glXDestroyContext(display,context);XFree(visual);XCloseDisplay(display);std::cout<<"Full-frame video and transparent controls composition passed\n"; +} diff --git a/tests/casting_browser_probe.cpp b/tests/casting_browser_probe.cpp new file mode 100644 index 0000000..a4a3a37 --- /dev/null +++ b/tests/casting_browser_probe.cpp @@ -0,0 +1,43 @@ +// Product integration test: receive the real stream in Framely's CEF engine. +#include "include/capi/cef_app_capi.h" +#include "include/capi/cef_client_capi.h" +#include "include/capi/cef_load_handler_capi.h" +#include "include/cef_api_hash.h" +#include +#include +#include +#include +#include +#include +#include +#include +template struct Handler { + T api{};std::atomic refs{1}; + Handler(){api.base.size=sizeof(T);api.base.add_ref=[](cef_base_ref_counted_t* p){++reinterpret_cast(p)->refs;};api.base.release=[](cef_base_ref_counted_t* p){--reinterpret_cast(p)->refs;return 0;};api.base.has_one_ref=[](cef_base_ref_counted_t* p){return int(reinterpret_cast(p)->refs==1);};api.base.has_at_least_one_ref=[](cef_base_ref_counted_t* p){return int(reinterpret_cast(p)->refs>0);};} + T* acquire(){api.base.add_ref(&api.base);return &api;} +}; +static Handler client;static Handler render;static Handler display;static Handler life;static Handler load; +static bool passed=false,failed=false,closed=false;static unsigned paints=0;static std::vector pixels;static int width=0,height=0; +static void str(cef_string_t& s,const std::string& v){cef_string_utf8_to_utf16(v.data(),v.size(),&s);} +static std::string text(const cef_string_t* s){cef_string_utf8_t o{};cef_string_utf16_to_utf8(s->str,s->length,&o);std::string r(o.str,o.length);cef_string_utf8_clear(&o);return r;} +int main(int argc,char** argv){ + std::cout.setf(std::ios::unitbuf); + const std::string url=argc>1?argv[1]:"", runtime=argc>2?argv[2]:""; + cef_main_args_t args{argc,argv};if(std::strcmp(cef_api_hash(CEF_API_VERSION,0),CEF_API_HASH_PLATFORM))return 2;int child=cef_execute_process(&args,nullptr,nullptr);if(child>=0)return child;if(argc<3)return 2; + auto* cef=std::getenv("FRAMELY_CEF_ROOT");if(!cef)return 2; + cef_settings_t settings{};settings.size=sizeof(settings);settings.windowless_rendering_enabled=1;str(settings.root_cache_path,runtime+"/cache");str(settings.log_file,runtime+"/cef.log");str(settings.resources_dir_path,cef);str(settings.locales_dir_path,std::string(cef)+"/locales"); + if(!cef_initialize(&args,&settings,nullptr,nullptr))return 3; + render.api.get_view_rect=[](cef_render_handler_t*,cef_browser_t*,cef_rect_t* r){*r={0,0,960,720};}; + render.api.on_paint=[](cef_render_handler_t*,cef_browser_t*,cef_paint_element_type_t type,size_t,const cef_rect_t*,const void* data,int w,int h){if(type==PET_VIEW){paints++;width=w;height=h;pixels.assign((const unsigned char*)data,(const unsigned char*)data+size_t(w)*h*4);}}; + display.api.on_console_message=[](cef_display_handler_t*,cef_browser_t*,cef_log_severity_t,const cef_string_t* m,const cef_string_t*,int){auto s=text(m);std::cout<is_main(frame))return;cef_string_t script{},source{}; + str(script,R"JS((()=>{let tries=0;const timer=setInterval(()=>{const video=document.querySelector('video');if(video?.readyState>=2&&video.currentTime>1){clearInterval(timer);if(video.videoWidth!==640||video.videoHeight!==360){console.error('CAST_BROWSER_FAIL unexpected dimensions '+video.videoWidth+'x'+video.videoHeight);return;}console.log('CAST_BROWSER_PASS '+JSON.stringify({width:video.videoWidth,height:video.videoHeight,time:video.currentTime,frames:video.getVideoPlaybackQuality?.().totalVideoFrames}));}else if(++tries>350){clearInterval(timer);console.error('CAST_BROWSER_FAIL '+document.body.innerText+' video='+JSON.stringify(video?{ready:video.readyState,time:video.currentTime,error:video.error?.message}:null));}},100);})())JS"); + frame->execute_java_script(frame,&script,&source,1);cef_string_utf16_clear(&script);}; + client.api.get_render_handler=[](cef_client_t*){return render.acquire();};client.api.get_display_handler=[](cef_client_t*){return display.acquire();};client.api.get_life_span_handler=[](cef_client_t*){return life.acquire();};client.api.get_load_handler=[](cef_client_t*){return load.acquire();}; + cef_window_info_t win{};win.size=sizeof(win);win.windowless_rendering_enabled=1;win.runtime_style=CEF_RUNTIME_STYLE_ALLOY;cef_browser_settings_t bs{};bs.size=sizeof(bs);bs.windowless_frame_rate=30;cef_string_t uri{};str(uri,url);auto* browser=cef_browser_host_create_browser_sync(&win,&client.api,&uri,&bs,nullptr,nullptr);cef_string_utf16_clear(&uri);if(!browser)return 4; + auto* host=browser->get_host(browser);auto end=std::chrono::steady_clock::now()+std::chrono::seconds(40); + while(std::chrono::steady_clock::now()close_browser(host,1);for(int i=0;i<400&&!closed;i++){cef_do_message_loop_work();std::this_thread::sleep_for(std::chrono::milliseconds(5));}bool ok=passed&&!failed&&paints>0&&closed;std::cout<<"CAST_BROWSER_RESULT pass="< int main() { - for (const auto& page : {"settings","updates","about","terminal","files","apk-containers","apk-settings"}) assert(manager_navigation_page(page)); + for (const auto& page : {"casting","casting-picture","casting-receive","casting-devices","settings","updates","about","terminal","files","apk-containers","apk-settings"}) assert(manager_navigation_page(page)); assert(!manager_navigation_page("unknown")); const std::string origin="http://localhost:19626"; assert(allow_plugin_navigation(origin+"/window",origin,true,true)); diff --git a/tests/test_install_handoff.py b/tests/test_install_handoff.py new file mode 100644 index 0000000..8b43154 --- /dev/null +++ b/tests/test_install_handoff.py @@ -0,0 +1,82 @@ +"""Keep an installer launched from the UI alive when the UI is stopped.""" +import json +import os +import pathlib +import subprocess +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).parents[1] + + +class InstallHandoffTests(unittest.TestCase): + def run_handoff(self, cgroup, repair=False, launch_status=0): + source = (ROOT / 'packaging/install.sh').read_text() + block = source.split('# A terminal opened by Framely', 1)[1].split('cd "$base"', 1)[0] + with tempfile.TemporaryDirectory() as temp: + folder = pathlib.Path(temp) + log = folder / 'handoff.json' + # The package path intentionally includes spaces: arguments must survive. + package = folder / 'release package' + package.mkdir() + cat = folder / 'cat' + cat.write_text('#!/bin/sh\nprintf "%s\\n" "$TEST_CGROUP"\n') + runner = folder / 'systemd-run' + runner.write_text('''#!/usr/bin/env python3 +import json, os, pathlib, sys +pathlib.Path(os.environ['TEST_HANDOFF_LOG']).write_text(json.dumps(sys.argv[1:])) +sys.exit(int(os.environ['TEST_LAUNCH_STATUS'])) +''') + cat.chmod(0o755) + runner.chmod(0o755) + env = dict(os.environ, PATH=str(folder) + ':' + os.environ['PATH'], + TEST_CGROUP=cgroup, TEST_HANDOFF_LOG=str(log), + TEST_LAUNCH_STATUS=str(launch_status)) + result = subprocess.run( + ['bash', '-e', '-c', 'base=$1; steam_user=steamos; repair=$2\n' + + '# A terminal opened by Framely' + block + + '\necho FOREGROUND_INSTALL', 'install-handoff', str(package), + 'true' if repair else 'false'], + env=env, capture_output=True, text=True, timeout=10) + return result, json.loads(log.read_text()) if log.exists() else None, package + + def test_ui_and_plugin_terminals_hand_off_before_installing(self): + for cgroup in [ + '0::/system.slice/framely-session.service', + '0::/system.slice/framely.service', + '0::/user.slice/user-1000.slice/user@1000.service/app.slice/framely-ui-recovery.service', + '0::/system.slice/framely-backend-demo.service/child', + '1:name=systemd:/system.slice/framely-session.service\n0::/other', + ]: + with self.subTest(cgroup=cgroup): + result, args, package = self.run_handoff(cgroup) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn('FOREGROUND_INSTALL', result.stdout) + self.assertEqual(args[-3:], ['/usr/bin/bash', str(package / 'install.sh'), 'steamos']) + self.assertIn('--no-block', args) + self.assertIn('--property=StandardInput=null', args) + self.assertIn('--property=StandardOutput=journal', args) + self.assertIn('--property=StandardError=journal', args) + self.assertNotIn('--pipe', args) + self.assertNotIn('--wait', args) + + def test_repair_keeps_arguments_and_launch_failure_stops(self): + result, args, _ = self.run_handoff('0::/system.slice/framely-session.service', repair=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(args[-2:], ['--repair', 'steamos']) + result, _, _ = self.run_handoff('0::/system.slice/framely-session.service', launch_status=1) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('FOREGROUND_INSTALL', result.stdout) + + def test_ssh_and_independent_update_workers_do_not_recurse(self): + for cgroup in [ + '0::/user.slice/user-1000.slice/session-42.scope', + '0::/system.slice/framely-install-123-456.service', + '0::/system.slice/framely-update-123.service', + '0::/system.slice/framely-session.service-other', + ]: + with self.subTest(cgroup=cgroup): + result, args, _ = self.run_handoff(cgroup) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIsNone(args) + self.assertIn('FOREGROUND_INSTALL', result.stdout) diff --git a/tools/build-media.sh b/tools/build-media.sh new file mode 100755 index 0000000..dbc6917 --- /dev/null +++ b/tools/build-media.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +base=$(cd -- "$(dirname -- "$0")/.." && pwd) +cd "$base" +export CARGO_HOME=${CARGO_HOME:-/tmp/framely-cast-cargo} +( +if [[ -n ${FRAMELY_SYSROOT:-} ]]; then + export PATH="$FRAMELY_SYSROOT/usr/bin:$PATH" + export PKG_CONFIG_PATH="$FRAMELY_SYSROOT/usr/lib/aarch64-linux-gnu/pkgconfig:$FRAMELY_SYSROOT/usr/share/pkgconfig" + export PKG_CONFIG_SYSROOT_DIR="$FRAMELY_SYSROOT" + export LIBCLANG_PATH="$FRAMELY_SYSROOT/usr/lib/llvm-19/lib" + export LD_LIBRARY_PATH="$FRAMELY_SYSROOT/usr/lib/aarch64-linux-gnu:$FRAMELY_SYSROOT/usr/lib/llvm-19/lib:${LD_LIBRARY_PATH:-}" + export BINDGEN_EXTRA_CLANG_ARGS="-I$FRAMELY_SYSROOT/usr/lib/llvm-19/lib/clang/19/include -I$FRAMELY_SYSROOT/usr/include -I$FRAMELY_SYSROOT/usr/include/aarch64-linux-gnu" + export RUSTFLAGS="${RUSTFLAGS:-} -L native=$FRAMELY_SYSROOT/usr/lib/aarch64-linux-gnu" +fi +cargo build --release --locked --manifest-path media/capture/Cargo.toml +) +mkdir -p media/bin +cp media/capture/target/release/framely-capture media/capture/target/release/framely-panel-grab media/bin/ +# Native dependencies use the target system's GStreamer and GUPnP libraries. +python3 tools/fetch-media.py +python3 media/patch-uxplay.py media/source/UxPlay-1.73.7 +cmake -S media/source/UxPlay-1.73.7 -B media/source/uxplay-build -DNO_X11_DEPS=ON -DCMAKE_BUILD_TYPE=Release +cmake --build media/source/uxplay-build -j"${FRAMELY_BUILD_JOBS:-4}" +cp media/source/uxplay-build/uxplay media/bin/ +g++ -std=c++17 -O2 -shared -fPIC media/native/airplay_sink.cpp -o media/bin/libgstframely.so $(pkg-config --cflags --libs gstreamer-app-1.0 gstreamer-video-1.0) +g++ -std=c++17 -O2 -Inative/vendor media/native/receiver.cpp -o media/bin/framely-receiver $(pkg-config --cflags --libs gupnp-1.6 gstreamer-app-1.0 gstreamer-video-1.0) +mkdir -p media/bin/upnp +cp media/upnp/*.xml media/bin/upnp/ +cp media/source/UxPlay-1.73.7/LICENSE media/bin/LICENSE.uxplay +cp media/source/UxPlay-1.73.7/lib/llhttp/LICENSE-MIT media/bin/LICENSE.llhttp diff --git a/tools/fetch-media.py b/tools/fetch-media.py new file mode 100644 index 0000000..5366bdf --- /dev/null +++ b/tools/fetch-media.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""Fetch immutable, checksum-verified upstream media dependencies.""" +import hashlib,io,json,tarfile,urllib.request +from pathlib import Path +base=Path(__file__).resolve().parent.parent +pins=json.loads((base/'media/dependencies.json').read_text()) +for name,pin in pins.items(): + payload=urllib.request.urlopen(pin['url'],timeout=120).read() + if hashlib.sha256(payload).hexdigest()!=pin['sha256']: + raise SystemExit(f'{name}: upstream archive checksum mismatch') + target=base/'media'/('bin' if name=='mediamtx' else 'source') + target.mkdir(exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(payload),mode='r:gz') as archive: + archive.extractall(target,filter='data') + if name=='mediamtx':(target/'mediamtx').chmod(0o755) diff --git a/tools/package-release.sh b/tools/package-release.sh index c9321be..62cb060 100755 --- a/tools/package-release.sh +++ b/tools/package-release.sh @@ -9,13 +9,29 @@ cargo build --release --locked npm ci --no-audit --no-fund npm run build bash tools/build-native.sh "$cef" +bash tools/build-media.sh binary=${CARGO_TARGET_DIR:-target}/release/framely version=$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -1) -build_id="$version-$(find "$binary" target/native ui/dist sdk templates/plugin examples/showcase packaging tools docs assets/branding assets/search README.md README.zh-CN.md LICENSE -type f ! -path '*/__pycache__/*' -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -c1-12)" +build_id="$version-$(find "$binary" target/native media/bin ui/dist sdk templates/plugin examples/showcase packaging tools docs assets/branding assets/search README.md README.zh-CN.md LICENSE -type f ! -path '*/__pycache__/*' -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -c1-12)" stage="$base/release/framely-$build_id" [[ ! -e $stage ]] || { echo 'Release already exists.' >&2; exit 1; } -mkdir -p "$stage"/{bin,lib/cef,lib/openvr,share/ui,share/licenses,tools} +mkdir -p "$stage"/{bin,lib/cef,lib/openvr,lib/media,share/ui,share/licenses,tools} cp "$binary" "$stage/bin/" +cp media/bin/framely-panel-grab "$stage/bin/" +cp media/bin/framely-capture media/bin/mediamtx media/bin/uxplay media/bin/framely-receiver media/bin/libgstframely.so "$stage/lib/media/" +cp -a media/bin/upnp "$stage/lib/media/" +cp media/bin/LICENSE "$stage/share/licenses/mediamtx.txt" +cp media/bin/LICENSE.uxplay "$stage/share/licenses/uxplay.txt" +cp media/bin/LICENSE.llhttp "$stage/share/licenses/uxplay-llhttp.txt" +cp media/capture/LICENSE.framecorder "$stage/share/licenses/panel-capture.txt" +cp media/THIRD_PARTY_NOTICES.md "$stage/share/licenses/media-notices.md" +mkdir -p "$stage/share/source/media/capture" "$stage/share/source/tools" "$stage/share/source/native/vendor" +cp -a media/native media/upnp media/patch-uxplay.py media/dependencies.json "$stage/share/source/media/" +cp -a media/capture/src media/capture/shaders media/capture/Cargo.toml media/capture/Cargo.lock media/capture/build.rs media/capture/LICENSE.framecorder "$stage/share/source/media/capture/" +cp tools/build-media.sh tools/fetch-media.py "$stage/share/source/tools/" +cp native/vendor/json.hpp "$stage/share/source/native/vendor/" +tar -czf "$stage/share/source/media/uxplay.tar.gz" -C media/source UxPlay-1.73.7 + cp target/native/framely-vr "$stage/lib/cef/" cp -a "$cef/Release/." "$stage/lib/cef/" cp -a "$cef/Resources/." "$stage/lib/cef/" diff --git a/tools/test-casting-media.py b/tools/test-casting-media.py new file mode 100644 index 0000000..65f0394 --- /dev/null +++ b/tools/test-casting-media.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Exercise consent and real UPnP SOAP actions against native media workers.""" +import argparse,json,os,select,socket,subprocess,tempfile,time,urllib.request +from pathlib import Path +from http.server import ThreadingHTTPServer,SimpleHTTPRequestHandler +from threading import Thread +parser=argparse.ArgumentParser();parser.add_argument('--binaries',required=True);parser.add_argument('--upnp',required=True);args=parser.parse_args() +binaries=Path(args.binaries).resolve() +with tempfile.TemporaryDirectory(prefix='framely-media-test-') as directory: + root=Path(directory);events=socket.socket(socket.AF_UNIX,socket.SOCK_DGRAM);events.bind(str(root/'events.sock'));events.settimeout(5) + token='0123456789abcdef0123456789abcdef';env=dict(os.environ,GST_PLUGIN_PATH=str(binaries),FRAMELY_CAST_DIR=directory,FRAMELY_CAST_ID=token) + p=subprocess.Popen(['gst-launch-1.0','-q','videotestsrc','is-live=true','!','video/x-raw,width=320,height=240,framerate=10/1','!','framelyvideosink'],env=env,stderr=subprocess.PIPE) + try: + event=json.loads(events.recv(4096));assert event['event']=='video' and event['width']==320,event + frame=root/(token+'.frame');time.sleep(.2);assert not frame.exists(),'video visible before consent' + (root/(token+'.accept')).touch();time.sleep(.3);assert frame.stat().st_size==16+320*240*4 + finally: + p.terminate();p.wait(timeout=3) + assert not p.stderr.read(), 'GStreamer consent sink emitted errors' + print('PASS: video consent gate') + # Capture only a temporary null sink, not the user's active audio output. + import array + sink_name=f'framely_cast_test_{os.getpid()}' + module=subprocess.check_output(['pactl','load-module','module-null-sink',f'sink_name={sink_name}','rate=48000','channels=2'],text=True).strip() + env['PULSE_SINK']=sink_name + audio_token='abcdef0123456789abcdef0123456789ab' + # Frame spatializes ordinary streams; this test-only node name keeps its + # own temporary sink route intact without changing system settings. + audio_env=dict(env,FRAMELY_CAST_ID=audio_token,PULSE_PROP='node.name=framely-test-filter-chain') + p=subprocess.Popen(['gst-launch-1.0','-q','audiotestsrc','is-live=true','wave=sine','!','audio/x-raw,rate=48000,channels=2','!','framelyaudiosink'],env=audio_env,stderr=subprocess.PIPE) + def peak(): + raw=subprocess.check_output(['ffmpeg','-hide_banner','-loglevel','error','-f','pulse','-i',sink_name+'.monitor','-t','0.2','-acodec','pcm_s16le','-f','s16le','pipe:1']) + samples=array.array('h');samples.frombytes(raw);return max(map(abs,samples),default=0) + try: + event=json.loads(events.recv(4096)) + while event['id']!=audio_token:event=json.loads(events.recv(4096)) + assert event['event']=='audio',event + time.sleep(.3);assert peak()<10,'audio audible before consent' + (root/(audio_token+'.accept')).touch();time.sleep(.3);assert peak()>1000,'audio not audible after consent' + (root/(audio_token+'.accept')).unlink();time.sleep(.3);assert peak()<10,'audio continues after pause' + print('PASS: audio consent gate and pause') + finally: + p.terminate();p.wait(timeout=3);subprocess.run(['pactl','unload-module',module],check=True) + assert not p.stderr.read(),'GStreamer audio sink emitted errors' + + # A complete renderer description, with the same XML shipped by Framely. + xml=root/'upnp';xml.mkdir() + import shutil + for source in Path(args.upnp).glob('*.xml'):shutil.copyfile(source,xml/source.name) + services=''.join(f'urn:schemas-upnp-org:service:{s}:1urn:upnp-org:serviceId:{s}/{s}.xml/{s}/control/{s}/event' for s in ['AVTransport','RenderingControl','ConnectionManager']) + (xml/'device.xml').write_text(f'10urn:schemas-upnp-org:device:MediaRenderer:1Framely testFramelyTestuuid:56ce7555-6baf-4360-bbd6-61a32b06b6aa{services}') + subprocess.run(['ffmpeg','-hide_banner','-loglevel','error','-f','lavfi','-i','testsrc2=size=320x240:rate=10','-t','8','-c:v','libx264','-pix_fmt','yuv420p','-movflags','+faststart',str(root/'video.mp4')],check=True) + class Handler(SimpleHTTPRequestHandler): + def __init__(self,*a,**k):super().__init__(*a,directory=directory,**k) + def log_message(self,*a):pass + def send_head(self): + # Real media servers support HTTP Range; SimpleHTTPRequestHandler does not. + file=Path(self.translate_path(self.path)).open('rb');size=os.fstat(file.fileno()).st_size + value=self.headers.get('Range');self.remaining=size + if value: + import re + match=re.fullmatch(r'bytes=(\d+)-(\d*)',value);assert match,value + start=int(match[1]);end=min(int(match[2]) if match[2] else size-1,size-1) + assert 0<=start<=end{escape(str(v))}' for k,v in fields.items()) + urn=f'urn:schemas-upnp-org:service:{service}:1' + body=f'{content}' + req=urllib.request.Request(f'http://{host}:{port}/{service}/control',data=body.encode(),headers={'Content-Type':'text/xml; charset=utf-8','SOAPAction':f'"{urn}#{action}"'}) + try: + with urllib.request.build_opener(urllib.request.ProxyHandler({})).open(req,timeout=5) as r: + assert r.status==200 + return r.read().decode() + except urllib.error.HTTPError as e:raise AssertionError(e.read().decode()) + sequence=0 + def control(kind,id,**fields): + global sequence + sequence+=1 + p.stdin.write(json.dumps(dict(kind=kind,id=id,request=sequence,**fields))+'\n');p.stdin.flush() + while True: + reply=read() + if reply.get('request')==sequence: + assert 'error' not in reply,reply + return reply['result'] + soap('SetAVTransportURI',dict(InstanceID=0,CurrentURI=uri,CurrentURIMetaData='')) + request=read() + while request['event']=='ready':request=read() + assert request['event']=='request',request + id=request['id'];frame=root/(id+'.frame');time.sleep(.2);assert not frame.exists(),'DLNA media played before consent' + p.stdin.write(json.dumps({'kind':'accept','id':id,'request':1})+'\n');p.stdin.flush() + response=read();assert response['result'] is True,response + event=json.loads(events.recv(4096)) + while event['id']!=id:event=json.loads(events.recv(4096)) + assert event['width']==320,event + time.sleep(.2);assert frame.stat().st_size==16+320*240*4 + status=control('status',id) + assert 7.9<=status['duration']<=8.1,status + control('pause',id,paused=True);assert control('status',id)['paused'] is True + soap('SetVolume',dict(InstanceID=0,Channel='Master',DesiredVolume=37),'RenderingControl') + assert '37' in soap('GetVolume',dict(InstanceID=0,Channel='Master'),'RenderingControl') + assert abs(control('status',id)['volume']-.37)<.001 + soap('SetMute',dict(InstanceID=0,Channel='Master',DesiredMute=1),'RenderingControl') + assert '1' in soap('GetMute',dict(InstanceID=0,Channel='Master'),'RenderingControl') + control('seek',id,seconds=2);time.sleep(.15) + assert control('status',id)['position']>=1.9 + assert '00:00:08' in soap('GetPositionInfo',dict(InstanceID=0)) + # A second request must leave the first session and frame available. + soap('SetAVTransportURI',dict(InstanceID=0,CurrentURI=uri,CurrentURIMetaData='')) + request=read() + while request.get('event')!='request':request=read() + second=request['id'];control('accept',second) + deadline=time.monotonic()+5 + while not (root/(second+'.frame')).exists() and time.monotonic()NO_MEDIA_PRESENT' in soap('GetTransportInfo',dict(InstanceID=0)) + # Repeated requests/rejections exercise session reclamation beyond the cap. + for _ in range(18): + soap('SetAVTransportURI',dict(InstanceID=0,CurrentURI=uri,CurrentURIMetaData='')) + request=read() + while request.get('event')!='request':request=read() + control('reject',request['id']) + print('PASS: DLNA consent, decoding, pause, seek, volume, concurrent sessions and reclamation') + # A separate renderer exercises GUPnP discovery and outgoing SOAP calls. + target_dir=root/'target';target_dir.mkdir();target_xml=root/'target-upnp';shutil.copytree(xml,target_xml) + target_udn='uuid:084442d0-ed68-427b-8dc8-ae4b310f6830' + description=(target_xml/'device.xml').read_text().replace('uuid:56ce7555-6baf-4360-bbd6-61a32b06b6aa',target_udn) + (target_xml/'device.xml').write_text(description) + target=subprocess.Popen([str(binaries/'framely-receiver'),str(target_dir),str(target_xml),'1'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,bufsize=1) + target_replies=Queue() + def target_reader(): + for line in target.stdout:target_replies.put(json.loads(line)) + Thread(target=target_reader,daemon=True).start() + try: + deadline=time.monotonic()+12 + while True: + found=control('devices','') + if any(d['id']==target_udn for d in found):break + assert time.monotonic()10000,capture.stderr.read().decode()+helper.stderr.read().decode() + probe=subprocess.run(['ffprobe','-v','error','-framerate','30','-count_frames','-show_entries','stream=width,height,codec_name,nb_read_frames','-of','json',str(path)],capture_output=True,text=True,check=True) + stream=json.loads(probe.stdout)['streams'][0];assert stream['width']==640 and stream['codec_name']=='h264' and int(stream['nb_read_frames'])>=10,stream + print(name,stream) + print('PASS: panel capture, Iris encoding, cropped eye view and raw stereo') diff --git a/tools/test-install-handoff.py b/tools/test-install-handoff.py new file mode 100644 index 0000000..6214759 --- /dev/null +++ b/tools/test-install-handoff.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Reproduce a terminal's service stopping during install, without root changes.""" +import argparse +import os +import pathlib +import shlex +import subprocess +import tempfile +import time + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--installer', type=pathlib.Path, + default=pathlib.Path(__file__).parents[1] / 'packaging/install.sh') +args = parser.parse_args() +source = args.installer.read_text() +block = source.split('# A terminal opened by Framely', 1)[1].split('cd "$base"', 1)[0] +parent = f'framely-hook-install-check-{os.getpid()}' +with tempfile.TemporaryDirectory(prefix='framely-install-check-') as work: + directory = pathlib.Path(work) + # Use the user manager to test the same cgroup semantics without root access. + wrapper = directory / 'systemd-run' + wrapper.write_text('#!/bin/sh\nexec /usr/bin/systemd-run --user "$@"\n') + wrapper.chmod(0o755) + (directory / 'install.sh').write_text( + 'set -e\nsleep 0.5\n' + + 'systemctl --user stop ' + shlex.quote(parent + '.service') + '\n' + + 'cat /proc/self/cgroup > ' + shlex.quote(str(directory / 'worker.cgroup')) + '\n' + + 'echo SURVIVED > ' + shlex.quote(str(directory / 'passed')) + '\n') + (directory / 'entry.sh').write_text( + 'set -e\nbase=' + shlex.quote(work) + '\nsteam_user=' + shlex.quote(str(os.getuid())) + + '\nrepair=false\n# A terminal opened by Framely' + block) + (directory / 'parent.sh').write_text( + 'set -e\nexport PATH=' + shlex.quote(work) + ':"$PATH"\n' + + '/usr/bin/bash ' + shlex.quote(str(directory / 'entry.sh')) + '\n' + + 'sleep 30\n') + try: + subprocess.run(['systemd-run', '--user', '--collect', '--quiet', + '--unit=' + parent, '--property=Type=exec', + '/usr/bin/bash', str(directory / 'parent.sh')], check=True) + deadline = time.monotonic() + 15 + while not (directory / 'passed').exists() and time.monotonic() < deadline: + time.sleep(0.1) + assert (directory / 'passed').exists(), 'Worker did not survive the parent service stop' + cgroup = (directory / 'worker.cgroup').read_text() + assert 'framely-install-' in cgroup and parent not in cgroup, cgroup + status = subprocess.run(['systemctl', '--user', 'is-active', parent + '.service'], + capture_output=True, text=True) + assert status.returncode != 0, 'Parent service was not stopped' + print('PASS: installer worker survived the terminal service being stopped') + finally: + subprocess.run(['systemctl', '--user', 'stop', parent + '.service'], + capture_output=True) diff --git a/ui/locales/en-US.json b/ui/locales/en-US.json index 20ef4d9..7d1f52d 100644 --- a/ui/locales/en-US.json +++ b/ui/locales/en-US.json @@ -464,7 +464,7 @@ "搜索通知应用": "Search notification apps", "搜索应用": "Search apps", "全局开关优先；关闭全局开关不会改变应用偏好。": "Global switches take priority. App preferences are kept when a global switch is off.", - "应用": "Apps", + "应用": "Apply", "未找到应用": "No matching apps", "本体通知，包括检查更新提醒。": "Framely notifications, including update reminders.", "启动台": "Launcher", @@ -1054,6 +1054,174 @@ "自动重命名上传的文件，保留原文件。": "Rename incoming files and keep the originals.", "不上传同名文件，保留原文件。": "Leave matching files unchanged without uploading them.", "用上传的文件替换原文件。": "Replace existing files with the incoming files.", - "开始上传": "Start upload" + "开始上传": "Start upload", + "投屏": "Casting", + "观看头显画面，接收投屏或投到其他设备。": "Watch the headset, receive media, or cast to another device.", + "网页观看": "Watch in browser", + "正在串流": "Streaming", + "未开始串流": "Stream stopped", + "停止串流": "Stop streaming", + "开始串流": "Start streaming", + "关闭预览": "Hide preview", + "显示预览": "Show preview", + "打开观看网页": "Open viewer", + "复制观看地址": "Copy viewer address", + "画面": "Picture", + "采集方式": "Capture source", + "屏幕采集": "Display capture", + "输出模式": "Output mode", + "单眼画面": "Single-eye view", + "双眼原始画面": "Raw stereo view", + "输出经过镜头畸变校正的单眼视角，包含透视和空间窗口。": "A lens-corrected view from one eye, including passthrough and spatial windows.", + "保留双眼屏幕布局、原始方向与镜头畸变，输出高度按原始比例计算。": "Preserve both eyes, panel orientation, and lens distortion. Height follows the native aspect ratio.", + "SteamVR 提供固定来源画面；下面的分辨率和帧率仅调整串流输出。": "SteamVR provides a fixed source image. Resolution and frame rate below affect the stream output.", + "主力眼": "Dominant eye", + "SteamVR 设置：左眼": "SteamVR setting: left eye", + "SteamVR 设置：右眼": "SteamVR setting: right eye", + "读取 SteamVR 设置": "Use SteamVR setting", + "左眼": "Left eye", + "右眼": "Right eye", + "水平视野（度）": "Horizontal field of view (degrees)", + "最大有效范围": "Widest covered view", + "水平中心（度）": "Horizontal center (degrees)", + "垂直中心（度）": "Vertical center (degrees)", + "视野留空使用最大有效范围。垂直视野随画幅计算；超出画面覆盖范围的部分会显示黑边。": "Leave the field of view empty for the widest covered view. Vertical field of view follows the aspect ratio. Areas outside the display appear black.", + "视频与声音": "Video and audio", + "输出宽度（像素）": "Output width (pixels)", + "输出高度（像素）": "Output height (pixels)", + "输出帧率（fps）": "Output frame rate (fps)", + "视频码率（Mbps）": "Video bitrate (Mbps)", + "网页视频使用 H.264，声音使用 Opus。屏幕采集使用 Iris 硬件编码；SteamVR 使用软件编码。": "Browser video uses H.264 and audio uses Opus. Display capture uses Iris hardware encoding; SteamVR uses software encoding.", + "系统声音": "System audio", + "麦克风": "Microphone", + "接收投屏": "Receive casting", + "设备名称": "Device name", + "AirPlay 接收": "AirPlay receiver", + "接收一路音频或屏幕镜像。": "Receive one audio stream or screen mirror.", + "DLNA 接收": "DLNA receiver", + "接收其他设备发送的媒体播放地址。": "Receive media playback URLs from other devices.", + "收到投屏后先询问是否打开；接受后显示对应的音频或视频控制窗口。": "Each cast asks for permission. Accept to open its audio or video controls.", + "设置尚未应用": "Unapplied changes", + "已应用": "Applied", + "应用画面设置会重新启动当前串流。": "Applying picture settings restarts the current stream.", + "正在应用…": "Applying…", + "观看连接失败，请重新连接。": "The viewing connection failed. Reconnect to try again.", + "串流尚未就绪，请稍后重试。": "The stream is not ready. Try again shortly.", + "正在连接画面…": "Connecting to the stream…", + "开始串流后显示头显画面。": "Start streaming to see the headset view.", + "头显画面": "Headset view", + "点击播放器声音按钮即可收听。": "Unmute the player to hear audio.", + "音频投屏": "Audio cast", + "音量": "Volume", + "播放进度": "Playback position", + "停止投屏": "Stop cast", + "无法复制，请手动复制观看地址。": "Copy failed. Copy the viewer address manually.", + "投到其他设备": "Cast to another device", + "选择同一局域网中的 DLNA 设备，发送当前头显画面。": "Select a DLNA device on the same network to show the headset view.", + "查找设备": "Find devices", + "目标设备": "Target device", + "开始投屏": "Start cast", + "停止设备投屏": "Stop device playback", + "开启网络管理面板后，其他设备才能获取投屏画面。": "Enable network access so other devices can retrieve the stream.", + "请先开始头显串流。": "Start the headset stream first.", + "当前投屏": "Current casts", + "音频或视频": "Audio or video", + "打开投屏": "Open cast", + "拒绝": "Reject", + "播放": "Play", + "暂停": "Pause", + "已复制": "Copied", + "视频": "Video", + "加载中…": "Loading…", + "当前浏览器不支持 H.264，请使用支持 H.264 的浏览器观看。": "This browser does not support H.264. Use a browser with H.264 support to watch.", + "网页视频优先使用 H.264，兼容 VP8；声音使用 Opus。屏幕采集使用硬件编码，VP8 兼容输出和 SteamVR 使用软件编码。": "Web video prefers H.264 with VP8 compatibility; audio uses Opus. Screen capture uses hardware encoding. VP8 compatibility and SteamVR use software encoding.", + "返回投屏设置": "Back to casting settings", + "正在启动…": "Starting…", + "操作未完成": "Action could not be completed", + "分享你看到的画面": "Share what you see", + "启动后，手机和电脑可以通过网页实时观看。": "Start streaming to watch live on a phone or computer.", + "在其他设备上观看": "Watch on another device", + "连接同一网络，打开观看地址。": "Connect to the same network and open the viewing link.", + "网页访问设置": "Web access settings", + "画面来源": "Capture source", + "输出画面": "Output picture", + "校正畸变，适合普通观看": "Corrected perspective for everyday viewing", + "保留双眼布局和镜头畸变": "Original stereo layout and lens distortion", + "取景": "Framing", + "调整取景": "Adjust framing", + "取景预览期间，共享画面会切换为完整视角。": "While framing, the shared stream switches to the full reference view.", + "应用取景": "Apply framing", + "完整视野": "Full field of view", + "已设置取景范围": "Custom framing", + "直接在实时画面上移动和缩放取景框。": "Move and resize the frame directly on the live picture.", + "跟随 SteamVR": "Follow SteamVR", + "当前：左眼": "Current: left eye", + "当前：右眼": "Current: right eye", + "画幅与分辨率": "Aspect ratio and resolution", + "画面比例": "Aspect ratio", + "原始双眼比例由头显屏幕决定。": "The headset display determines the original stereo aspect ratio.", + "分辨率": "Resolution", + "当前分辨率：{0} × {1}": "Current resolution: {0} × {1}", + "流畅度与画质": "Motion and quality", + "帧率": "Frame rate", + "视频码率": "Video bitrate", + "更省带宽": "Less bandwidth", + "画面更清晰": "Sharper picture", + "声音": "Audio", + "其他设备会在投屏列表中看到这个名称。": "Other devices see this name in their casting list.", + "接收媒体播放地址，可打开多个窗口。": "Receive media URLs and open multiple windows.", + "收到投屏后先询问，接受后才打开播放器。": "Ask before opening a received stream in the player.", + "正在播放": "Playing", + "等待确认": "Awaiting approval", + "停止": "Stop", + "还没有收到投屏": "No incoming streams", + "开启接收后，在其他设备上选择 Framely。": "Enable receiving, then select Framely on another device.", + "找到 {0} 台设备": "Found {0} devices", + "附近设备": "Nearby devices", + "正在查找…": "Searching…", + "重新查找": "Search again", + "需要开启网页访问，设备才能读取投屏画面。": "Enable web access so devices can read the stream.", + "设置网页访问": "Set up web access", + "先开始头显串流，再选择播放设备。": "Start the headset stream, then select a playback device.", + "正在投屏": "Casting", + "投到此设备": "Cast to this device", + "正在查找设备": "Searching for devices", + "未找到播放设备": "No playback devices found", + "查找附近设备": "Find nearby devices", + "确认播放设备已开启 DLNA，并与头显连接同一网络。": "Enable DLNA on the playback device and connect it to the headset's network.", + "有未应用的修改": "Unapplied changes", + "设置已保存": "Settings saved", + "应用后会重新启动串流。": "Applying changes restarts the stream.", + "应用修改": "Apply changes", + "无法播放，请重新连接。": "Unable to play. Reconnect to try again.", + "此浏览器暂不支持全屏。": "Fullscreen is unavailable in this browser.", + "实时画面": "Live picture", + "连接已断开": "Disconnected", + "正在连接头显": "Connecting to your headset", + "画面即将开始，请稍候。": "Your live picture will start shortly.", + "暂时无法观看": "Unable to watch right now", + "确认头显已开始串流，然后重新连接。": "Make sure the headset is streaming, then reconnect.", + "实时": "Live", + "取景框，使用方向键移动": "Framing area. Use arrow keys to move.", + "缩小取景范围": "Narrow the field of view", + "视野": "Field of view", + "扩大取景范围": "Widen the field of view", + "重置取景": "Reset framing", + "拖动取景框移动画面，拖动四角调整范围。也可以用方向键移动。": "Drag the frame to move it, or drag a corner to resize. Arrow keys also move the frame.", + "画面与声音": "Picture and audio", + "投到设备": "Cast to device", + "头显与局域网设备": "Headset and local devices", + "把头显画面分享给同一网络中的手机或电脑。": "Share your headset view with phones and computers on the same network.", + "选择画面来源，直接调整取景和输出画质。": "Choose a capture source, adjust framing and set picture quality.", + "接收其他设备的音频和视频，在头显中打开。": "Open audio and video from other devices in your headset.", + "选择同一网络中的设备，播放当前头显画面。": "Play your headset view on a device on the same network.", + "包含透视、Steam 菜单和空间窗口。": "Includes passthrough, Steam menus and spatial windows.", + "使用 SteamVR 提供的镜像画面。": "Uses the mirror picture provided by SteamVR.", + "开启声音": "Enable sound", + "静音": "Mute", + "完整画面": "Show full picture", + "填满屏幕": "Fill screen", + "退出全屏": "Exit fullscreen", + "全屏": "Fullscreen" } } diff --git a/ui/locales/zh-CN.json b/ui/locales/zh-CN.json index bbcd8c6..4b06d1e 100644 --- a/ui/locales/zh-CN.json +++ b/ui/locales/zh-CN.json @@ -1054,6 +1054,174 @@ "自动重命名上传的文件，保留原文件。": "自动重命名上传的文件，保留原文件。", "不上传同名文件，保留原文件。": "不上传同名文件，保留原文件。", "用上传的文件替换原文件。": "用上传的文件替换原文件。", - "开始上传": "开始上传" + "开始上传": "开始上传", + "投屏": "投屏", + "观看头显画面，接收投屏或投到其他设备。": "观看头显画面，接收投屏或投到其他设备。", + "网页观看": "网页观看", + "正在串流": "正在串流", + "未开始串流": "未开始串流", + "停止串流": "停止串流", + "开始串流": "开始串流", + "关闭预览": "关闭预览", + "显示预览": "显示预览", + "打开观看网页": "打开观看网页", + "复制观看地址": "复制观看地址", + "画面": "画面", + "采集方式": "采集方式", + "屏幕采集": "屏幕采集", + "输出模式": "输出模式", + "单眼画面": "单眼画面", + "双眼原始画面": "双眼原始画面", + "输出经过镜头畸变校正的单眼视角，包含透视和空间窗口。": "输出经过镜头畸变校正的单眼视角，包含透视和空间窗口。", + "保留双眼屏幕布局、原始方向与镜头畸变，输出高度按原始比例计算。": "保留双眼屏幕布局、原始方向与镜头畸变，输出高度按原始比例计算。", + "SteamVR 提供固定来源画面；下面的分辨率和帧率仅调整串流输出。": "SteamVR 提供固定来源画面；下面的分辨率和帧率仅调整串流输出。", + "主力眼": "主力眼", + "SteamVR 设置：左眼": "SteamVR 设置：左眼", + "SteamVR 设置：右眼": "SteamVR 设置：右眼", + "读取 SteamVR 设置": "读取 SteamVR 设置", + "左眼": "左眼", + "右眼": "右眼", + "水平视野（度）": "水平视野（度）", + "最大有效范围": "最大有效范围", + "水平中心（度）": "水平中心（度）", + "垂直中心（度）": "垂直中心（度）", + "视野留空使用最大有效范围。垂直视野随画幅计算；超出画面覆盖范围的部分会显示黑边。": "视野留空使用最大有效范围。垂直视野随画幅计算；超出画面覆盖范围的部分会显示黑边。", + "视频与声音": "视频与声音", + "输出宽度（像素）": "输出宽度（像素）", + "输出高度（像素）": "输出高度（像素）", + "输出帧率（fps）": "输出帧率（fps）", + "视频码率（Mbps）": "视频码率（Mbps）", + "网页视频使用 H.264，声音使用 Opus。屏幕采集使用 Iris 硬件编码；SteamVR 使用软件编码。": "网页视频使用 H.264，声音使用 Opus。屏幕采集使用 Iris 硬件编码；SteamVR 使用软件编码。", + "系统声音": "系统声音", + "麦克风": "麦克风", + "接收投屏": "接收投屏", + "设备名称": "设备名称", + "AirPlay 接收": "AirPlay 接收", + "接收一路音频或屏幕镜像。": "接收一路音频或屏幕镜像。", + "DLNA 接收": "DLNA 接收", + "接收其他设备发送的媒体播放地址。": "接收其他设备发送的媒体播放地址。", + "收到投屏后先询问是否打开；接受后显示对应的音频或视频控制窗口。": "收到投屏后先询问是否打开；接受后显示对应的音频或视频控制窗口。", + "设置尚未应用": "设置尚未应用", + "已应用": "已应用", + "应用画面设置会重新启动当前串流。": "应用画面设置会重新启动当前串流。", + "正在应用…": "正在应用…", + "观看连接失败，请重新连接。": "观看连接失败，请重新连接。", + "串流尚未就绪，请稍后重试。": "串流尚未就绪，请稍后重试。", + "正在连接画面…": "正在连接画面…", + "开始串流后显示头显画面。": "开始串流后显示头显画面。", + "头显画面": "头显画面", + "点击播放器声音按钮即可收听。": "点击播放器声音按钮即可收听。", + "音频投屏": "音频投屏", + "音量": "音量", + "播放进度": "播放进度", + "停止投屏": "停止投屏", + "无法复制，请手动复制观看地址。": "无法复制，请手动复制观看地址。", + "投到其他设备": "投到其他设备", + "选择同一局域网中的 DLNA 设备，发送当前头显画面。": "选择同一局域网中的 DLNA 设备，发送当前头显画面。", + "查找设备": "查找设备", + "目标设备": "目标设备", + "开始投屏": "开始投屏", + "停止设备投屏": "停止设备投屏", + "开启网络管理面板后，其他设备才能获取投屏画面。": "开启网络管理面板后，其他设备才能获取投屏画面。", + "请先开始头显串流。": "请先开始头显串流。", + "当前投屏": "当前投屏", + "音频或视频": "音频或视频", + "打开投屏": "打开投屏", + "拒绝": "拒绝", + "播放": "播放", + "暂停": "暂停", + "已复制": "已复制", + "视频": "视频", + "加载中…": "加载中…", + "当前浏览器不支持 H.264，请使用支持 H.264 的浏览器观看。": "当前浏览器不支持 H.264，请使用支持 H.264 的浏览器观看。", + "网页视频优先使用 H.264，兼容 VP8；声音使用 Opus。屏幕采集使用硬件编码，VP8 兼容输出和 SteamVR 使用软件编码。": "网页视频优先使用 H.264，兼容 VP8；声音使用 Opus。屏幕采集使用硬件编码，VP8 兼容输出和 SteamVR 使用软件编码。", + "返回投屏设置": "返回投屏设置", + "正在启动…": "正在启动…", + "操作未完成": "操作未完成", + "分享你看到的画面": "分享你看到的画面", + "启动后，手机和电脑可以通过网页实时观看。": "启动后，手机和电脑可以通过网页实时观看。", + "在其他设备上观看": "在其他设备上观看", + "连接同一网络，打开观看地址。": "连接同一网络，打开观看地址。", + "网页访问设置": "网页访问设置", + "画面来源": "画面来源", + "输出画面": "输出画面", + "校正畸变，适合普通观看": "校正畸变，适合普通观看", + "保留双眼布局和镜头畸变": "保留双眼布局和镜头畸变", + "取景": "取景", + "调整取景": "调整取景", + "取景预览期间，共享画面会切换为完整视角。": "取景预览期间，共享画面会切换为完整视角。", + "应用取景": "应用取景", + "完整视野": "完整视野", + "已设置取景范围": "已设置取景范围", + "直接在实时画面上移动和缩放取景框。": "直接在实时画面上移动和缩放取景框。", + "跟随 SteamVR": "跟随 SteamVR", + "当前：左眼": "当前：左眼", + "当前：右眼": "当前：右眼", + "画幅与分辨率": "画幅与分辨率", + "画面比例": "画面比例", + "原始双眼比例由头显屏幕决定。": "原始双眼比例由头显屏幕决定。", + "分辨率": "分辨率", + "当前分辨率：{0} × {1}": "当前分辨率：{0} × {1}", + "流畅度与画质": "流畅度与画质", + "帧率": "帧率", + "视频码率": "视频码率", + "更省带宽": "更省带宽", + "画面更清晰": "画面更清晰", + "声音": "声音", + "其他设备会在投屏列表中看到这个名称。": "其他设备会在投屏列表中看到这个名称。", + "接收媒体播放地址，可打开多个窗口。": "接收媒体播放地址，可打开多个窗口。", + "收到投屏后先询问，接受后才打开播放器。": "收到投屏后先询问，接受后才打开播放器。", + "正在播放": "正在播放", + "等待确认": "等待确认", + "停止": "停止", + "还没有收到投屏": "还没有收到投屏", + "开启接收后，在其他设备上选择 Framely。": "开启接收后，在其他设备上选择 Framely。", + "找到 {0} 台设备": "找到 {0} 台设备", + "附近设备": "附近设备", + "正在查找…": "正在查找…", + "重新查找": "重新查找", + "需要开启网页访问，设备才能读取投屏画面。": "需要开启网页访问，设备才能读取投屏画面。", + "设置网页访问": "设置网页访问", + "先开始头显串流，再选择播放设备。": "先开始头显串流，再选择播放设备。", + "正在投屏": "正在投屏", + "投到此设备": "投到此设备", + "正在查找设备": "正在查找设备", + "未找到播放设备": "未找到播放设备", + "查找附近设备": "查找附近设备", + "确认播放设备已开启 DLNA，并与头显连接同一网络。": "确认播放设备已开启 DLNA，并与头显连接同一网络。", + "有未应用的修改": "有未应用的修改", + "设置已保存": "设置已保存", + "应用后会重新启动串流。": "应用后会重新启动串流。", + "应用修改": "应用修改", + "无法播放，请重新连接。": "无法播放，请重新连接。", + "此浏览器暂不支持全屏。": "此浏览器暂不支持全屏。", + "实时画面": "实时画面", + "连接已断开": "连接已断开", + "正在连接头显": "正在连接头显", + "画面即将开始，请稍候。": "画面即将开始，请稍候。", + "暂时无法观看": "暂时无法观看", + "确认头显已开始串流，然后重新连接。": "确认头显已开始串流，然后重新连接。", + "实时": "实时", + "取景框，使用方向键移动": "取景框，使用方向键移动", + "缩小取景范围": "缩小取景范围", + "视野": "视野", + "扩大取景范围": "扩大取景范围", + "重置取景": "重置取景", + "拖动取景框移动画面，拖动四角调整范围。也可以用方向键移动。": "拖动取景框移动画面，拖动四角调整范围。也可以用方向键移动。", + "画面与声音": "画面与声音", + "投到设备": "投到设备", + "头显与局域网设备": "头显与局域网设备", + "把头显画面分享给同一网络中的手机或电脑。": "把头显画面分享给同一网络中的手机或电脑。", + "选择画面来源，直接调整取景和输出画质。": "选择画面来源，直接调整取景和输出画质。", + "接收其他设备的音频和视频，在头显中打开。": "接收其他设备的音频和视频，在头显中打开。", + "选择同一网络中的设备，播放当前头显画面。": "选择同一网络中的设备，播放当前头显画面。", + "包含透视、Steam 菜单和空间窗口。": "包含透视、Steam 菜单和空间窗口。", + "使用 SteamVR 提供的镜像画面。": "使用 SteamVR 提供的镜像画面。", + "开启声音": "开启声音", + "静音": "静音", + "完整画面": "完整画面", + "填满屏幕": "填满屏幕", + "退出全屏": "退出全屏", + "全屏": "全屏" } } diff --git a/ui/src/cast-framing-editor.tsx b/ui/src/cast-framing-editor.tsx new file mode 100644 index 0000000..b2a1245 --- /dev/null +++ b/ui/src/cast-framing-editor.tsx @@ -0,0 +1,49 @@ +import React,{useEffect,useRef} from 'react'; +import {IconMinus,IconPlus,IconRestore} from '@tabler/icons-react'; +import {CastVideo} from './cast-viewer'; +import {clamp,fitFrame,frameFromSettings,settingsFromFrame,type FrameRect,type ReferenceView} from './cast-framing'; +import type {CastSettings} from './casting'; +import {t} from './i18n'; + +export function FramingEditor({settings,view,onChange,onError}:{settings:CastSettings;view:ReferenceView|null;onChange:(patch:Partial)=>void;onError:(error:string)=>void}){ + const stage=useRef(null),box=useRef(null),zoom=useRef(null),readout=useRef(null); + const current=useRef({x:0,y:0,width:1,height:1}),gesture=useRef<{x:number;y:number;rect:FrameRect;corner:string|null}|null>(null); + const aspect=settings.width/settings.height; + function draw(rect:FrameRect){ + current.current=rect; + if(box.current){box.current.style.left=`${rect.x*100}%`;box.current.style.top=`${rect.y*100}%`;box.current.style.width=`${rect.width*100}%`;box.current.style.height=`${rect.height*100}%`;} + if(view){const fov=settingsFromFrame(rect,view).horizontalFov;if(zoom.current)zoom.current.value=String(fov);if(readout.current)readout.current.textContent=`${Math.round(fov)}°`;} + } + function commit(){if(view)onChange(settingsFromFrame(current.current,view));} + useEffect(()=>{if(view)draw(frameFromSettings(settings,aspect,view));},[view?.tanHalfHorizontal,view?.centerTanX,view?.centerTanY,aspect,settings.horizontalFov,settings.centerX,settings.centerY]); + function scale(width:number){if(!view)return;const r=current.current;const next=fitFrame({...r,width,x:r.x+(r.width-width)/2,y:r.y+(r.height-width*view.aspect/aspect)/2},aspect,view.aspect,Math.tan(5*Math.PI/180)/view.tanHalfHorizontal);draw(next);} + function down(e:React.PointerEvent){if(!view||e.button!==0)return;e.preventDefault();e.currentTarget.setPointerCapture(e.pointerId);gesture.current={x:e.clientX,y:e.clientY,rect:{...current.current},corner:(e.target as HTMLElement).closest('[data-corner]')?.dataset.corner??null};} + function move(e:React.PointerEvent){const g=gesture.current,bounds=stage.current?.getBoundingClientRect();if(!g||!bounds||!view)return;const dx=(e.clientX-g.x)/bounds.width,dy=(e.clientY-g.y)/bounds.height; + if(!g.corner){draw(fitFrame({...g.rect,x:g.rect.x+dx,y:g.rect.y+dy},aspect,view.aspect));return;} + const left=g.corner.includes('w'),top=g.corner.includes('n'),ratio=aspect/view.aspect; + const delta=Math.abs(dx)>Math.abs(dy*ratio)?dx*(left?-1:1):dy*ratio*(top?-1:1); + const anchorX=left?g.rect.x+g.rect.width:g.rect.x,anchorY=top?g.rect.y+g.rect.height:g.rect.y; + const maximum=Math.min(left?anchorX:1-anchorX,(top?anchorY:1-anchorY)*ratio); + const width=clamp(g.rect.width+delta,Math.min(maximum,Math.tan(5*Math.PI/180)/view.tanHalfHorizontal),maximum),height=width/ratio; + draw({x:left?anchorX-width:anchorX,y:top?anchorY-height:anchorY,width,height}); + } + function end(e:React.PointerEvent){if(!gesture.current)return;gesture.current=null;if(e.currentTarget.hasPointerCapture(e.pointerId))e.currentTarget.releasePointerCapture(e.pointerId);commit();} + function keyboard(e:React.KeyboardEvent){if(!view)return;const delta=e.shiftKey?.05:.01,r=current.current;const changes:Record>={ArrowLeft:{x:r.x-delta},ArrowRight:{x:r.x+delta},ArrowUp:{y:r.y-delta},ArrowDown:{y:r.y+delta}};if(changes[e.key]){e.preventDefault();draw(fitFrame({...r,...changes[e.key]},aspect,view.aspect));commit();}} + const maximum=view?2*Math.atan(Math.min(1,aspect/view.aspect)*view.tanHalfHorizontal)*180/Math.PI:90; + return
+
+ + {view&&
+ } +
+
+ + + + +
+

{t('拖动取景框移动画面，拖动四角调整范围。也可以用方向键移动。')}

+
; +} diff --git a/ui/src/cast-framing.ts b/ui/src/cast-framing.ts new file mode 100644 index 0000000..251a7d7 --- /dev/null +++ b/ui/src/cast-framing.ts @@ -0,0 +1,26 @@ +export type FrameRect={x:number;y:number;width:number;height:number}; +export type ReferenceView={tanHalfHorizontal:number;centerTanX:number;centerTanY:number;aspect:number}; +export const clamp=(v:number,min:number,max:number)=>Math.max(min,Math.min(max,v)); +export const frameRatios=[{label:'16:9',value:16/9},{label:'16:10',value:16/10},{label:'4:3',value:4/3},{label:'1:1',value:1},{label:'9:16',value:9/16}]; +export function framePresets(aspect:number){ + return [1280,1920,2560,3840].map(edge=>aspect>=1?{width:edge,height:Math.round(edge/aspect/2)*2}:{width:Math.round(edge*aspect/2)*2,height:edge}); +} +export function fitFrame(rect:FrameRect,aspect:number,referenceAspect=1,minWidth=.1):FrameRect{ + const ratio=aspect/referenceAspect; + const width=clamp(rect.width,Math.min(minWidth,Math.min(1,ratio)),Math.min(1,ratio)); + const height=width/ratio; + return {width,height,x:clamp(rect.x,0,1-width),y:clamp(rect.y,0,1-height)}; +} +export function frameFromSettings(settings:{horizontalFov:number|null;centerX:number;centerY:number},aspect:number,view:ReferenceView):FrameRect{ + const maxWidth=Math.min(1,aspect/view.aspect); + const width=settings.horizontalFov===null?maxWidth:Math.tan(settings.horizontalFov*Math.PI/360)/view.tanHalfHorizontal; + const height=width*view.aspect/aspect; + const x=.5+(Math.tan(settings.centerX*Math.PI/180)-view.centerTanX)/(2*view.tanHalfHorizontal)-width/2; + const y=.5+(Math.tan(settings.centerY*Math.PI/180)-view.centerTanY)*view.aspect/(2*view.tanHalfHorizontal)-height/2; + return fitFrame({x,y,width,height},aspect,view.aspect,Math.tan(5*Math.PI/180)/view.tanHalfHorizontal); +} +export function settingsFromFrame(rect:FrameRect,view:ReferenceView){ + return {horizontalFov:2*Math.atan(rect.width*view.tanHalfHorizontal)*180/Math.PI, + centerX:Math.atan(view.centerTanX+(rect.x+rect.width/2-.5)*2*view.tanHalfHorizontal)*180/Math.PI, + centerY:Math.atan(view.centerTanY+(rect.y+rect.height/2-.5)*2*view.tanHalfHorizontal/view.aspect)*180/Math.PI}; +} diff --git a/ui/src/cast-player.tsx b/ui/src/cast-player.tsx new file mode 100644 index 0000000..563a6b0 --- /dev/null +++ b/ui/src/cast-player.tsx @@ -0,0 +1,20 @@ +import React,{useEffect,useRef,useState} from 'react'; +import {IconMusic,IconPlayerPause,IconPlayerPlay,IconPlayerStop,IconVolume,IconVolumeOff} from '@tabler/icons-react'; +import {api} from './api'; +import {t} from './i18n'; +type Session={id:string;protocol:string;width?:number;height?:number;error?:string}; +export function CastPlayer(){ + const id=location.pathname.split('/').pop()!,[session,setSession]=useState(null),[status,setStatus]=useState({paused:false,position:0,duration:0,seekable:false}),[error,setError]=useState(''),[volume,setVolume]=useState(1),[seek,setSeek]=useState(null),[busy,setBusy]=useState(false),[visible,setVisible]=useState(true); + const timer=useRef|null>(null),root=useRef(null),video=!!session?.width&&!!session?.height; + function reveal(){setVisible(true);if(timer.current)clearTimeout(timer.current);if(video&&!status.paused)timer.current=setTimeout(()=>{if(!root.current?.querySelector(':focus-visible'))setVisible(false);},3000);} + useEffect(()=>{reveal();return()=>{if(timer.current)clearTimeout(timer.current);};},[video,status.paused]); + useEffect(()=>{let alive=true;const poll=()=>void Promise.all([api('cast.window',{id}),api('cast.control',{id,action:'status'})]).then(([session,status])=>{if(alive){setSession(session);setStatus(status);}}).catch(e=>{if(alive)setError(e.message);});poll();const interval=setInterval(poll,1000);return()=>{alive=false;clearInterval(interval);};},[id]); + async function control(action:string,params:Record={}){setBusy(true);try{await api('cast.control',{id,action,...params});setError('');if(action==='stop')setSession(null);}catch(e){setError(e instanceof Error?e.message:String(e));}finally{setBusy(false);reveal();}} + const time=(seconds:number)=>`${Math.floor(seconds/60)}:${String(Math.floor(seconds%60)).padStart(2,'0')}`; + return
{if(!(e.target as HTMLElement).closest('button,input'))reveal();}} onFocusCapture={reveal}> + {!video&&

{t('音频投屏')}

{session?.protocol??t('正在连接')}

} +
Framely{session?.protocol}
+
{session?.protocol==='DLNA'&&status.seekable&&status.duration>0&&}
setVolume(+e.target.value)} onPointerUp={()=>void control('volume',{volume})} onKeyUp={()=>void control('volume',{volume})}/>
+ {error&&

{error}

} +
; +} diff --git a/ui/src/cast-viewer.tsx b/ui/src/cast-viewer.tsx new file mode 100644 index 0000000..d5813f6 --- /dev/null +++ b/ui/src/cast-viewer.tsx @@ -0,0 +1,55 @@ +import React,{useEffect,useRef,useState} from 'react'; +import {IconArrowLeft,IconPlayerPause,IconPlayerPlay,IconVolume,IconVolumeOff,IconMaximize,IconMinimize,IconRefresh,IconAspectRatio} from '@tabler/icons-react'; +import {t} from './i18n'; + +export function CastVideo({active,onError,controls=true,aspectRatio,videoRef,onState,showStatus=true}:{active:boolean;onError?:(error:string)=>void;controls?:boolean;aspectRatio?:number;videoRef?:React.RefObject;onState?:(state:'connecting'|'playing'|'failed')=>void;showStatus?:boolean}){ + const localVideo=useRef(null),video=videoRef??localVideo,failure=useRef(onError);failure.current=onError; + const [state,setState]=useState<'connecting'|'playing'|'failed'>('connecting'); + useEffect(()=>{onState?.(state);},[state,onState]); + const [ratio,setRatio]=useState(16/9),[retry,setRetry]=useState(0); + useEffect(()=>{ + if(!active)return; + const peer=new RTCPeerConnection(),abort=new AbortController();let resource:string|undefined,closed=false; + setState('connecting'); + peer.addTransceiver('video',{direction:'recvonly'});peer.addTransceiver('audio',{direction:'recvonly'}); + peer.ontrack=event=>{if(video.current){const stream=video.current.srcObject as MediaStream|null??new MediaStream();stream.addTrack(event.track);video.current.srcObject=stream;void video.current.play().catch(()=>{});}}; + peer.onconnectionstatechange=()=>{if(peer.connectionState==='failed'){setState('failed');failure.current?.(t('观看连接失败，请重新连接。'));}}; + void(async()=>{ + try{ + await peer.setLocalDescription(await peer.createOffer()); + if(peer.iceGatheringState!=='complete')await new Promise((resolve,reject)=>{ + const check=()=>{if(peer.iceGatheringState==='complete'){cleanup();resolve();}},cancel=()=>{cleanup();reject(new DOMException('Aborted','AbortError'));}; + const timeout=setTimeout(()=>{cleanup();resolve();},4000); + const cleanup=()=>{clearTimeout(timeout);peer.removeEventListener('icegatheringstatechange',check);abort.signal.removeEventListener('abort',cancel);}; + peer.addEventListener('icegatheringstatechange',check);abort.signal.addEventListener('abort',cancel,{once:true});check(); + }); + const response=await fetch('/cast/whep',{method:'POST',headers:{'Content-Type':'application/sdp'},body:peer.localDescription?.sdp,signal:abort.signal}); + if(!response.ok)throw Error(await response.text()||t('串流尚未就绪，请稍后重试。')); + resource=response.headers.get('Location')??undefined; + if(closed){if(resource)void fetch(resource,{method:'DELETE'});return;} + await peer.setRemoteDescription({type:'answer',sdp:await response.text()}); + }catch(error){if(!abort.signal.aborted){setState('failed');failure.current?.(String(error));}} + })(); + return()=>{closed=true;abort.abort();peer.close();if(resource)void fetch(resource,{method:'DELETE'}).catch(()=>{});if(video.current)video.current.srcObject=null;}; + },[active,retry]); + return
; +} +export function CastViewer(){ + const video=useRef(null),root=useRef(null),timer=useRef|null>(null); + const [key,setKey]=useState(0),[error,setError]=useState(''),[state,setState]=useState<'connecting'|'playing'|'failed'>('connecting'),[visible,setVisible]=useState(true),[paused,setPaused]=useState(false),[muted,setMuted]=useState(true),[volume,setVolume]=useState(1),[fullscreen,setFullscreen]=useState(false),[fill,setFill]=useState(true); + function reveal(){setVisible(true);if(timer.current)clearTimeout(timer.current);if(state==='playing'&&!paused)timer.current=setTimeout(()=>{if(!root.current?.querySelector(':focus-visible'))setVisible(false);},3000);} + useEffect(()=>{reveal();return()=>{if(timer.current)clearTimeout(timer.current);};},[state,paused,key]); + useEffect(()=>{const changed=()=>setFullscreen(!!document.fullscreenElement);document.addEventListener('fullscreenchange',changed);return()=>document.removeEventListener('fullscreenchange',changed);},[]); + useEffect(()=>{const el=video.current;if(!el)return;const play=()=>setPaused(false),pause=()=>setPaused(true);el.addEventListener('play',play);el.addEventListener('pause',pause);return()=>{el.removeEventListener('play',play);el.removeEventListener('pause',pause);};},[key]); + useEffect(()=>{if(video.current){video.current.muted=muted;video.current.volume=volume;}},[muted,volume,key,state]); + function reconnect(){setError('');setPaused(false);setKey(v=>v+1);reveal();} + function togglePlay(){const el=video.current;if(!el)return;if(el.paused)void el.play().catch(()=>setError(t('无法播放，请重新连接。')));else el.pause();reveal();} + async function toggleFullscreen(){try{if(document.fullscreenElement)await document.exitFullscreen();else if(root.current?.requestFullscreen)await root.current.requestFullscreen();else (video.current as HTMLVideoElement&{webkitEnterFullscreen?:()=>void})?.webkitEnterFullscreen?.();}catch{setError(t('此浏览器暂不支持全屏。'));}reveal();} + return
{if(e.pointerType==='mouse')reveal();}} onClick={e=>{if(!(e.target as HTMLElement).closest('button,a,input')){reveal();}}} onFocusCapture={reveal} onBlurCapture={reveal} onKeyDown={e=>{if((e.target as HTMLElement).closest('button,a,input'))return;if(e.code==='Space'){e.preventDefault();togglePlay();}else if(e.key==='Escape')reveal();}}> + +
Framely{t('头显画面')}
{state==='playing'?t('实时画面'):state==='connecting'?t('正在连接'):t('连接已断开')}
+ {state!=='playing'&&
{state==='connecting'?<>

{t('正在连接头显')}

{t('画面即将开始，请稍候。')}

:<>

{t('暂时无法观看')}

{error||t('确认头显已开始串流，然后重新连接。')}

}
} + {error&&state==='playing'&&

{error}

} +
{setVolume(+e.target.value);setMuted(false);reveal();}}/>
{t('实时')}
+
; +} diff --git a/ui/src/casting.css b/ui/src/casting.css new file mode 100644 index 0000000..5d3dcaf --- /dev/null +++ b/ui/src/casting.css @@ -0,0 +1,13 @@ +.cast-workspace{max-width:1060px;margin:auto;padding-bottom:16px;font-size:16px;line-height:1.5}.cast-workspace button,.cast-workspace a,.cast-watch button{display:inline-flex;align-items:center;justify-content:center;gap:9px;white-space:nowrap}.cast-workspace button{min-height:44px;border-radius:8px}.cast-workspace h1{font-size:28px;letter-spacing:-.035em;margin:0;font-weight:650}.cast-workspace h2{font-size:18px;font-weight:600;margin:0 0 18px}.cast-workspace h3{font-size:14px;font-weight:500;color:var(--muted);margin:22px 0 10px}.cast-workspace p{margin:0;color:var(--muted)}.cast-page-heading{display:flex;align-items:center;justify-content:space-between;gap:24px;margin:4px 0 28px}.cast-page-heading p{margin-top:8px;font-size:14px}.cast-status{font-size:13px;white-space:nowrap;color:var(--muted)}.cast-status.live{color:var(--blue)}.cast-error{border-left:3px solid #ed9696;background:#2c2020;padding:14px 18px;margin-bottom:24px;border-radius:0 8px 8px 0}.cast-error p{margin-top:4px;overflow-wrap:anywhere;color:var(--text)} +.cast-video-stage{position:relative;background:#080808;display:grid;place-items:center;overflow:hidden;isolation:isolate}.cast-video-stage video{position:absolute;width:100%;height:100%;object-fit:contain;inset:0}.cast-video-stage>span{position:absolute;color:var(--muted);padding:24px;text-align:center;pointer-events:none}.cast-video-failure{position:absolute;text-align:center;padding:24px}.cast-video-failure button{margin-top:16px}.cast-monitor{position:relative;overflow:hidden;border-radius:12px;border:1px solid var(--line)}.cast-monitor-empty{position:absolute;inset:0;display:flex;align-items:center;justify-content:center;flex-direction:column;padding:24px;text-align:center;pointer-events:none}.cast-monitor-empty svg{color:var(--blue);margin-bottom:22px}.cast-monitor-empty h2{font-size:23px;margin-bottom:8px}.cast-monitor:has(.cast-monitor-empty) .cast-video-stage>span{display:none}.cast-stream-bar{display:flex;justify-content:space-between;align-items:center;gap:20px;padding:20px 0}.cast-stream-bar>div{display:flex;flex-direction:column;gap:4px}.cast-stream-bar strong{font-size:16px;font-weight:500}.cast-stream-bar>div span{font-size:13px;color:var(--muted);display:flex;gap:9px;font-variant-numeric:tabular-nums}.cast-stream-bar i{height:12px;border-left:1px solid var(--line)}.cast-stream-bar button{padding:12px 23px}.cast-share{margin-top:12px;padding:24px 0;border-top:1px solid var(--line)}.cast-share>div:first-child{display:flex;align-items:center;gap:15px}.cast-share h2{margin:0;font-size:16px}.cast-share p{font-size:13px;margin-top:4px}.cast-share-link{display:flex;gap:8px;align-items:center;background:#161616;border:1px solid var(--line);border-radius:8px;padding:8px;margin-top:18px}.cast-share-link code{flex:1;min-width:0;overflow-wrap:anywhere;padding:0 10px;font-size:13px}.cast-icon-link{width:44px;height:44px;color:var(--text)}.cast-disclosure{border-top:1px solid var(--line);margin-top:16px}.cast-disclosure>summary{cursor:pointer;padding:22px 0;color:var(--muted);font-size:14px} +.cast-setting-section{padding:26px 0;border-top:1px solid var(--line)}.cast-setting-section:first-of-type{border-top:0;padding-top:0}.cast-source-options{display:grid;grid-template-columns:1fr 1fr;gap:12px}.cast-source-options button{padding:20px;text-align:left;justify-content:flex-start;background:#1c1c1c;white-space:normal;gap:16px;min-height:108px}.cast-source-options button>span{flex:1;display:flex;flex-direction:column;gap:6px}.cast-source-options strong{font-size:16px;font-weight:600}.cast-source-options small,.cast-choice-group small{font-size:13px;color:var(--muted);font-weight:400}.cast-source-options svg{flex-shrink:0}.cast-source-options button.selected,.cast-choice-group button.selected,.cast-ratio-options button.selected,.cast-resolution-options button.selected{border-color:var(--blue);background:#202830;color:var(--text)}.cast-source-options .selected>svg:first-child{color:var(--blue)}.cast-choice-group{display:flex;flex-wrap:wrap;gap:8px}.cast-choice-group button{min-width:78px;padding:11px 18px;flex-direction:column;gap:3px;background:#1c1c1c;font-size:14px}.cast-choice-group small{font-size:12px}.cast-setting-title{display:flex;align-items:center;justify-content:space-between;gap:16px;margin-bottom:18px}.cast-setting-title h2{display:flex;align-items:center;gap:10px;margin:0}.cast-framing-summary{display:flex;align-items:center;gap:16px;min-height:80px;color:var(--muted)}.cast-framing-summary>span{color:var(--text)}.cast-framing-summary small{display:block;color:var(--muted);font-size:13px;margin-top:4px}.cast-ratio-options{display:grid;grid-template-columns:repeat(5,1fr);gap:8px}.cast-ratio-options button{flex-direction:column;gap:12px;height:100px;background:#1c1c1c}.cast-ratio-options strong{font-size:13px;font-weight:500}.cast-ratio-shape{display:block;height:28px;max-width:55px;border:1.5px solid var(--muted);border-radius:2px}.selected .cast-ratio-shape{border-color:var(--blue)}.cast-resolution-options{display:grid;grid-template-columns:repeat(4,1fr);gap:8px}.cast-resolution-options button{font-size:13px;padding:13px 8px;background:#1c1c1c;font-variant-numeric:tabular-nums}.cast-help{font-size:13px;margin-top:12px!important}.cast-bitrate{margin-top:26px;max-width:620px}.cast-bitrate label{display:flex;justify-content:space-between;margin-bottom:14px;font-size:14px}.cast-bitrate output{font-variant-numeric:tabular-nums;color:var(--blue)}.cast-bitrate input{width:100%;accent-color:var(--blue)}.cast-bitrate>div{display:flex;justify-content:space-between;font-size:12px;color:var(--muted);margin-top:8px}.cast-save{display:flex;align-items:center;justify-content:space-between;gap:16px;position:sticky;bottom:-24px;z-index:5;background:#212121;border-top:1px solid var(--line);padding:18px 0;margin-top:18px}.cast-save span{font-size:14px}.cast-save small{display:block;font-size:12px;color:var(--muted);margin-top:4px}.cast-save button{padding:12px 22px} +.cast-framing-stage{position:relative;aspect-ratio:1;max-width:640px;margin:auto;overflow:hidden;border-radius:12px;background:#080808;touch-action:none}.cast-framing-stage .cast-video-stage{height:100%;width:100%}.cast-crop-box{position:absolute;cursor:move;touch-action:none;border:2px solid var(--blue);box-shadow:0 0 0 1000px rgb(0 0 0/.58);border-radius:2px;box-sizing:border-box;outline-offset:5px}.cast-crop-grid{position:absolute;inset:0;pointer-events:none;background:linear-gradient(to right,transparent 33%,rgb(255 255 255/.3) 33%,rgb(255 255 255/.3) 33.3%,transparent 33.3%,transparent 66.6%,rgb(255 255 255/.3) 66.6%,rgb(255 255 255/.3) 66.9%,transparent 66.9%),linear-gradient(to bottom,transparent 33%,rgb(255 255 255/.3) 33%,rgb(255 255 255/.3) 33.3%,transparent 33.3%,transparent 66.6%,rgb(255 255 255/.3) 66.6%,rgb(255 255 255/.3) 66.9%,transparent 66.9%)}.cast-crop-handle{position:absolute;width:32px;height:32px;border:3px solid #fff;box-sizing:border-box}.cast-crop-handle.nw{left:-2px;top:-2px;border-right:0;border-bottom:0;cursor:nwse-resize}.cast-crop-handle.ne{right:-2px;top:-2px;border-left:0;border-bottom:0;cursor:nesw-resize}.cast-crop-handle.sw{left:-2px;bottom:-2px;border-right:0;border-top:0;cursor:nesw-resize}.cast-crop-handle.se{right:-2px;bottom:-2px;border-left:0;border-top:0;cursor:nwse-resize}.cast-zoom-controls{display:flex;gap:8px;align-items:center;margin-top:16px}.cast-zoom-controls>button{padding:10px}.cast-zoom-controls>label{flex:1;display:flex;gap:12px;align-items:center;font-size:13px}.cast-zoom-controls input{flex:1;width:100%;min-width:0;accent-color:var(--blue)}.cast-zoom-controls output{width:36px;text-align:right;font-variant-numeric:tabular-nums}.cast-editor-actions{display:flex;align-items:center;gap:10px;margin-top:18px}.cast-editor-actions p{flex:1;font-size:12px}.cast-name-field{display:flex;flex-direction:column;gap:10px;max-width:480px}.cast-name-field input{width:100%;margin:0;font-size:16px}.cast-name-field small{font-size:13px;color:var(--muted)}.cast-protocols{display:grid;grid-template-columns:1fr 1fr;gap:24px}.cast-protocols>section{padding:24px 0;border-top:1px solid var(--line)}.cast-protocols svg{color:var(--blue);margin-bottom:12px}.cast-protocols h2{margin-bottom:5px;font-size:22px}.cast-protocols p{font-size:13px;margin-bottom:20px}.cast-consent-note{display:flex;gap:10px;align-items:center;padding:12px 0 24px;font-size:13px}.cast-session-list article{display:flex;align-items:center;gap:15px;padding:16px 0}.cast-session-list article>div{flex:1;display:flex;flex-direction:column;gap:4px}.cast-session-list span{font-size:13px;color:var(--muted)}.cast-empty-inline{padding:30px 0;display:flex;flex-direction:column;gap:8px;color:var(--muted)}.cast-empty-inline p{color:var(--text);margin-top:8px}.cast-empty-inline span{font-size:13px}.cast-device-toolbar{display:flex;justify-content:space-between;align-items:center;margin-bottom:22px;color:var(--muted);font-size:14px}.cast-context-note{display:flex;gap:12px;align-items:center;background:#202830;border-radius:8px;padding:16px;margin:12px 0;font-size:13px}.cast-context-note>span{flex:1}.cast-context-note a{color:var(--blue);text-decoration:none}.cast-context-note svg{flex-shrink:0}.cast-device-list{margin-top:24px}.cast-device-list article{display:flex;align-items:center;gap:20px;padding:24px 0;border-top:1px solid var(--line)}.cast-device-list article>div:nth-child(2){flex:1;min-width:0}.cast-device-list h2{font-size:17px;margin:0 0 5px;overflow-wrap:anywhere}.cast-device-list span{font-size:13px;color:var(--muted)}.cast-device-list button span{color:inherit}.cast-device-icon{width:64px;height:64px;background:#1c1c1c;border-radius:12px;display:grid;place-items:center;color:var(--blue);flex-shrink:0}.cast-empty{display:flex;align-items:center;flex-direction:column;text-align:center;padding:80px 24px;color:var(--muted)}.cast-empty h2{margin:24px 0 10px;color:var(--text)}.cast-empty p{font-size:14px;max-width:350px}.cast-device-skeleton>div{display:flex;gap:20px;align-items:center;height:112px}.cast-device-skeleton span{width:64px;height:64px;border-radius:12px;background:#292929}.cast-device-skeleton i{height:14px;width:40%;background:#292929;border-radius:4px}.cast-device-skeleton b{height:44px;width:110px;background:#292929;border-radius:8px;margin-left:auto} +.cast-watch{position:fixed;inset:0;height:100dvh;width:100%;background:#080808;color:#f0f1f3;overflow:hidden;font-size:14px;isolation:isolate;outline:none}.cast-watch .cast-video-stage{position:absolute;inset:0;width:100%;height:100%;aspect-ratio:auto!important}.cast-watch.video-fill video{object-fit:cover}.cast-watch-overlay{position:absolute;left:0;right:0;z-index:2;opacity:0;pointer-events:none;transition:opacity .22s ease,transform .22s ease}.controls-visible .cast-watch-overlay,.cast-watch:has(:focus-visible) .cast-watch-overlay{opacity:1;pointer-events:auto;transform:translateY(0)}.cast-watch-top{top:0;display:flex;align-items:center;gap:16px;padding:calc(20px + env(safe-area-inset-top)) calc(24px + env(safe-area-inset-right)) 50px calc(24px + env(safe-area-inset-left));background:linear-gradient(rgb(0 0 0/.8),transparent);transform:translateY(-8px)}.cast-watch-top>div{display:flex;flex-direction:column;gap:2px}.cast-watch-top strong{font-size:16px;font-weight:650;letter-spacing:-.03em}.cast-watch-top>div span{color:#b6b8bc;font-size:12px}.cast-watch-connection{margin-left:auto;font-size:12px;color:#d8dce0}.cast-watch-icon{width:44px;height:44px;flex-shrink:0;background:rgb(20 20 20/.7);border:1px solid rgb(255 255 255/.15);border-radius:8px;color:#f0f1f3;padding:0;cursor:pointer;text-decoration:none;display:inline-flex;align-items:center;justify-content:center}.cast-watch-icon:hover{background:rgb(60 60 60/.9);border-color:rgb(255 255 255/.4)}.cast-watch-icon:disabled{opacity:.4;cursor:default}.cast-watch-icon:focus-visible,.cast-watch input:focus-visible{outline:2px solid var(--blue);outline-offset:3px}.cast-watch-bottom{bottom:0;padding:64px calc(24px + env(safe-area-inset-right)) calc(20px + env(safe-area-inset-bottom)) calc(24px + env(safe-area-inset-left));background:linear-gradient(transparent,rgb(0 0 0/.82));transform:translateY(8px)}.cast-watch-controls{display:flex;align-items:center;gap:12px;max-width:1320px;margin:auto}.cast-watch-sound{display:flex;align-items:center;gap:12px}.cast-watch-sound input{width:96px;height:4px;accent-color:var(--blue)}.cast-watch-live{margin-left:10px;font-size:12px;color:#c4c7cb}.cast-watch-tools{display:flex;gap:8px;margin-left:auto}.cast-watch-message{position:absolute;inset:0;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;padding:100px 32px;z-index:1;pointer-events:none}.cast-watch-message h1{font-size:25px;letter-spacing:-.03em;font-weight:600;margin:22px 0 10px}.cast-watch-message p{color:#b6b8bc;max-width:440px;overflow-wrap:anywhere;font-size:14px;margin:0}.cast-watch-message button{pointer-events:auto;margin-top:24px;min-height:44px;font-size:14px}.cast-spinner{width:30px;height:30px;border:2px solid #333;border-top-color:var(--blue);border-radius:50%;animation:cast-spin .8s linear infinite}.cast-watch-toast{position:absolute;bottom:120px;left:50%;transform:translateX(-50%);padding:12px 18px;max-width:80%;border-radius:8px;background:#292929;z-index:3}.cast-watch:not(.controls-visible):not(:has(:focus-visible)){cursor:none}@keyframes cast-spin{to{transform:rotate(360deg)}} +.cast-player{height:100dvh;display:flex;flex-direction:column;background:#000;overflow:hidden}.cast-player-video{flex:1}.cast-player-audio{flex:1;text-align:center;padding:16px}.cast-player footer{height:112px;flex-shrink:0;display:flex;align-items:center;gap:16px;padding:16px;background:#212121;box-sizing:border-box}.cast-player footer label{display:flex;align-items:center;gap:12px;flex:1}.cast-player input[type=range]{width:100%} +@media(max-width:767px){.cast-workspace h1{font-size:25px}.cast-page-heading{align-items:flex-start;gap:12px;margin-bottom:22px}.cast-status{font-size:11px;margin-top:8px}.cast-page-heading p{font-size:13px}.cast-source-options{grid-template-columns:1fr}.cast-source-options button{min-height:88px;padding:16px}.cast-resolution-options{grid-template-columns:repeat(2,1fr)}.cast-choice-group button{flex:1;min-width:62px;padding:10px 12px}.cast-ratio-options{gap:6px}.cast-ratio-options button{height:86px;padding:8px 3px}.cast-ratio-shape{height:24px;max-width:42px}.cast-share-link button span{display:none}.cast-share-link code{font-size:11px}.cast-monitor-empty svg{width:32px;height:32px;margin-bottom:12px}.cast-monitor-empty h2{font-size:18px}.cast-monitor-empty p{font-size:12px}.cast-stream-bar{gap:12px}.cast-stream-bar button{padding:11px 14px}.cast-save{bottom:-16px}.cast-editor-actions{flex-wrap:wrap;justify-content:flex-end}.cast-editor-actions p{flex-basis:100%}.cast-zoom-controls{flex-wrap:wrap}.cast-zoom-controls>label{min-width:170px}.cast-zoom-controls>button:last-child{margin-left:auto;font-size:12px}.cast-protocols{grid-template-columns:1fr;gap:0}.cast-context-note{flex-wrap:wrap}.cast-context-note a{margin-left:35px}.cast-context-note>span{flex-basis:70%}.cast-device-list article{gap:14px;flex-wrap:wrap}.cast-device-icon{width:52px;height:52px}.cast-device-list article button{margin-left:66px}.cast-empty{padding:50px 16px}.cast-watch-top{gap:12px;padding:calc(12px + env(safe-area-inset-top)) calc(12px + env(safe-area-inset-right)) 40px calc(12px + env(safe-area-inset-left))}.cast-watch-bottom{padding:48px calc(12px + env(safe-area-inset-right)) calc(12px + env(safe-area-inset-bottom)) calc(12px + env(safe-area-inset-left))}.cast-watch-controls{gap:8px}.cast-watch-sound input{display:none}.cast-watch-live{margin-left:0}.cast-watch-tools{gap:6px}.cast-watch-message h1{font-size:22px}} +@media(max-width:359px){.cast-watch-live{display:none}.cast-watch-connection{font-size:10px}.cast-watch-icon{width:40px;height:44px}} +@media(max-height:480px) and (orientation:landscape){.cast-watch-top{padding-top:calc(8px + env(safe-area-inset-top));padding-bottom:24px}.cast-watch-bottom{padding-top:28px;padding-bottom:calc(8px + env(safe-area-inset-bottom))}.cast-watch-message{padding:66px 24px}.cast-watch-message h1{margin-top:10px}.cast-watch-message button{margin-top:14px}} +@media(prefers-reduced-motion:reduce){.cast-watch-overlay{transition:none}.cast-spinner{animation:none;border-color:var(--blue)}} +.cast-player{position:relative;display:block;height:100dvh;overflow:hidden;background:transparent;padding:0}.cast-player-audio{background:#151515;text-align:left}.cast-audio-art{position:absolute;inset:0;display:flex;align-items:center;justify-content:center;flex-direction:column;padding-bottom:100px}.cast-audio-art svg{color:var(--blue);margin-bottom:28px}.cast-audio-art h1{font-size:28px}.cast-audio-art p{color:var(--muted);font-size:16px;margin-top:10px}.cast-player-heading{position:absolute;left:0;right:0;top:0;padding:22px 28px 52px;background:linear-gradient(rgb(0 0 0/.8),transparent);display:flex;align-items:center;gap:14px;font-size:16px;opacity:0;transition:opacity .2s}.cast-player-heading span{font-weight:600}.cast-player-heading strong{color:var(--muted);font-size:14px;font-weight:400}.cast-player footer{position:absolute;left:0;right:0;bottom:0;height:auto;display:flex;flex-direction:column;gap:14px;background:linear-gradient(transparent,rgb(0 0 0/.92));padding:54px 24px 20px;opacity:0;pointer-events:none;transition:opacity .2s}.cast-player.controls-visible header,.cast-player.controls-visible footer,.cast-player:has(:focus-visible) header,.cast-player:has(:focus-visible) footer{opacity:1;pointer-events:auto}.cast-player-buttons{display:flex;align-items:center;gap:12px;width:100%}.cast-player-buttons button{min-width:48px;min-height:48px;padding:8px;border-radius:8px;background:rgb(25 25 25/.86)}.cast-player-buttons input{flex:1;min-width:0;max-width:170px;accent-color:var(--blue)}.cast-player-buttons .cast-player-stop{margin-left:auto;font-size:15px;padding:8px 14px}.cast-player footer .cast-player-progress{width:100%;font-size:13px;font-variant-numeric:tabular-nums;gap:12px}.cast-player-progress input{flex:1;min-width:0;accent-color:var(--blue)}.cast-player-error{position:absolute;left:24px;right:24px;bottom:120px;padding:12px 18px;background:#292929;font-size:14px;border-radius:8px}.cast-player-audio .cast-player-heading{background:none}.cast-player-audio footer{background:#1d1d1d;padding:20px 24px}@media(max-width:480px){.cast-player-buttons{gap:8px}.cast-player-buttons .cast-player-stop span{display:none}.cast-player footer{padding-left:16px;padding-right:16px}.cast-player-buttons input{max-width:110px}}@media(prefers-reduced-motion:reduce){.cast-player header,.cast-player footer{transition:none}} +.cast-choice-group small{white-space:normal;line-height:1.45;max-width:100%;text-align:center}.cast-framing-section{scroll-margin-top:16px}.cast-framing-stage{max-width:min(640px,calc(100dvh - 330px))}@media(max-width:767px){.cast-choice-group button{min-width:0}.cast-framing-stage{max-width:100%}.cast-watch-icon{min-height:44px}} +.cast-framing-stage{max-width:min(640px,calc(100dvh - 400px))}.cast-editor-actions{position:sticky;bottom:-24px;background:#151515;padding:14px 0;z-index:6}@media(max-width:767px){.cast-choice-group button>span{white-space:normal}.cast-framing-stage{max-width:100%}.cast-editor-actions{bottom:-16px}} diff --git a/ui/src/casting.tsx b/ui/src/casting.tsx new file mode 100644 index 0000000..4c5a707 --- /dev/null +++ b/ui/src/casting.tsx @@ -0,0 +1,82 @@ +import React,{useEffect,useRef,useState} from 'react'; +import {IconPlayerPlay,IconPlayerStop,IconExternalLink,IconCopy,IconCheck,IconDeviceVisionPro,IconDeviceDesktop,IconCrop,IconEye,IconDeviceTv,IconRefresh,IconCast,IconMusic,IconVideo,IconWifi,IconArrowRight} from '@tabler/icons-react'; +import {NetworkSettings,type NetworkPanel} from './network'; +import {api} from './api'; +import {SwitchRow} from './switch'; +import {t} from './i18n'; +import {CastVideo} from './cast-viewer'; +import {FramingEditor} from './cast-framing-editor'; +import {framePresets,frameRatios,type ReferenceView} from './cast-framing'; +import './casting.css'; + +export type CastSettings={source:'screen'|'steamVR';output:'eye'|'raw';eye:'steamVR'|'left'|'right';width:number;height:number;fps:number;bitrateMbps:number;horizontalFov:number|null;centerX:number;centerY:number;systemAudio:boolean;microphone:boolean;airplay:boolean;dlna:boolean;receiverName:string}; +type CastSession={id:string;protocol:string;accepted:boolean;width?:number;height?:number}; +type Device={id:string;name:string}; +type Runtime={running:boolean;error:string|null;dominantEye:number|null;watchPath:string|null;elapsedSeconds:number|null;preview?:boolean;view?:ReferenceView|null;receiverError?:string|null;sessions?:CastSession[];watchAddresses?:string[]}; +export type CastPage='casting'|'casting-picture'|'casting-receive'|'casting-devices'; +const titles:Record={casting:['网页观看','把头显画面分享给同一网络中的手机或电脑。'],'casting-picture':['画面与声音','选择画面来源，直接调整取景和输出画质。'],'casting-receive':['接收投屏','接收其他设备的音频和视频，在头显中打开。'],'casting-devices':['投到设备','选择同一网络中的设备，播放当前头显画面。']}; +function Choices({label,value,options,onChange,disabled=false}:{label:string;value:T;options:{value:T;label:string;description?:string}[];onChange:(value:T)=>void;disabled?:boolean}){ + return
{options.map((option,i)=>)}
; +} +export function Casting({config,refresh,network,page='casting'}:{config:CastSettings;refresh:()=>Promise;network:NetworkPanel;page?:CastPage}){ + const [draft,setDraft]=useState(config),[runtime,setRuntime]=useState(null),[pending,setPending]=useState(''),[error,setError]=useState(''),[saved,setSaved]=useState(false),[epoch,setEpoch]=useState(0),[devices,setDevices]=useState([]),[sending,setSending]=useState(''),[copied,setCopied]=useState(false),[editing,setEditing]=useState(false),[searched,setSearched]=useState(false); + const beforeFraming=useRef|null>(null),editorActive=useRef(false),deviceScan=useRef(0),alive=useRef(true); + const captureSettings=({airplay,dlna,receiverName,...capture}:CastSettings)=>capture; + const captureDirty=JSON.stringify(captureSettings(draft))!==JSON.stringify(captureSettings(config)),dirty=JSON.stringify(draft)!==JSON.stringify(config),busy=!!pending,screen=draft.source==='screen',eye=screen&&draft.output==='eye'; + const aspect=screen&&draft.output==='raw'?2:draft.width/draft.height; + const ratio=frameRatios.find(r=>Math.abs(r.value-aspect)<.004)?.label; + useEffect(()=>{setDraft(config);},[JSON.stringify(config)]); + useEffect(()=>{alive.current=true;const poll=()=>void api('cast.status').then(v=>{if(alive.current)setRuntime(v);}).catch(e=>{if(alive.current)setError(e.message);});poll();const timer=setInterval(poll,2000);return()=>{alive.current=false;deviceScan.current++;clearInterval(timer);if(editorActive.current)void api('cast.preview.end').catch(()=>{});};},[]); + function change(patch:Partial){setDraft(v=>({...v,...patch}));setSaved(false);} + async function action(name:string,fn:()=>Promise){setPending(name);setError('');try{await fn();setRuntime(await api('cast.status'));}catch(e){setError(e instanceof Error?e.message:String(e));}finally{if(alive.current)setPending('');}} + async function apply(){await api('cast.settings.save',draft);if(editing){await api('cast.preview.end',{apply:true});editorActive.current=false;setEditing(false);}else if(runtime?.running&&captureDirty){await api('cast.stop');await api('cast.start');}await refresh();setSaved(true);setEpoch(v=>v+1);} + async function beginFraming(){beforeFraming.current={horizontalFov:draft.horizontalFov,centerX:draft.centerX,centerY:draft.centerY};setRuntime(await api('cast.preview.start',draft));editorActive.current=true;setEditing(true);setEpoch(v=>v+1);} + async function cancelFraming(){await api('cast.preview.end');editorActive.current=false;setEditing(false);if(beforeFraming.current)change(beforeFraming.current);setEpoch(v=>v+1);} + useEffect(()=>{if(editing){const frame=requestAnimationFrame(()=>document.querySelector('.cast-framing-section')?.scrollIntoView({block:'start',behavior:matchMedia('(prefers-reduced-motion: reduce)').matches?'instant':'smooth'}));return()=>cancelAnimationFrame(frame);}},[editing]); + useEffect(()=>{if(page!=='casting-picture'&&editorActive.current)void action('preview',cancelFraming);},[page]); + async function scan(){const token=++deviceScan.current;setSearched(true);for(let i=0;i<3;i++){const list=await api('cast.devices');if(token!==deviceScan.current||!alive.current)return;setDevices(list);if(i<2)await new Promise(resolve=>setTimeout(resolve,1000));}} + useEffect(()=>{if(page==='casting-devices')void action('scan',scan);},[page]); + async function copy(){const address=['localhost','127.0.0.1','[::1]'].includes(location.hostname)?runtime?.watchAddresses?.[0]??new URL('/cast/watch',location.origin).href:new URL('/cast/watch',location.origin).href;if(navigator.clipboard)await navigator.clipboard.writeText(address);else{const input=document.createElement('textarea');input.value=address;document.body.append(input);input.select();document.execCommand('copy');input.remove();}setCopied(true);setTimeout(()=>setCopied(false),2500);} + const errorMessage=error||(page==='casting-receive'||page==='casting-devices'?runtime?.receiverError:runtime?.error); + const presets=framePresets(aspect); + function chooseAspect(value:number){const edge=Math.max(draft.width,draft.height);const size=value>=1?{width:edge,height:Math.round(edge/value/2)*2}:{width:Math.round(edge*value/2)*2,height:edge};change(size);} + return
+

{t(titles[page][0])}

{t(titles[page][1])}

{page==='casting'&&{pending==='start'?t('正在启动…'):runtime?.running?t('正在串流'):t('未开始串流')}}
+ {errorMessage&&
{t('操作未完成')}

{errorMessage}

} + {page==='casting'&&<> +
{!runtime?.running&&

{t('分享你看到的画面')}

{t('启动后，手机和电脑可以通过网页实时观看。')}

}
+
{t(config.source==='screen'?'屏幕采集':'SteamVR')}{config.width} × {config.height} {config.fps} fps
+ {runtime?.running&&

{t('在其他设备上观看')}

{t('连接同一网络，打开观看地址。')}

{runtime.watchAddresses?.[0]??new URL('/cast/watch',location.origin).href}
} +
{t('网页访问设置')}
+ } + {page==='casting-picture'&&<> +

{t('画面来源')}

+ {([{value:'screen',label:'屏幕采集',description:'包含透视、Steam 菜单和空间窗口。',icon:IconDeviceVisionPro},{value:'steamVR',label:'SteamVR',description:'使用 SteamVR 提供的镜像画面。',icon:IconDeviceDesktop}] as const).map(option=>)} +
+ {screen&&

{t('输出画面')}

change({output})}/>
} + {eye&&

{t('取景')}

{!editing&&}
+ {editing?<>

{t('取景预览期间，共享画面会切换为完整视角。')}

:<>
{draft.horizontalFov===null?t('完整视野'):t('已设置取景范围')}{t('直接在实时画面上移动和缩放取景框。')}

{t('主力眼')}

change({eye})}/>} +
} +

{t('画幅与分辨率')}

{!(screen&&draft.output==='raw')&&<>

{t('画面比例')}

{frameRatios.map(r=>)}
} + {screen&&draft.output==='raw'&&

{t('原始双眼比例由头显屏幕决定。')}

} +

{t('分辨率')}

{presets.map(size=>)}
+ {!presets.some(size=>size.width===draft.width&&size.height===draft.height)&&

{t('当前分辨率：{0} × {1}',{0:draft.width,1:draft.height})}

} +
+

{t('流畅度与画质')}

{t('帧率')}

({value:fps,label:`${fps} fps`}))} onChange={fps=>change({fps})}/>
change({bitrateMbps:+e.target.value})}/>
{t('更省带宽')}{t('画面更清晰')}
+

{t('声音')}

change({systemAudio})}/>change({microphone})}/>
+ } + {page==='casting-receive'&&<> +
+

AirPlay

{t('接收一路音频或屏幕镜像。')}

change({airplay})}/>

DLNA

{t('接收媒体播放地址，可打开多个窗口。')}

change({dlna})}/>
+

{t('收到投屏后先询问，接受后才打开播放器。')}

+

{t('当前投屏')}

{runtime?.sessions?.length?
{runtime.sessions.map(session=>
{session.width?:}
{session.protocol}{session.accepted?t('正在播放'):t('等待确认')}
{session.accepted?:<>}
)}
:

{t('还没有收到投屏')}

{t('开启接收后，在其他设备上选择 Framely。')}
}
+ } + {page==='casting-devices'&&<> +
{devices.length?t('找到 {0} 台设备',{0:devices.length}):t('附近设备')}
+ {!network.enabled&&
{t('需要开启网页访问，设备才能读取投屏画面。')}{t('设置网页访问')}
} + {!runtime?.running&&
{t('先开始头显串流，再选择播放设备。')}{t('网页观看')}
} + {devices.length?
{devices.map(device=>

{device.name}

{sending===device.id?t('正在投屏'):'DLNA'}
)}
:pending==='scan'?
{[0,1,2].map(n=>
)}
:

{searched?t('未找到播放设备'):t('查找附近设备')}

{t('确认播放设备已开启 DLNA，并与头显连接同一网络。')}

} + } + {(page==='casting-picture'||page==='casting-receive')&&!editing&&
{dirty?t('有未应用的修改'):saved?t('已应用'):t('设置已保存')}{captureDirty&&runtime?.running&&{t('应用后会重新启动串流。')}}
} +
; +} diff --git a/ui/src/main.tsx b/ui/src/main.tsx index 5768122..e7da9c8 100644 --- a/ui/src/main.tsx +++ b/ui/src/main.tsx @@ -1,3 +1,6 @@ +import {CastPlayer} from './cast-player'; +import {Casting,type CastSettings,type CastPage} from './casting'; +import {CastViewer} from './cast-viewer'; import {MobilePagePicker,managerPages,useMobileManager,useMobileViewport} from './mobile-manager'; import {FileManager} from './file-manager'; import {TerminalManager} from './terminal'; @@ -27,11 +30,13 @@ import React,{useEffect,useRef,useState} from 'react';import{createRoot}from'rea export type Manifest=Presentation&{id:string;name:string;version:string;author:string;description:string;backend?:{runAs:string;autostart:boolean;memoryLimitMiB?:number};lifecycle?:{runAs?:string};dependencies?:Record;optionalDependencies?:Record;conflicts?:Record;exclusiveResources?:string[];engines?:{framely?:string};ui:{quickPage?:string;windows:Record;launch?:Record;launcherActions?:{id:string;label:string;target:{type:string;entry?:string;window?:string}}[]}}; export type Plugin={manifest:Manifest;enabled:boolean;favorite:boolean;order:number;source?:string;error?:string}; type Source={id:string;name:string;url:string;enabled:boolean;allowHttp:boolean}; -type Status={agreement?:AgreementState;version:string;build?:string;systemUpdate?:any;previousRelease?:string;database:{feedback?:{haptics:boolean;sounds:boolean};launcher?:LauncherConfig;launcherFavorites?:string[];steamFavorites?:number[];launcherOrder?:string[];notificationSettings?:NotificationConfig;language?:string;proxy?:ProxyConfig;networkPanel?:NetworkPanel;plugins:Record;sources:Source[];subscriptions?:Subscription[];sourceOrigins?:Record}>;safeMode:boolean;updateSource?:{url:string}|null;updateChannel?:'stable'|'testing'};notifications:NotificationEntry[];inbox?:NotificationEntry[];notificationBadge?:number;runtime?:Record;running:string[]}; +type Status={agreement?:AgreementState;version:string;build?:string;systemUpdate?:any;previousRelease?:string;database:{feedback?:{haptics:boolean;sounds:boolean};launcher?:LauncherConfig;launcherFavorites?:string[];steamFavorites?:number[];launcherOrder?:string[];notificationSettings?:NotificationConfig;language?:string;proxy?:ProxyConfig;networkPanel?:NetworkPanel;casting?:CastSettings;plugins:Record;sources:Source[];subscriptions?:Subscription[];sourceOrigins?:Record}>;safeMode:boolean;updateSource?:{url:string}|null;updateChannel?:'stable'|'testing'};notifications:NotificationEntry[];inbox?:NotificationEntry[];notificationBadge?:number;runtime?:Record;running:string[]}; const managerMode=location.pathname==='/manager'; const empty:Status={version:'0.4.0',database:{plugins:{},sources:[],safeMode:false},notifications:[],running:[]}; function App(){ + if(location.pathname==='/cast/watch')return ; + if(location.pathname.startsWith('/cast/window/'))return ; const mobile=useMobileManager();useMobileViewport(managerMode&&mobile); const[catalogTarget,setCatalogTarget]=useState<{plugin:string;source?:string}|null>(null);const runtimeLabels:Record={starting:t("初始化中"),running:t("运行中"),stopping:t("停止中"),stopped:t("已停止"),recovering:t("等待恢复"),failed:t("失败"),crashed:t("异常退出"),ready:t("已就绪"),installed:t("已安装"),'waiting-dependency':t("等待依赖")};const [status,setStatus]=useState(empty),[error,setError]=useState(''),[tab,setTab]=useState(managerMode?managerTab(location.hash):'favorites'),[page,setPage]=useState(!managerMode&&location.hash.startsWith('#plugin:')?location.hash.slice(8):null),[busy,setBusy]=useState(false),[query,setQuery]=useState(''),[manage,setManage]=useState(null),[install,setInstall]=useState(null),[confirmRunAs,setConfirmRunAs]=useState(false),[logs,setLogs]=useState(null),[importing,setImporting]=useState(false),[transfer,setTransfer]=useState(null),[downloadRequest,setDownloadRequest]=useState(null),[relationAction,setRelationAction]=useState(null),[reviewAgreement,setReviewAgreement]=useState(false),[agreementDeclined,setAgreementDeclined]=useState(false),[languageFiles,setLanguageFiles]=useState([]),[logExport,setLogExport]=useState(''),[exportingLogs,setExportingLogs]=useState(false); useEffect(()=>{const ticket=install?.ticket;return()=>{if(ticket)void api('job.cancel',{job:ticket}).catch(()=>{});};},[install?.ticket]); @@ -42,7 +47,7 @@ function App(){ const apkPanel=managerMode&&tab.startsWith('apk'); const managerSection=sectionFor(tab); useEffect(()=>{if(managerMode){try{localStorage.setItem('framely.manager.'+sectionFor(tab),tab);if(!location.hash.startsWith('#plugin:'))history.replaceState(null,'',location.pathname+location.search+'#'+tab);}catch{}}},[tab]); - function switchManager(section:ManagerSection){let target: string=({plugins:'installed',apk:'apk',terminal:'terminal',files:'files',settings:'settings'})[section];try{const remembered=localStorage.getItem('framely.manager.'+section);if(remembered&§ionFor(remembered)===section&&managerTab('#'+remembered)===remembered)target=remembered;}catch{}setPage(null);setManage(null);setTab(target);history.replaceState(null,'',location.pathname+location.search+'#'+target);} + function switchManager(section:ManagerSection){let target: string=({plugins:'installed',apk:'apk',terminal:'terminal',files:'files',casting:'casting',settings:'settings'})[section];try{const remembered=localStorage.getItem('framely.manager.'+section);if(remembered&§ionFor(remembered)===section&&managerTab('#'+remembered)===remembered)target=remembered;}catch{}setPage(null);setManage(null);setTab(target);history.replaceState(null,'',location.pathname+location.search+'#'+target);} async function exportLogs(){ setExportingLogs(true);setLogExport('');setError(''); try{setLogExport(await exportDiagnosticLogs()); @@ -92,7 +97,7 @@ function App(){ if(parts[1]==='notifications')return ; if(windowPlugin){const p=status.database.plugins[windowPlugin];return
{p?.enabled&&!status.database.safeMode?:

{t("插件已停用或正在启动。")}

}{error&&

{error}

}
;} return
{managerMode?<>{managerSection==='plugins'&&}{mobile&&key==='catalog'?void loadCatalog():(setPage(null),setTab(key),history.replaceState(null,'',location.pathname+location.search+'#'+key))}/> }:<>}
{error&&

{error}

}{status.database.safeMode&&

{t("安全模式：第三方插件暂停运行。")}

} - {selected?<>
{selected.manifest.name}{managerMode&&}
{selected.enabled&&!status.database.safeMode&&selected.manifest.ui.quickPage?:

{t("插件已停用、处于安全模式或没有快捷页面。")}

}:
+ {selected?<>
{selected.manifest.name}{managerMode&&}
{selected.enabled&&!status.database.safeMode&&selected.manifest.ui.quickPage?:

{t("插件已停用、处于安全模式或没有快捷页面。")}

}:
{(tab==='favorites'||tab==='installed')&&<> {managerMode&&

{t("已安装")}

{plugins.length}{t("个插件，")}{plugins.filter(p=>p.enabled).length}{t("个已启用")}

setQuery(e.target.value)}/>
} {managerMode&&
{t("插件")}{t("状态")}{t("版本")}{t("操作")}
}
{listedPlugins.map(p=>
@@ -105,6 +110,7 @@ function App(){ {tab==='notifications'&&!managerMode&&<>}/>{readError&&

{readError}

}} {managerMode&&(filesVisited||tab==='files')&&} {tab==='terminal'&&managerMode&&} + {managerSection==='casting'&&managerMode&&status.database.casting&&} {apkPanel&&setTab(view==='apps'?'apk':'apk-'+view)}/>} {tab==='launcher-settings'&&managerMode&&<>

{t('启动台')}

{t('启动方式、菜单与分类。')}

} {tab==='notification-settings'&&managerMode&&} diff --git a/ui/src/manager-switcher.tsx b/ui/src/manager-switcher.tsx index 7812b4e..a481742 100644 --- a/ui/src/manager-switcher.tsx +++ b/ui/src/manager-switcher.tsx @@ -1,15 +1,16 @@ import React from 'react'; -import {IconPuzzle,IconBrandAndroid,IconTerminal2,IconFolder,IconSettings} from '@tabler/icons-react'; +import {IconPuzzle,IconBrandAndroid,IconTerminal2,IconFolder,IconSettings,IconCast} from '@tabler/icons-react'; import {t} from './i18n'; -export type ManagerSection='plugins'|'apk'|'terminal'|'files'|'settings'; +export type ManagerSection='plugins'|'apk'|'terminal'|'files'|'casting'|'settings'; export function sectionFor(tab:string):ManagerSection { + if(['casting','casting-picture','casting-receive','casting-devices'].includes(tab))return 'casting'; if(tab.startsWith('apk'))return 'apk'; if(tab==='terminal'||tab==='files')return tab; return ['settings','launcher-settings','about','updates'].includes(tab)?'settings':'plugins'; } -export function managerTab(hash:string){const key=hash.slice(1);return ['installed','catalog','sources','notification-settings','settings','launcher-settings','about','updates','terminal','files','apk','apk-containers','apk-cleanup','apk-settings'].includes(key)?key:'installed';} +export function managerTab(hash:string){const key=hash.slice(1);return ['casting','casting-picture','casting-receive','casting-devices','installed','catalog','sources','notification-settings','settings','launcher-settings','about','updates','terminal','files','apk','apk-containers','apk-cleanup','apk-settings'].includes(key)?key:'installed';} export function ManagerSwitcher({section,onChange}:{section:ManagerSection;onChange:(section:ManagerSection)=>void}) { return
- {([['plugins','插件'],['apk','APK'],['terminal','终端'],['files','文件'],['settings','设置']] as const).map(([key,label])=>)} + {([['plugins','插件'],['apk','APK'],['terminal','终端'],['files','文件'],['casting','投屏'],['settings','设置']] as const).map(([key,label])=>)}
; } diff --git a/ui/src/mobile-manager.css b/ui/src/mobile-manager.css index b089ada..ff174c0 100644 --- a/ui/src/mobile-manager.css +++ b/ui/src/mobile-manager.css @@ -19,7 +19,7 @@ .manager .shell-header>.icon-button{display:none} .manager .shell-header .manager-install-button{font-size:0;min-height:44px;width:44px;padding:10px;border-radius:12px;order:3} .manager .shell-header .manager-install-button svg{width:22px;height:22px} - .manager .shell-header .manager-switcher{position:fixed;z-index:30;left:0;right:0;bottom:0;width:100%;max-width:none;display:grid;grid-template-columns:repeat(5,minmax(0,1fr));border:0;border-top:1px solid #ffffff18;border-radius:0;background:#202022f5;gap:0;padding:6px 6px calc(6px + env(safe-area-inset-bottom,0px));overflow:visible} + .manager .shell-header .manager-switcher{position:fixed;z-index:30;left:0;right:0;bottom:0;width:100%;max-width:none;display:grid;grid-template-columns:repeat(6,minmax(0,1fr));border:0;border-top:1px solid #ffffff18;border-radius:0;background:#202022f5;gap:0;padding:6px 6px calc(6px + env(safe-area-inset-bottom,0px));overflow:visible} .manager .manager-switcher button{display:flex;flex-direction:column;align-items:center;justify-content:center;min-height:56px;padding:5px 2px;gap:3px;font-size:11px;border-radius:12px;color:#b2b2ba} .manager .manager-switcher button.active{background:transparent;color:#a2c7df} .manager-section-icon{display:flex;padding:3px 12px;border-radius:999px} diff --git a/ui/src/mobile-manager.tsx b/ui/src/mobile-manager.tsx index 93c8858..4df9f3d 100644 --- a/ui/src/mobile-manager.tsx +++ b/ui/src/mobile-manager.tsx @@ -12,9 +12,9 @@ export function MobileSheet({title,onClose,children}:{title:string;onClose:()=>v const key=(e:KeyboardEvent)=>{if(e.key==='Escape'){if((e.target as HTMLElement)?.closest('.framely-select')?.querySelector('[aria-expanded="true"]'))return;e.preventDefault();e.stopPropagation();closeRef.current();}if(e.key==='Tab'){const list=items(),first=list[0],last=list.at(-1);if(e.shiftKey&&document.activeElement===first){e.preventDefault();last?.focus();}else if(!e.shiftKey&&document.activeElement===last){e.preventDefault();first?.focus();}}};document.addEventListener('keydown',key,true);return()=>{document.removeEventListener('keydown',key,true);if(previous?.isConnected&&!document.querySelector('.modal[aria-modal="true"]'))previous.focus();};},[]); return
{if(e.target===e.currentTarget)onClose();}}>

{title}

{children}
; } -export const managerPages=(section:ManagerSection):[string,string,string][]=>section==='settings'?[['settings',t('通用'),'settings'],['launcher-settings',t('启动台'),'launcher'],['updates',t('更新'),'refresh'],['about',t('关于'),'info']]:section==='apk'?[['apk',t('应用'),'download'],['apk-containers',t('容器'),'plugins'],['apk-cleanup',t('数据清理'),'settings'],['apk-settings',t('设置'),'settings']]:section==='plugins'?[['installed',t('已安装'),'plugins'],['catalog',t('插件库'),'download'],['sources',t('插件源导航'),'link'],['notification-settings',t('通知设置导航'),'bell']]:[]; +export const managerPages=(section:ManagerSection):[string,string,string][]=>section==='settings'?[['settings',t('通用'),'settings'],['launcher-settings',t('启动台'),'launcher'],['updates',t('更新'),'refresh'],['about',t('关于'),'info']]:section==='apk'?[['apk',t('应用'),'download'],['apk-containers',t('容器'),'plugins'],['apk-cleanup',t('数据清理'),'settings'],['apk-settings',t('设置'),'settings']]:section==='casting'?[['casting',t('网页观看'),'connection'],['casting-picture',t('画面与声音'),'settings'],['casting-receive',t('接收投屏'),'download'],['casting-devices',t('投到设备'),'connection']]:section==='plugins'?[['installed',t('已安装'),'plugins'],['catalog',t('插件库'),'download'],['sources',t('插件源导航'),'link'],['notification-settings',t('通知设置导航'),'bell']]:[]; export function MobilePagePicker({section,tab,onChange}:{section:ManagerSection;tab:string;onChange:(tab:string)=>void}){ - const [open,setOpen]=useState(false),pages=managerPages(section),label=pages.find(p=>p[0]===tab)?.[1]??t(section==='terminal'?'终端':'文件'); + const [open,setOpen]=useState(false),pages=managerPages(section),label=pages.find(p=>p[0]===tab)?.[1]??t(section==='terminal'?'终端':section==='casting'?'投屏':'文件'); useEffect(()=>setOpen(false),[section,tab]); return
{open&&setOpen(false)}>
{pages.map(([key,label])=>)}
}
; } diff --git a/ui/src/notifications.tsx b/ui/src/notifications.tsx index cfa5414..9f43b96 100644 --- a/ui/src/notifications.tsx +++ b/ui/src/notifications.tsx @@ -43,7 +43,7 @@ export function NotificationCenter({items,inbox=false,refresh,plugins={},safeMod return
{inbox&&

{t('通知')}

{t('收件箱')}{visible.length}

} {inbox&&controls} - {visible.map(entry=>)} + {visible.map(entry=>)} {inbox&&!visible.length&&

{t('收件箱为空')}

{t('需要保留的通知会出现在这里。')}

}
; }