From d22cc04cb3dfdc5ced576cd0ccb3b6ec65486c8a Mon Sep 17 00:00:00 2001 From: Toastconcern Date: Wed, 21 Oct 2026 19:39:36 -0500 Subject: [PATCH] Frame Home as the real SteamVR environment on a PC (Windows too), not a home app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the Steam Frame a layer inside SteamVR's compositor draws the home where it would draw Aurora, so the home is the environment itself: at boot, during game loads, behind the Steam menu, with the controllers and passthrough, and it steps aside for a game the instant it opens. On a PC this was faked with SteamVR's home-app slot and a flat 360 photo in the gaps, which launched and handed off like an app and showed a picture, not the live home. This does it properly on any PC that runs SteamVR, the Frame's way: - Windows: the compositor is Direct3D 11, so the layer is a proxy d3d11.dll beside the compositor (layer/ frame_home_layer_d3d11.cpp, built from the .def that forwards every other export to the real d3d11.dll, kept as fh_d3d11real.dll). It wraps the compositor's device, finds Aurora's draws by their shader bytecode and draws the home over the sky, leaving the floor and stars off; everything else passes straight through. Pure pass-through in any process that isn't vrcompositor. - The home's frames cross from Frame Home (Vulkan) to the compositor (D3D11): the layer owns kSlots shared textures and hands Frame Home their NT handles over a named pipe; Frame Home imports them (VK_KHR_external_memory_win32) and renders into them. No cross-API lock — a slot is published only once the GPU has finished it and the four rotate, as on Linux. The shared State stays Frame Home's. - The Environments panel runs frame-home --system and keeps it alive (one at a time, a pid file across panel restarts), the same systemEnv path as the Frame. The installer places/removes the proxy and marks the layer with a root/layer folder, in place of the old home-app registration and background pictures. - Safe like the Vulkan layer: no home, or one stale over half a second, and the compositor draws Aurora as usual; the layer turns itself off after two unhealthy compositor starts, and root/layer/off turns it off by hand. Linux keeps its Vulkan layer (unchanged). Verified without a headset on the null driver: the proxy loads and wraps the compositor without crashing, recognises Aurora's shaders, and the frame hand-over completes (Frame Home connects, imports the textures, renders). The compositor only draws the environment while presenting to a headset, so the final on-headset image is the one step the null driver can't exercise. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build.yml | 4 +- CMakeLists.txt | 10 + docs/pc-steamvr.md | 42 +- frame/pc/install-frame-home-pc.ps1 | 138 +++---- layer/d3d11_proxy.def | 57 +++ layer/frame_home_layer_d3d11.cpp | 623 +++++++++++++++++++++++++++++ scripts/build-studio.sh | 9 +- src/env_export.h | 180 ++++++++- src/env_share.h | 42 +- src/home_platform.h | 17 + src/hsl_renderer.cpp | 80 ++++ src/hsl_renderer.h | 16 + src/main.cpp | 27 +- src/switcher.cpp | 28 +- 14 files changed, 1156 insertions(+), 117 deletions(-) create mode 100644 layer/d3d11_proxy.def create mode 100644 layer/frame_home_layer_d3d11.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bcdbd6f..9c1beb0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,9 +54,9 @@ jobs: run: | export PATH="$HOME/llvm-mingw/bin:$PATH" cmake -S . -B build/pc-win64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DFRAME_HOME_PC=ON -DCMAKE_TOOLCHAIN_FILE=cmake/llvm-mingw.cmake -DWINDOWS_ARCH=x86_64 -DLLVM_MINGW="$HOME/llvm-mingw" - cmake --build build/pc-win64 --target frame-home frame-switcher --parallel + cmake --build build/pc-win64 --target frame-home frame-switcher d3d11.dll --parallel mkdir -p "build/stage/FrameHome-PC-windows-x64" dist - cp build/pc-win64/frame-home.exe build/pc-win64/frame-switcher.exe frame/pc/install-frame-home-pc.ps1 \ + cp build/pc-win64/frame-home.exe build/pc-win64/frame-switcher.exe build/pc-win64/d3d11.dll frame/pc/install-frame-home-pc.ps1 \ "frame/pc/Install Frame Home on this PC.bat" "frame/pc/Remove Frame Home from this PC.bat" "build/stage/FrameHome-PC-windows-x64/" cp LICENSE build/stage/FrameHome-PC-windows-x64/LICENSE.txt (cd build/stage && zip -qr ../../dist/FrameHome-PC-windows-x64.zip FrameHome-PC-windows-x64) diff --git a/CMakeLists.txt b/CMakeLists.txt index 090dd17..ff86a46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,6 +139,16 @@ if(NOT FRAME_HOME_PC) set_target_properties(frame-home PROPERTIES BUILD_RPATH "${OPENVR_LIB_DIR}" INSTALL_RPATH "${OPENVR_LIB_DIR}") endif() +# The Windows compositor layer: a proxy d3d11.dll that wraps vrcompositor's Direct3D 11 device and draws the home in +# Aurora's place (layer/frame_home_layer_d3d11.cpp). The .def forwards every other d3d11 export to the real library. +if(WIN32 AND FRAME_HOME_PC) + add_library(frame-home-d3d11 MODULE layer/frame_home_layer_d3d11.cpp layer/d3d11_proxy.def) + set_target_properties(frame-home-d3d11 PROPERTIES OUTPUT_NAME d3d11 PREFIX "") + target_include_directories(frame-home-d3d11 PRIVATE src) + target_link_libraries(frame-home-d3d11 PRIVATE dxguid uuid) + target_link_options(frame-home-d3d11 PRIVATE -static -static-libgcc -static-libstdc++) +endif() + # fenv-tool: convert homes and models into .fenv environments. add_executable(fenv-tool src/fenv_tool.cpp) target_link_libraries(fenv-tool PRIVATE frame-scene) diff --git a/docs/pc-steamvr.md b/docs/pc-steamvr.md index a398451..aaf2c9f 100644 --- a/docs/pc-steamvr.md +++ b/docs/pc-steamvr.md @@ -1,8 +1,10 @@ # Frame Home on a PC (SteamVR) -Frame Home also runs on a PC with SteamVR: the same renderer, homes and settings as on the Steam Frame, as SteamVR's -environment whenever no game is open. The **Environments** panel in the Steam menu switches homes, as on the headset. -It works with any headset SteamVR drives, including a Steam Frame streaming from the PC. +Frame Home also runs on a PC with SteamVR, exactly the way it runs on the Steam Frame: a small layer inside SteamVR's +compositor draws your home where it would draw Aurora, so your home *is* the SteamVR environment — it shows at boot, +during game loads and behind the Steam menu, with the controllers and passthrough, never a separate app that launches +and hands off. The **Environments** panel in the Steam menu switches homes, as on the headset. It works with any +headset SteamVR drives, including a Steam Frame streaming from the PC. On a PC, Frame Home shows the environments in Frame Studio's **My Environments** (`Documents\Frame Studio\Environments`): save something in Studio and it's in VR straight away. Bundles (`.fbundle`) dropped there are unpacked by the panel. @@ -26,16 +28,24 @@ and takes effect when SteamVR starts. ## How it works -**Windows.** Frame Home becomes SteamVR's *home app*: SteamVR starts it whenever no game is open, and brings it back -when a game quits. For the moments before it draws (SteamVR starting, a game closing) SteamVR shows its classic -environment instead of Aurora, with its background set to a 360-degree picture of your current home taken from its -spawn. So you see your home, not Aurora. The panel makes a new picture whenever you switch homes. +Both platforms work the Steam Frame's way: a layer inside SteamVR's compositor draws your home in Aurora's place, and +`frame-home --system` (which the Environments panel starts and keeps running) renders it. Because the swap happens +inside the compositor, your home shows wherever Aurora would — at boot, during game loads, behind the Steam menu, +with the controllers and passthrough — and steps aside for a game the instant it opens, coming back when it quits. -**Linux.** Exactly the Steam Frame's way: a small Vulkan layer in SteamVR's compositor draws Frame Home in Aurora's -place, and `frame-home --system` renders it. (Linux PCs are supported but haven't been tested on a desktop yet.) +**Windows.** SteamVR's compositor is Direct3D 11, so the layer is a proxy `d3d11.dll` placed beside the compositor +(`SteamVR\bin\win64`). It wraps the compositor's device, finds Aurora's own draws by their shader bytecode and draws +the home over them; every other Direct3D call passes straight through to the real `d3d11.dll`, kept beside it as +`fh_d3d11real.dll`. Frame Home renders in Vulkan and shares each frame to the compositor as a Direct3D 11 texture. -The installer saves the SteamVR settings it changes (`enableHomeApp`, `environmentMode`, `background` and its -projection) and puts them back exactly when you remove Frame Home. +**Linux.** The same, as a Vulkan layer in the compositor (`layer/frame_home_layer.cpp`), since the Linux compositor +is Vulkan. (Linux PCs are supported but haven't been tested on a desktop yet.) + +If the layer ever can't draw the home — Frame Home isn't running, or has stopped for more than half a second — the +compositor draws Aurora as usual, so Frame Home stopping or crashing never blanks the headset. The layer turns itself +off if the compositor starts twice without a healthy frame, and `%LOCALAPPDATA%\Frame Home\layer\off` turns it off by +hand. The installer saves the SteamVR settings it changes (`enableHomeApp`, `environmentMode`) and puts them back, +and removing the two files from `SteamVR\bin\win64` takes the layer off entirely. ## Files @@ -44,15 +54,19 @@ projection) and puts them back exactly when you remove Frame Home. | `%LOCALAPPDATA%\Frame Home\bin` | `frame-home.exe`, `frame-switcher.exe` | | `%LOCALAPPDATA%\Frame Home\current` | the environment shown | | `%LOCALAPPDATA%\Frame Home\settings.json` | Frame Home's settings (the panel's Settings page) | -| `%LOCALAPPDATA%\Frame Home\logs` | logs | -| `%LOCALAPPDATA%\Frame Home\background` | the picture SteamVR shows while Frame Home starts | +| `%LOCALAPPDATA%\Frame Home\logs` | logs (`frame-home.log` the renderer, `frame-layer.log` the compositor layer) | +| `%LOCALAPPDATA%\Frame Home\layer` | the layer's off switch and crash guard; its folder marks the layer installed | | `Documents\Frame Studio\Environments` | the environments | +| `SteamVR\bin\win64\d3d11.dll`, `fh_d3d11real.dll` | the compositor layer and the real Direct3D 11 beside it | On Linux everything is in `~/frame-home`, as on the headset. ## Troubleshooting -- **Aurora still shows after installing:** SteamVR was running with its settings window open; restart SteamVR. +- **Aurora still shows after installing:** the layer loads when the compositor starts, so restart SteamVR (or your + headset) once after installing. If it still shows, see `frame-layer.log`. - **Nothing in the Steam menu:** the panel starts with SteamVR. In SteamVR's settings, **Startup / Shutdown > Choose Startup Overlay Apps** should list **Environments** switched on. - **Black world:** Frame Home needs a GPU with Vulkan 1.2. Its log is `%LOCALAPPDATA%\Frame Home\logs\frame-home.log`. +- **Turn it off without uninstalling:** make a file `%LOCALAPPDATA%\Frame Home\layer\off` (the layer checks every + second and draws Aurora); delete it to turn the home back on. diff --git a/frame/pc/install-frame-home-pc.ps1 b/frame/pc/install-frame-home-pc.ps1 index b3c2198..908e0f4 100644 --- a/frame/pc/install-frame-home-pc.ps1 +++ b/frame/pc/install-frame-home-pc.ps1 @@ -1,128 +1,122 @@ -# Frame Home on a Windows PC running SteamVR: your homes become SteamVR's environment, and the Environments panel -# in the Steam menu switches them. It shows the environments in Frame Studio's My Environments -# (Documents\Frame Studio\Environments): save one in Studio and it's there. +# Frame Home on a Windows PC running SteamVR: your environments become SteamVR's environment itself — drawn where +# Aurora would be, so they show at boot, during game loads and behind the Steam menu, with the controllers and +# passthrough. The Environments panel in the Steam menu switches them. It shows the environments in Frame Studio's +# My Environments (Documents\Frame Studio\Environments): save one in Studio and it's there. # # powershell -ExecutionPolicy Bypass -File install-frame-home-pc.ps1 install or update -# powershell -ExecutionPolicy Bypass -File install-frame-home-pc.ps1 -Uninstall SteamVR's own environment back +# powershell -ExecutionPolicy Bypass -File install-frame-home-pc.ps1 -Uninstall SteamVR's own Aurora back # -# How: Frame Home becomes SteamVR's home app (SteamVR runs it whenever no game is open), with SteamVR's classic -# environment in place of Aurora showing a 360-degree picture of the current home for the moments before Frame Home -# draws. The SteamVR settings this changes are saved first and put back exactly on -Uninstall. Works with SteamVR -# running (through its API) or not (its config files, read when it next starts). +# How: a small Direct3D 11 layer (d3d11.dll beside SteamVR's compositor) draws the home in Aurora's place, fed by +# frame-home --system; the real d3d11.dll is kept beside it as fh_d3d11real.dll and every call it makes passes +# straight through. The Environments panel runs frame-home --system and switches homes. -Uninstall takes the layer +# off (deleting those two files) and SteamVR is exactly as before. param([switch]$Uninstall) $ErrorActionPreference = 'Stop' $here = Split-Path -Parent $MyInvocation.MyCommand.Path $root = $(if ($env:FRAME_HOME_ROOT) { $env:FRAME_HOME_ROOT } else { Join-Path $env:LOCALAPPDATA 'Frame Home' }) # (FRAME_HOME_ROOT: tests) $bin = Join-Path $root 'bin' +$layer = Join-Path $root 'layer' function Say($m) { Write-Host "frame-install: $m" } -# SteamVR's config folder, from OpenVR's own record of where things are. +# SteamVR, from OpenVR's own record of where things are: its config folder and where it's installed (for the compositor). $vrpath = Join-Path $env:LOCALAPPDATA 'openvr\openvrpaths.vrpath' if (-not (Test-Path $vrpath)) { Say "SteamVR isn't installed on this PC (no $vrpath). Install SteamVR from Steam first."; exit 1 } $openvr = Get-Content $vrpath -Raw | ConvertFrom-Json $cfg = @($openvr.config)[0] -if (-not $cfg) { Say "SteamVR's config folder isn't listed in $vrpath"; exit 1 } +$runtime = @($openvr.runtime)[0] +if (-not $cfg -or -not $runtime) { Say "SteamVR's folders aren't listed in $vrpath"; exit 1 } +$bin64 = Join-Path $runtime 'bin\win64' +$proxy = Join-Path $bin64 'd3d11.dll' +$real = Join-Path $bin64 'fh_d3d11real.dll' $running = [bool](Get-Process vrserver -ErrorAction SilentlyContinue) -# JSON as SteamVR writes it: UTF-8 without a byte-order mark, replaced in one step. -function Load($p) { - if (-not (Test-Path $p)) { return [pscustomobject]@{} } - try { return Get-Content $p -Raw | ConvertFrom-Json } catch { Say "$p does not read: left as it is"; exit 1 } -} +function StopHome { Get-Process frame-home, frame-switcher -ErrorAction SilentlyContinue | Stop-Process -Force; Start-Sleep -Milliseconds 400 } + +# JSON as SteamVR writes it: UTF-8 without a byte-order mark. +function Load($p) { if (-not (Test-Path $p)) { return [pscustomobject]@{} } try { return Get-Content $p -Raw | ConvertFrom-Json } catch { Say "$p does not read: left as it is"; exit 1 } } function Save($p, $o) { New-Item -ItemType Directory -Force -Path (Split-Path -Parent $p) | Out-Null - $json = $o | ConvertTo-Json -Depth 32 - [IO.File]::WriteAllText("$p.tmp", $json, (New-Object Text.UTF8Encoding $false)) + [IO.File]::WriteAllText("$p.tmp", ($o | ConvertTo-Json -Depth 32), (New-Object Text.UTF8Encoding $false)) Move-Item -Force "$p.tmp" $p } -function SetProp($o, $name, $value) { - if ($o.PSObject.Properties[$name]) { $o.$name = $value } else { $o | Add-Member -NotePropertyName $name -NotePropertyValue $value } -} +function SetProp($o, $name, $value) { if ($o.PSObject.Properties[$name]) { $o.$name = $value } else { $o | Add-Member -NotePropertyName $name -NotePropertyValue $value } } -$manifests = @((Join-Path $root 'frame-home.vrmanifest'), (Join-Path $root 'frame-switcher.vrmanifest')) +# enableHomeApp off (no separate home app) and environmentMode 1 (the compositor draws Aurora, which the layer +# replaces). Saved once as they were, put back on -Uninstall. Also the panel keeps them so while SteamVR runs. $before = Join-Path $root 'steamvr-before.json' - -# SteamVR not running: the same registration, in its config files. -function Offline($install) { - $ac = Join-Path $cfg 'appconfig.json' - $a = Load $ac - if (-not $a.PSObject.Properties['manifest_paths']) { SetProp $a 'manifest_paths' @() } - $paths = [System.Collections.ArrayList]@($a.manifest_paths) - foreach ($m in $manifests) { - if ($install -and -not $paths.Contains($m)) { [void]$paths.Add($m) } - if (-not $install -and $paths.Contains($m)) { $paths.Remove($m) } - } - $a.manifest_paths = @($paths) - Save $ac $a +function Settings($install) { $vs = Join-Path $cfg 'steamvr.vrsettings' $s = Load $vs if (-not $s.PSObject.Properties['steamvr']) { SetProp $s 'steamvr' ([pscustomobject]@{}) } $sv = $s.steamvr if ($install) { - if (-not (Test-Path $before)) { # what was there before Frame Home, saved once - $was = [ordered]@{ + if (-not (Test-Path $before)) { + Save $before ([pscustomobject]@{ enableHomeApp = [bool]$sv.enableHomeApp environmentMode = $(if ($sv.PSObject.Properties['environmentMode']) { [int]$sv.environmentMode } else { 1 }) - background = [string]$sv.background - backgroundUseDomeProjection = [bool]$sv.backgroundUseDomeProjection - backgroundCameraHeight = $(if ($sv.PSObject.Properties['backgroundCameraHeight']) { [double]$sv.backgroundCameraHeight } else { 1.6 }) - backgroundDomeRadius = [double]$sv.backgroundDomeRadius - } - Save $before ([pscustomobject]$was) + }) } - SetProp $sv 'enableHomeApp' $true - SetProp $sv 'environmentMode' 0 + SetProp $sv 'enableHomeApp' $false + SetProp $sv 'environmentMode' 1 } else { $was = $(if (Test-Path $before) { Get-Content $before -Raw | ConvertFrom-Json } else { $null }) SetProp $sv 'enableHomeApp' $(if ($was) { [bool]$was.enableHomeApp } else { $false }) SetProp $sv 'environmentMode' $(if ($was) { [int]$was.environmentMode } else { 1 }) - SetProp $sv 'background' $(if ($was) { [string]$was.background } else { '' }) - SetProp $sv 'backgroundUseDomeProjection' $(if ($was) { [bool]$was.backgroundUseDomeProjection } else { $false }) - if ($was) { SetProp $sv 'backgroundCameraHeight' ([double]$was.backgroundCameraHeight); SetProp $sv 'backgroundDomeRadius' ([double]$was.backgroundDomeRadius) } Remove-Item -Force -ErrorAction SilentlyContinue $before - Remove-Item -Recurse -Force -ErrorAction SilentlyContinue (Join-Path $root 'background') - foreach ($m in $manifests) { Remove-Item -Force -ErrorAction SilentlyContinue $m } } Save $vs $s +} + +# The panel, registered with SteamVR (frame-switcher --install) or, when SteamVR isn't running, in its config files. +$switcherManifest = Join-Path $root 'frame-switcher.vrmanifest' +function PanelOffline($install) { + $ac = Join-Path $cfg 'appconfig.json' + $a = Load $ac + if (-not $a.PSObject.Properties['manifest_paths']) { SetProp $a 'manifest_paths' @() } + $paths = [System.Collections.ArrayList]@($a.manifest_paths) + if ($install -and -not $paths.Contains($switcherManifest)) { [void]$paths.Add($switcherManifest) } + if (-not $install -and $paths.Contains($switcherManifest)) { $paths.Remove($switcherManifest) } + $a.manifest_paths = @($paths) + Save $ac $a Save (Join-Path $cfg 'vrappconfig\framehome.switcher.vrappconfig') ([pscustomobject]@{ autolaunch = [bool]$install; last_launch_time = '0' }) } -function StopAll { Get-Process frame-home, frame-switcher -ErrorAction SilentlyContinue | Stop-Process -Force; Start-Sleep -Milliseconds 500 } - if ($Uninstall) { - if ($running) { - & (Join-Path $bin 'frame-home.exe') --uninstall - & (Join-Path $bin 'frame-switcher.exe') --uninstall - } else { - Offline $false - } - StopAll - Say "SteamVR's own environment is back (your environments are kept in Documents\Frame Studio\Environments)" + StopHome + if ($running) { & (Join-Path $bin 'frame-switcher.exe') --uninstall } else { PanelOffline $false } + Settings $false + Remove-Item -Force -ErrorAction SilentlyContinue $proxy # the layer off: SteamVR loads the system d3d11 again + Remove-Item -Force -ErrorAction SilentlyContinue $real + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue $layer + Say "Frame Home is off: SteamVR's own environment is back (restart SteamVR to see it). Your environments are kept in Documents\Frame Studio\Environments" exit 0 } -foreach ($x in 'frame-home.exe', 'frame-switcher.exe') { +foreach ($x in 'frame-home.exe', 'frame-switcher.exe', 'd3d11.dll') { if (-not (Test-Path (Join-Path $here $x))) { Say "this installer has no $x"; exit 1 } } -# Runs here, before anything installed is touched (a missing Vulkan driver shows up now, not as a black world later). +# Frame Home runs on this PC (a missing Vulkan driver shows up now, not as a black world later). & (Join-Path $here 'frame-home.exe') --version | Out-Null if ($LASTEXITCODE -ne 0) { Say "Frame Home can't run on this PC (Vulkan driver?): nothing was changed"; exit 1 } -StopAll -New-Item -ItemType Directory -Force -Path $bin, (Join-Path ([Environment]::GetFolderPath('MyDocuments')) 'Frame Studio\Environments') | Out-Null +StopHome +New-Item -ItemType Directory -Force -Path $bin, $layer, (Join-Path ([Environment]::GetFolderPath('MyDocuments')) 'Frame Studio\Environments') | Out-Null foreach ($x in 'frame-home.exe', 'frame-switcher.exe') { Copy-Item -Force (Join-Path $here $x) (Join-Path $bin $x) } +# The compositor layer: keep the real d3d11.dll (from System32 — SteamVR's bin\win64 has none of its own) as +# fh_d3d11real.dll, then drop our proxy in as d3d11.dll. (We copy the real one only once, never our own proxy.) +if (-not (Test-Path $bin64)) { Say "SteamVR's compositor folder isn't where $vrpath says ($bin64)"; exit 1 } +if (-not (Test-Path $real)) { Copy-Item -Force (Join-Path $env:windir 'System32\d3d11.dll') $real } +Copy-Item -Force (Join-Path $here 'd3d11.dll') $proxy + +Settings $true if ($running) { - & (Join-Path $here 'frame-switcher.exe') --install - & (Join-Path $here 'frame-home.exe') --install - if ($LASTEXITCODE -ne 0) { Say "SteamVR refused Frame Home ($LASTEXITCODE)"; exit 1 } - Say "installed: Frame Home is SteamVR's environment now" + & (Join-Path $bin 'frame-switcher.exe') --install # registers the panel, auto-launches it, starts it now + Say "installed. SteamVR is running, so restart it (or your headset) once to load the environment layer; the Environments panel is in the Steam menu." } else { - # (with SteamVR not running these write their manifests and say so) - & (Join-Path $here 'frame-switcher.exe') --install | Out-Null - & (Join-Path $here 'frame-home.exe') --install | Out-Null - Offline $true - Say "installed: Frame Home becomes SteamVR's environment when SteamVR next starts" + & (Join-Path $bin 'frame-switcher.exe') --install | Out-Null + PanelOffline $true + Say "installed: your environment becomes SteamVR's when SteamVR next starts." } exit 0 diff --git a/layer/d3d11_proxy.def b/layer/d3d11_proxy.def new file mode 100644 index 0000000..549125d --- /dev/null +++ b/layer/d3d11_proxy.def @@ -0,0 +1,57 @@ +; Frame Home's d3d11.dll proxy (layer/frame_home_layer_d3d11.cpp): D3D11CreateDevice is ours, so we can wrap the +; SteamVR compositor's device and draw the home where it would draw Aurora. Every other export forwards to the real +; d3d11.dll, copied beside us as fh_d3d11real.dll by the installer, so the compositor behaves exactly as before in +; every way we don't touch, and so does any other program that happens to load this proxy. +LIBRARY d3d11 +EXPORTS + D3D11CreateDevice + CreateDirect3D11DeviceFromDXGIDevice = fh_d3d11real.CreateDirect3D11DeviceFromDXGIDevice + CreateDirect3D11SurfaceFromDXGISurface = fh_d3d11real.CreateDirect3D11SurfaceFromDXGISurface + D3D11CoreCreateDevice = fh_d3d11real.D3D11CoreCreateDevice + D3D11CoreCreateLayeredDevice = fh_d3d11real.D3D11CoreCreateLayeredDevice + D3D11CoreGetLayeredDeviceSize = fh_d3d11real.D3D11CoreGetLayeredDeviceSize + D3D11CoreRegisterLayers = fh_d3d11real.D3D11CoreRegisterLayers + D3D11CreateDeviceAndSwapChain = fh_d3d11real.D3D11CreateDeviceAndSwapChain + D3D11CreateDeviceForD3D12 = fh_d3d11real.D3D11CreateDeviceForD3D12 + D3D11On12CreateDevice = fh_d3d11real.D3D11On12CreateDevice + D3DKMTCloseAdapter = fh_d3d11real.D3DKMTCloseAdapter + D3DKMTCreateAllocation = fh_d3d11real.D3DKMTCreateAllocation + D3DKMTCreateContext = fh_d3d11real.D3DKMTCreateContext + D3DKMTCreateDevice = fh_d3d11real.D3DKMTCreateDevice + D3DKMTCreateSynchronizationObject = fh_d3d11real.D3DKMTCreateSynchronizationObject + D3DKMTDestroyAllocation = fh_d3d11real.D3DKMTDestroyAllocation + D3DKMTDestroyContext = fh_d3d11real.D3DKMTDestroyContext + D3DKMTDestroyDevice = fh_d3d11real.D3DKMTDestroyDevice + D3DKMTDestroySynchronizationObject = fh_d3d11real.D3DKMTDestroySynchronizationObject + D3DKMTEscape = fh_d3d11real.D3DKMTEscape + D3DKMTGetContextSchedulingPriority = fh_d3d11real.D3DKMTGetContextSchedulingPriority + D3DKMTGetDeviceState = fh_d3d11real.D3DKMTGetDeviceState + D3DKMTGetDisplayModeList = fh_d3d11real.D3DKMTGetDisplayModeList + D3DKMTGetMultisampleMethodList = fh_d3d11real.D3DKMTGetMultisampleMethodList + D3DKMTGetRuntimeData = fh_d3d11real.D3DKMTGetRuntimeData + D3DKMTGetSharedPrimaryHandle = fh_d3d11real.D3DKMTGetSharedPrimaryHandle + D3DKMTLock = fh_d3d11real.D3DKMTLock + D3DKMTOpenAdapterFromHdc = fh_d3d11real.D3DKMTOpenAdapterFromHdc + D3DKMTOpenResource = fh_d3d11real.D3DKMTOpenResource + D3DKMTPresent = fh_d3d11real.D3DKMTPresent + D3DKMTQueryAdapterInfo = fh_d3d11real.D3DKMTQueryAdapterInfo + D3DKMTQueryAllocationResidency = fh_d3d11real.D3DKMTQueryAllocationResidency + D3DKMTQueryResourceInfo = fh_d3d11real.D3DKMTQueryResourceInfo + D3DKMTRender = fh_d3d11real.D3DKMTRender + D3DKMTSetAllocationPriority = fh_d3d11real.D3DKMTSetAllocationPriority + D3DKMTSetContextSchedulingPriority = fh_d3d11real.D3DKMTSetContextSchedulingPriority + D3DKMTSetDisplayMode = fh_d3d11real.D3DKMTSetDisplayMode + D3DKMTSetDisplayPrivateDriverFormat = fh_d3d11real.D3DKMTSetDisplayPrivateDriverFormat + D3DKMTSetGammaRamp = fh_d3d11real.D3DKMTSetGammaRamp + D3DKMTSetVidPnSourceOwner = fh_d3d11real.D3DKMTSetVidPnSourceOwner + D3DKMTSignalSynchronizationObject = fh_d3d11real.D3DKMTSignalSynchronizationObject + D3DKMTUnlock = fh_d3d11real.D3DKMTUnlock + D3DKMTWaitForSynchronizationObject = fh_d3d11real.D3DKMTWaitForSynchronizationObject + D3DKMTWaitForVerticalBlankEvent = fh_d3d11real.D3DKMTWaitForVerticalBlankEvent + D3DPerformance_BeginEvent = fh_d3d11real.D3DPerformance_BeginEvent + D3DPerformance_EndEvent = fh_d3d11real.D3DPerformance_EndEvent + D3DPerformance_GetStatus = fh_d3d11real.D3DPerformance_GetStatus + D3DPerformance_SetMarker = fh_d3d11real.D3DPerformance_SetMarker + EnableFeatureLevelUpgrade = fh_d3d11real.EnableFeatureLevelUpgrade + OpenAdapter10 = fh_d3d11real.OpenAdapter10 + OpenAdapter10_2 = fh_d3d11real.OpenAdapter10_2 diff --git a/layer/frame_home_layer_d3d11.cpp b/layer/frame_home_layer_d3d11.cpp new file mode 100644 index 0000000..fe8a1aa --- /dev/null +++ b/layer/frame_home_layer_d3d11.cpp @@ -0,0 +1,623 @@ +// Frame Home's Direct3D 11 compositor layer (Windows): the same idea as the Vulkan layer on the Frame and Linux +// (layer/frame_home_layer.cpp), but SteamVR's Windows compositor is Direct3D 11, so this lives inside it as a proxy +// d3d11.dll. D3D11CreateDevice is ours (d3d11_proxy.def forwards every other export to the real d3d11.dll, copied +// beside us as fh_d3d11real.dll), so when vrcompositor creates its device we wrap it and swap only the environment: +// where the compositor draws Aurora's sky we draw the newest frame Frame Home (frame-home --system) rendered of your +// home, looked up per pixel by direction (rotation reprojection), and Aurora's floor and stars we leave undrawn. +// Everything else the compositor does is untouched, so the home shows at boot, during game loads, behind the Steam +// menu, with the controllers and passthrough, exactly as Aurora would. Outside vrcompositor this is pure pass-through. +// +// Hand-over (env_share.h): Frame Home connects to our named pipe and sends its shared State's handle and the size it +// wants; we make kSlots shared textures, hand it their NT handles, and it renders the home into them over Vulkan. +// +// Safety: +// active only inside a process named vrcompositor.exe; everywhere else D3D11CreateDevice just calls the real one +// no published frame, or one older than half a second (Frame Home stopped or crashed): Aurora as usual +// %LOCALAPPDATA%\Frame Home\layer\off: pass-through (checked at start and each second) +// twice the compositor started with us swapping and never got a healthy frame in: we write `off` ourselves +// deleting fh_d3d11real.dll / this d3d11.dll from SteamVR's bin\win64 removes us entirely (the installer does) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../src/env_share.h" + +namespace fs = std::filesystem; + +namespace { + +// ── process / files ────────────────────────────────────────────────────────────────────────────── + +bool isCompositor() { + static int v = -1; + if (v < 0) { + wchar_t path[MAX_PATH] = {}; + GetModuleFileNameW(nullptr, path, MAX_PATH); + const wchar_t* base = wcsrchr(path, L'\\'); + base = base ? base + 1 : path; + v = _wcsicmp(base, L"vrcompositor.exe") == 0; + } + return v == 1; +} + +// %LOCALAPPDATA%\Frame Home (home_platform.h's root() on Windows), where the logs live. +fs::path dir() { + wchar_t* p = nullptr; + size_t n = 0; + _wdupenv_s(&p, &n, L"LOCALAPPDATA"); + fs::path d = p ? fs::path(p) / L"Frame Home" : fs::path(L"Frame Home"); + free(p); + return d; +} +// The layer's own folder (root\layer): its off switch and crash guard, the same names and place the Environments +// panel uses (rootDir()/layer/off, .../crashes, .../starting) and the installer makes to mark the layer present. +fs::path layerDir() { return dir() / L"layer"; } +bool fileExists(const fs::path& p) { std::error_code ec; return fs::exists(p, ec); } + +std::mutex g_logMutex; +void logf(const char* fmt, ...) { + std::lock_guard l(g_logMutex); + std::error_code ec; + fs::create_directories(dir() / "logs", ec); + FILE* f = nullptr; + if (_wfopen_s(&f, (dir() / "logs" / "frame-layer.log").wstring().c_str(), L"a") || !f) return; + SYSTEMTIME t; + GetLocalTime(&t); + std::fprintf(f, "%02d:%02d:%02d ", t.wHour, t.wMinute, t.wSecond); + va_list ap; + va_start(ap, fmt); + std::vfprintf(f, fmt, ap); + va_end(ap); + std::fputc('\n', f); + std::fclose(f); +} + +std::vector readFile(const fs::path& p) { + std::ifstream in(p, std::ios::binary); + return {std::istreambuf_iterator(in), std::istreambuf_iterator()}; +} + +uint64_t fnv(const void* data, size_t n) { + uint64_t h = 1469598103934665603ull; + const unsigned char* b = static_cast(data); + for (size_t i = 0; i < n; ++i) { h ^= b[i]; h *= 1099511628211ull; } + return h; +} + +// ── Aurora shaders, by the bytecode the compositor hands CreatePixelShader ───────────────────────── +// The compositor loads SteamVR's compiled pixel shaders (resources\shaders\d3d11\*.fxo) and creates them with the +// file's bytes. We hash each file whose name marks it part of the environment and, at CreatePixelShader, tag the +// ID3D11PixelShader it returns: the sky (and horizon) we replace with the home; the floor and stars we skip. +enum class Kind { None, Replace, Skip }; +std::unordered_map g_shaderKinds; // bytecode hash -> what to do + +void scanAuroraShaders() { + wchar_t path[MAX_PATH] = {}; + GetModuleFileNameW(nullptr, path, MAX_PATH); // ...\SteamVR\bin\win64\vrcompositor.exe + fs::path shaders = fs::path(path).parent_path().parent_path().parent_path() / L"resources" / L"shaders" / L"d3d11"; + std::error_code ec; + int replace = 0, skip = 0; + for (auto& e : fs::directory_iterator(shaders, ec)) { + if (e.path().extension() != ".fxo") continue; + const std::string name = e.path().filename().string(); + const bool ps = name.find("_ps.fxo") != std::string::npos; + if (!ps || name.find("aurora") == std::string::npos) continue; + // Aurora's sky and horizon (what you look out at) we draw the home into; its floor and its stars (the motes + // "sky" that is really the floor's ground glow) we leave off, as the Vulkan layer does. + Kind k = Kind::Replace; + if (name.find("floor") != std::string::npos) k = Kind::Skip; + const std::vector b = readFile(e.path()); + if (b.empty()) continue; + g_shaderKinds[fnv(b.data(), b.size())] = k; + (k == Kind::Replace ? replace : skip) += 1; + } + // The motes (stars) use their own non-aurora pixel shader on the aurora vertex shader; skip by name too. + for (const char* n : {"distort_geom_motes_ps.fxo", "geom_motes_ps.fxo"}) { + const std::vector b = readFile(shaders / n); + if (!b.empty()) { g_shaderKinds[fnv(b.data(), b.size())] = Kind::Skip; ++skip; } + } + logf("aurora shaders: %d replaced, %d skipped (from %s)", replace, skip, shaders.string().c_str()); +} + +// ── the home: shared state, the compositor's textures, and our draw resources ────────────────────── + +// Two locks, never held in the reverse order: g_shaderMutex guards the recognised shaders (written as the compositor +// creates them, read at each draw); g_texMutex guards the shared textures (rebuilt by the pipe thread when Frame Home +// connects or resizes, read while a draw is replaced). A replaced draw holds g_texMutex across the draw so the pipe +// thread can't free a texture mid-draw — the race that crashed the compositor. +std::mutex g_shaderMutex, g_texMutex; +std::atomic g_off{false}; +std::atomic g_connected{false}; +std::atomic g_replaced{0}, g_skipped{0}; // draws swapped for the home / left undrawn (a diagnostic) + +envshare::State* g_state = nullptr; // Frame Home's shared State (it owns it; we map its handle) +HANDLE g_stateMap = nullptr; + +ID3D11Device* g_device = nullptr; // the compositor's device (not addref'd: it outlives us) +ID3D11DeviceContext* g_context = nullptr; // its immediate context + +uint32_t g_generation = 0; // bumped each time Frame Home asks for new textures +uint32_t g_texW = 0, g_texH = 0; +struct SharedTex { ID3D11Texture2D* tex = nullptr; ID3D11ShaderResourceView* srv = nullptr; HANDLE handle = nullptr; }; +std::vector g_tex; // kSlots, on the compositor's device +int g_lastGood = -1; // newest slot drawn (held while Frame Home makes new textures) + +// Our draw resources on the compositor's device. +ID3D11PixelShader* g_homePs = nullptr; +ID3D11Buffer* g_homeCb = nullptr; // HomeCb below (b13): reprojection + brightness +ID3D11SamplerState* g_sampler = nullptr; +bool g_resourcesTried = false, g_resourcesOk = false; + +constexpr UINT kSrvSlot = 13, kCbSlot = 13, kSampSlot = 13; // high slots the compositor's own shaders don't use + +struct HomeCb { // matches the cbuffer in the replacement shader (b13) + float col[envshare::kSlots == 0 ? 1 : 2][4][4]; // per eye: the 4 columns of projection*viewRotation + float eye[2][4]; // per eye position (xyz) + float params[4]; // x: brightness (fade), y: 1 when there's something to show +}; + +// Per command buffer? D3D's immediate context is one object; the pixel shader bound on it, tracked by PSSetShader. +ID3D11PixelShader* g_boundPs = nullptr; + +// Which slot to draw this compositor frame: picked at the first replaced draw and held briefly, so both eyes and +// every aurora piece in the frame show the same home frame (frameSlot in the Vulkan layer). +int g_latchSlot = -1; +uint64_t g_latchNs = 0; +bool g_latchStamped = false; +constexpr uint64_t kLatchNs = 4000000ull; + +int liveSlot() { // the newest published slot, or -1 for Aurora (Frame Home gone or frozen 5 s) + if (g_off || !g_state || g_tex.empty()) return -1; + const uint64_t now = envshare::nowNs(); + if (now - g_state->publishNs.load() > 5000000000ull && now > g_state->busyUntilNs.load()) return -1; + const uint32_t k = g_state->latest.load(std::memory_order_acquire); + if (k < envshare::kSlots && g_state->slot[k].generation == g_generation) g_lastGood = static_cast(k); + return g_lastGood; +} +int frameSlot() { + const uint64_t now = envshare::nowNs(); + if (g_latchNs && now - g_latchNs < kLatchNs) return g_latchSlot; + g_latchSlot = liveSlot(); + g_latchNs = now; + g_latchStamped = false; + return g_latchSlot; +} + +// ── health: off switch and crash guard (mirrors the Vulkan layer) ────────────────────────────────── +std::atomic g_armed{false}; +uint64_t g_healthyFrames = 0; + +void startHealth() { + std::error_code ec; + fs::create_directories(layerDir(), ec); + if (fileExists(layerDir() / "off")) { g_off = true; logf("off: Aurora as usual (delete %s\\off to turn Frame Home back on)", dir().string().c_str()); return; } + int crashes = 0; + if (fileExists(layerDir() / "starting")) { + const std::vector c = readFile(layerDir() / "crashes"); + crashes = c.empty() ? 1 : std::atoi(std::string(c.begin(), c.end()).c_str()) + 1; + if (FILE* f = nullptr; !_wfopen_s(&f, (layerDir() / "crashes").wstring().c_str(), L"w") && f) { std::fprintf(f, "%d\n", crashes); std::fclose(f); } + } + if (crashes >= 2) { + if (FILE* f = nullptr; !_wfopen_s(&f, (layerDir() / "off").wstring().c_str(), L"w") && f) std::fclose(f); + DeleteFileW((layerDir() / "starting").wstring().c_str()); + g_off = true; + logf("the compositor stopped twice before a healthy frame: turned myself off (Aurora as usual)"); + } +} +void arm() { // we are about to start swapping Aurora: a crash from here that never got healthy counts + if (g_armed.exchange(true)) return; + if (FILE* f = nullptr; !_wfopen_s(&f, (layerDir() / "starting").wstring().c_str(), L"w") && f) std::fclose(f); +} +void healthy() { // a frame drawn and shown: the compositor got this far, so a later crash isn't ours + if (++g_healthyFrames == 30 && g_armed.exchange(false)) { + DeleteFileW((layerDir() / "starting").wstring().c_str()); + DeleteFileW((layerDir() / "crashes").wstring().c_str()); + } +} + +// ── the replacement pixel shader, compiled at startup ────────────────────────────────────────────── +// Reads the compositor's own constant buffer (b0) for this eye's position, forms the direction from the eye to the +// drawn point, looks it up in the home frame (projection * view rotation, column major, as Frame Home rendered it) +// and samples that eye's layer. The same maths as layer/env_ps.frag. +const char* kHlsl = R"( +Texture2DArray home : register(t13); +SamplerState samp : register(s13); +cbuffer Aurora : register(b0) { float4 _pad0[28]; float4 g_vEyePositionWs; } // g_vEyePositionWs at offset 448 +cbuffer Home : register(b13) { + float4 uCol[8]; // per eye (0: 0..3, 1: 4..7): the 4 columns of projection*viewRotation, tracking space + float4 uEyePos[2]; // per eye position Frame Home rendered from, tracking space (xyz) + float4 uParams; // x: brightness, y: 1 when there's something to show +}; +struct PSIn { float4 pos : SV_Position; float3 worldPos : TEXCOORD1; }; +float4 main(PSIn i) : SV_Target { + float3 eye = g_vEyePositionWs.xyz; + float3 dir = normalize(i.worldPos - eye); + int k = distance(eye, uEyePos[0].xyz) <= distance(eye, uEyePos[1].xyz) ? 0 : 1; + int b = k * 4; + float4 c = uCol[b + 0] * dir.x + uCol[b + 1] * dir.y + uCol[b + 2] * dir.z; + float3 rgb = float3(0, 0, 0); + if (c.w > 1e-5 && uParams.y > 0.5) { + float2 uv = (c.xy / c.w) * 0.5 + 0.5; + rgb = home.SampleLevel(samp, float3(uv, (float)k), 0.0).rgb; + } + return float4(rgb * uParams.x, 1.0); +} +)"; + +ID3DBlob* compileHlsl() { + HMODULE comp = LoadLibraryW(L"d3dcompiler_47.dll"); + if (!comp) { logf("no d3dcompiler_47.dll: Aurora stays"); return nullptr; } + using Fn = HRESULT(WINAPI*)(LPCVOID, SIZE_T, LPCSTR, const D3D_SHADER_MACRO*, ID3DInclude*, LPCSTR, LPCSTR, UINT, UINT, ID3DBlob**, ID3DBlob**); + auto d3dCompile = reinterpret_cast(GetProcAddress(comp, "D3DCompile")); + if (!d3dCompile) return nullptr; + ID3DBlob* code = nullptr; + ID3DBlob* err = nullptr; + const HRESULT hr = d3dCompile(kHlsl, std::strlen(kHlsl), "frame_home_env", nullptr, nullptr, "main", "ps_5_0", 0, 0, &code, &err); + if (FAILED(hr)) { logf("replacement shader failed to compile (0x%08lx): %s", hr, err ? static_cast(err->GetBufferPointer()) : ""); if (err) err->Release(); return nullptr; } + if (err) err->Release(); + return code; +} + +bool ensureResources() { + if (g_resourcesTried) return g_resourcesOk; + g_resourcesTried = true; + ID3DBlob* code = compileHlsl(); + if (!code) return false; + if (FAILED(g_device->CreatePixelShader(code->GetBufferPointer(), code->GetBufferSize(), nullptr, &g_homePs))) { code->Release(); logf("CreatePixelShader for the home failed"); return false; } + code->Release(); + D3D11_BUFFER_DESC bd = {}; + bd.ByteWidth = sizeof(HomeCb); + bd.Usage = D3D11_USAGE_DYNAMIC; + bd.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + bd.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + if (FAILED(g_device->CreateBuffer(&bd, nullptr, &g_homeCb))) { logf("home cbuffer failed"); return false; } + D3D11_SAMPLER_DESC sd = {}; + sd.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR; + sd.AddressU = sd.AddressV = sd.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + sd.MaxLOD = D3D11_FLOAT32_MAX; + if (FAILED(g_device->CreateSamplerState(&sd, &g_sampler))) { logf("home sampler failed"); return false; } + g_resourcesOk = true; + logf("home draw resources ready"); + return true; +} + +void fillCb(int slot) { + if (!g_homeCb) return; + const envshare::Slot& s = g_state->slot[slot]; + HomeCb cb = {}; + for (int e = 0; e < 2; ++e) { + std::memcpy(cb.col[e], s.viewProj[e], sizeof(float) * 16); // column major: 4 columns of 4 floats + for (int c = 0; c < 4; ++c) cb.eye[e][c] = s.eyePos[e][c]; + } + cb.params[0] = g_state->fade.load() / 1000.0f; + cb.params[1] = g_state->show.load() ? 1.0f : 0.0f; + D3D11_MAPPED_SUBRESOURCE m; + if (SUCCEEDED(g_context->Map(g_homeCb, 0, D3D11_MAP_WRITE_DISCARD, 0, &m))) { + std::memcpy(m.pData, &cb, sizeof cb); + g_context->Unmap(g_homeCb, 0); + } +} + +// ── the hooks: CreatePixelShader on the device, PSSetShader + the four draws on the context ───────── + +using PFN_CreatePixelShader = HRESULT(STDMETHODCALLTYPE*)(ID3D11Device*, const void*, SIZE_T, ID3D11ClassLinkage*, ID3D11PixelShader**); +using PFN_PSSetShader = void(STDMETHODCALLTYPE*)(ID3D11DeviceContext*, ID3D11PixelShader*, ID3D11ClassInstance* const*, UINT); +using PFN_Draw = void(STDMETHODCALLTYPE*)(ID3D11DeviceContext*, UINT, UINT); +using PFN_DrawIndexed = void(STDMETHODCALLTYPE*)(ID3D11DeviceContext*, UINT, UINT, INT); +using PFN_DrawInstanced = void(STDMETHODCALLTYPE*)(ID3D11DeviceContext*, UINT, UINT, UINT, UINT); +using PFN_DrawIndexedInstanced = void(STDMETHODCALLTYPE*)(ID3D11DeviceContext*, UINT, UINT, UINT, INT, UINT); + +PFN_CreatePixelShader g_realCreatePixelShader = nullptr; +PFN_PSSetShader g_realPSSetShader = nullptr; +PFN_Draw g_realDraw = nullptr; +PFN_DrawIndexed g_realDrawIndexed = nullptr; +PFN_DrawInstanced g_realDrawInstanced = nullptr; +PFN_DrawIndexedInstanced g_realDrawIndexedInstanced = nullptr; + +std::unordered_map g_psKind; // shaders the compositor made that we recognised + +HRESULT STDMETHODCALLTYPE CreatePixelShader_hook(ID3D11Device* dev, const void* code, SIZE_T len, ID3D11ClassLinkage* link, ID3D11PixelShader** out) { + const HRESULT hr = g_realCreatePixelShader(dev, code, len, link, out); + if (SUCCEEDED(hr) && out && *out && code && len) { + auto it = g_shaderKinds.find(fnv(code, len)); + if (it != g_shaderKinds.end()) { + std::lock_guard l(g_shaderMutex); + g_psKind[*out] = it->second; + } + } + return hr; +} + +void STDMETHODCALLTYPE PSSetShader_hook(ID3D11DeviceContext* ctx, ID3D11PixelShader* ps, ID3D11ClassInstance* const* inst, UINT n) { + if (ctx == g_context) g_boundPs = ps; + g_realPSSetShader(ctx, ps, inst, n); +} + +// What the compositor's about-to-draw pixel shader means for us (no texture access: that's under g_texMutex). +Kind classify(ID3D11DeviceContext* ctx) { + if (g_off || ctx != g_context || !g_boundPs) return Kind::None; + std::lock_guard l(g_shaderMutex); + auto it = g_psKind.find(g_boundPs); + return it == g_psKind.end() ? Kind::None : it->second; +} + +// Binds the home over the compositor's state for one draw, then puts the shader back. +struct HomeBind { + ID3D11DeviceContext* ctx; + ID3D11PixelShader* restore; + ID3D11ShaderResourceView* srv; + HomeBind(ID3D11DeviceContext* c, int slot, ID3D11PixelShader* r) : ctx(c), restore(r), srv(g_tex[slot].srv) { + g_realPSSetShader(ctx, g_homePs, nullptr, 0); + ctx->PSSetShaderResources(kSrvSlot, 1, &srv); + ctx->PSSetConstantBuffers(kCbSlot, 1, &g_homeCb); + ctx->PSSetSamplers(kSampSlot, 1, &g_sampler); + } + ~HomeBind() { + g_realPSSetShader(ctx, restore, nullptr, 0); // the compositor's aurora shader back, so our tracking holds + healthy(); + } +}; + +// Runs the compositor's draw, swapping the home in for an Aurora sky draw or dropping an Aurora floor/stars draw. +// The texture lock is held across a replaced draw so the pipe thread can't free the texture under it. +template +void replaceDraw(ID3D11DeviceContext* ctx, Draw&& draw) { + const Kind k = classify(ctx); + if (k == Kind::None) { draw(); return; } + if (k == Kind::Skip) { g_skipped.fetch_add(1, std::memory_order_relaxed); return; } + std::unique_lock lock(g_texMutex); + const int slot = frameSlot(); + if (slot < 0 || !ensureResources() || slot >= static_cast(g_tex.size()) || !g_tex[slot].srv) { lock.unlock(); draw(); return; } + fillCb(slot); + if (!g_latchStamped) { g_latchStamped = true; g_state->sampledNs.store(g_latchNs); } + g_replaced.fetch_add(1, std::memory_order_relaxed); + HomeBind b(ctx, slot, g_boundPs); + draw(); +} + +void STDMETHODCALLTYPE Draw_hook(ID3D11DeviceContext* ctx, UINT n, UINT start) { + replaceDraw(ctx, [&] { g_realDraw(ctx, n, start); }); +} +void STDMETHODCALLTYPE DrawIndexed_hook(ID3D11DeviceContext* ctx, UINT n, UINT start, INT base) { + replaceDraw(ctx, [&] { g_realDrawIndexed(ctx, n, start, base); }); +} +void STDMETHODCALLTYPE DrawInstanced_hook(ID3D11DeviceContext* ctx, UINT vpi, UINT inst, UINT sv, UINT si) { + replaceDraw(ctx, [&] { g_realDrawInstanced(ctx, vpi, inst, sv, si); }); +} +void STDMETHODCALLTYPE DrawIndexedInstanced_hook(ID3D11DeviceContext* ctx, UINT ipi, UINT inst, UINT si, INT bv, UINT sinst) { + replaceDraw(ctx, [&] { g_realDrawIndexedInstanced(ctx, ipi, inst, si, bv, sinst); }); +} + +// Swaps one vtable entry for our hook, keeping the original. (Many instances share one class vtable, so hooking the +// compositor's device and immediate context hooks them all; we gate on g_device/g_context anyway.) +template +void hookSlot(void* obj, int index, void* hook, T* real) { + void** vtable = *reinterpret_cast(obj); + DWORD old = 0; + if (VirtualProtect(&vtable[index], sizeof(void*), PAGE_EXECUTE_READWRITE, &old)) { + *real = reinterpret_cast(vtable[index]); + vtable[index] = hook; + VirtualProtect(&vtable[index], sizeof(void*), old, &old); + } +} + +// ── the hand-over pipe: Frame Home asks for textures, we make them and share them ────────────────── + +void destroyTextures() { + for (SharedTex& t : g_tex) { + if (t.srv) t.srv->Release(); + if (t.tex) t.tex->Release(); + if (t.handle) CloseHandle(t.handle); + } + g_tex.clear(); + g_lastGood = -1; +} + +bool makeTextures(uint32_t w, uint32_t h) { + destroyTextures(); + g_tex.resize(envshare::kSlots); + for (uint32_t i = 0; i < envshare::kSlots; ++i) { + D3D11_TEXTURE2D_DESC td = {}; + td.Width = w; + td.Height = h; + td.MipLevels = 1; + td.ArraySize = 2; // one layer per eye + td.Format = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB; + td.SampleDesc.Count = 1; + td.Usage = D3D11_USAGE_DEFAULT; + td.BindFlags = D3D11_BIND_SHADER_RESOURCE; + td.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED; + if (FAILED(g_device->CreateTexture2D(&td, nullptr, &g_tex[i].tex))) { logf("shared texture %u failed", i); destroyTextures(); return false; } + D3D11_SHADER_RESOURCE_VIEW_DESC vd = {}; + vd.Format = td.Format; + vd.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2DARRAY; + vd.Texture2DArray.MipLevels = 1; + vd.Texture2DArray.ArraySize = 2; + if (FAILED(g_device->CreateShaderResourceView(g_tex[i].tex, &vd, &g_tex[i].srv))) { logf("shared srv %u failed", i); destroyTextures(); return false; } + IDXGIResource1* res = nullptr; + if (FAILED(g_tex[i].tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast(&res))) || + FAILED(res->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, nullptr, &g_tex[i].handle))) { + if (res) res->Release(); + logf("shared handle %u failed", i); + destroyTextures(); + return false; + } + res->Release(); + } + g_texW = w; + g_texH = h; + return true; +} + +void adapterLuid(uint8_t out[8]) { + std::memset(out, 0, 8); + IDXGIDevice* dxgi = nullptr; + IDXGIAdapter* adapter = nullptr; + if (SUCCEEDED(g_device->QueryInterface(__uuidof(IDXGIDevice), reinterpret_cast(&dxgi))) && SUCCEEDED(dxgi->GetAdapter(&adapter))) { + DXGI_ADAPTER_DESC d; + if (SUCCEEDED(adapter->GetDesc(&d))) std::memcpy(out, &d.AdapterLuid, 8); + } + if (adapter) adapter->Release(); + if (dxgi) dxgi->Release(); +} + +void handleRequest(HANDLE pipe, DWORD clientDefaultPid) { + envshare::WinRequest req = {}; + DWORD got = 0; + if (!ReadFile(pipe, &req, sizeof req, &got, nullptr) || got != sizeof req || req.magic != envshare::kMagic) return; + envshare::WinReply reply = {}; + reply.magic = envshare::kMagic; + reply.generation = req.generation; + + std::lock_guard l(g_texMutex); // the draw path reads g_state and g_tex under this + // Frame Home's shared State: the handle it duplicated into us. Map it (a new mapping replaces an old one). + if (req.state) { + if (g_state) { UnmapViewOfFile(g_state); g_state = nullptr; } + if (g_stateMap) { CloseHandle(g_stateMap); g_stateMap = nullptr; } + g_stateMap = reinterpret_cast(static_cast(req.state)); + g_state = static_cast(MapViewOfFile(g_stateMap, FILE_MAP_ALL_ACCESS, 0, 0, req.stateSize ? req.stateSize : sizeof(envshare::State))); + if (!g_state) { logf("could not map Frame Home's state"); } + } + if (g_device && g_state && makeTextures(req.width, req.height)) { + g_generation = req.generation; + g_lastGood = -1; + DWORD pid = req.pid ? req.pid : clientDefaultPid; + HANDLE client = OpenProcess(PROCESS_DUP_HANDLE, FALSE, pid); + bool ok = client != nullptr; + for (uint32_t i = 0; i < envshare::kSlots && ok; ++i) { + HANDLE dup = nullptr; + ok = DuplicateHandle(GetCurrentProcess(), g_tex[i].handle, client, &dup, 0, FALSE, DUPLICATE_SAME_ACCESS); + reply.texture[i] = reinterpret_cast(dup); + } + if (client) CloseHandle(client); + if (ok) { + adapterLuid(reply.deviceLUID); + reply.ok = 1; + arm(); + logf("Frame Home connected: %ux%u, generation %u", req.width, req.height, req.generation); + } else { + logf("could not share the textures with Frame Home (pid %lu)", pid); + } + } + DWORD wrote = 0; + WriteFile(pipe, &reply, sizeof reply, &wrote, nullptr); + FlushFileBuffers(pipe); +} + +void serve() { + g_connected = false; + for (;;) { + if (fileExists(layerDir() / "off")) { g_off = true; return; } + HANDLE pipe = CreateNamedPipeA(envshare::kPipeName, PIPE_ACCESS_DUPLEX, + PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT, 1, + sizeof(envshare::WinReply) + 64, sizeof(envshare::WinRequest) + 64, 0, nullptr); + if (pipe == INVALID_HANDLE_VALUE) { std::this_thread::sleep_for(std::chrono::seconds(1)); continue; } + if (ConnectNamedPipe(pipe, nullptr) || GetLastError() == ERROR_PIPE_CONNECTED) { + ULONG pid = 0; + GetNamedPipeClientProcessId(pipe, &pid); + g_connected = true; + handleRequest(pipe, pid); + // Hold the connection: when it breaks, Frame Home has gone and we drop to Aurora. + char b; + DWORD n; + while (ReadFile(pipe, &b, 0, &n, nullptr) || GetLastError() == ERROR_MORE_DATA) { + if (fileExists(layerDir() / "off")) break; + } + g_connected = false; + } + DisconnectNamedPipe(pipe); + CloseHandle(pipe); + } +} + +// Watches the off switch each second (turn Frame Home off/on without restarting SteamVR). +void watchOff() { + uint64_t sec = 0; + for (;;) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + const bool off = fileExists(layerDir() / "off"); + if (off != g_off.load()) { g_off = off; logf(off ? "turned off: Aurora as usual" : "turned back on"); } + // A heartbeat (every 5 s) of what the layer is doing, for testing without a headset to look through. + if (++sec % 5 == 0) { + const uint64_t r = g_replaced.exchange(0), s = g_skipped.exchange(0); + if (r || s) logf("drew the home over %llu aurora draws, left %llu undrawn (last 5 s); connected=%d", + (unsigned long long)r, (unsigned long long)s, g_connected.load() ? 1 : 0); + } + } +} + +void wrap(ID3D11Device* dev, ID3D11DeviceContext* ctx) { + static std::once_flag once; + std::call_once(once, [&] { + g_device = dev; + g_context = ctx; + g_device->AddRef(); // our own long-lived references (the context's is the one GetImmediateContext gave us), + // so the vtables we hook stay valid however the compositor manages its own copies + startHealth(); + if (g_off) { logf("vrcompositor device wrapped, but Frame Home is off: Aurora as usual"); return; } + scanAuroraShaders(); + hookSlot(dev, 15, reinterpret_cast(CreatePixelShader_hook), &g_realCreatePixelShader); + hookSlot(ctx, 9, reinterpret_cast(PSSetShader_hook), &g_realPSSetShader); + hookSlot(ctx, 12, reinterpret_cast(DrawIndexed_hook), &g_realDrawIndexed); + hookSlot(ctx, 13, reinterpret_cast(Draw_hook), &g_realDraw); + hookSlot(ctx, 20, reinterpret_cast(DrawIndexedInstanced_hook), &g_realDrawIndexedInstanced); + hookSlot(ctx, 21, reinterpret_cast(DrawInstanced_hook), &g_realDrawInstanced); + std::thread(serve).detach(); + std::thread(watchOff).detach(); + logf("vrcompositor device wrapped: Frame Home is the environment"); + }); +} + +// ── the real d3d11.dll and our D3D11CreateDevice ─────────────────────────────────────────────────── + +using PFN_D3D11CreateDevice = HRESULT(WINAPI*)(IDXGIAdapter*, D3D_DRIVER_TYPE, HMODULE, UINT, const D3D_FEATURE_LEVEL*, UINT, UINT, + ID3D11Device**, D3D_FEATURE_LEVEL*, ID3D11DeviceContext**); +PFN_D3D11CreateDevice realCreateDevice() { + static PFN_D3D11CreateDevice f = [] { + HMODULE m = LoadLibraryW(L"fh_d3d11real.dll"); // the real d3d11.dll, copied beside us by the installer + return m ? reinterpret_cast(GetProcAddress(m, "D3D11CreateDevice")) : nullptr; + }(); + return f; +} + +} // namespace + +extern "C" HRESULT WINAPI D3D11CreateDevice( // exported by layer/d3d11_proxy.def + IDXGIAdapter* adapter, D3D_DRIVER_TYPE driverType, HMODULE software, UINT flags, + const D3D_FEATURE_LEVEL* levels, UINT numLevels, UINT sdk, + ID3D11Device** device, D3D_FEATURE_LEVEL* level, ID3D11DeviceContext** context) { + PFN_D3D11CreateDevice real = realCreateDevice(); + if (!real) return E_FAIL; + // Exactly as the caller asked (no tampering with its device/context or their lifetimes). If it's the compositor + // and it got a device, we take our own reference to the device and its immediate context (wrap() AddRefs them) + // and keep them for the process — so hooking their vtables is safe however the compositor manages its copies. + const HRESULT hr = real(adapter, driverType, software, flags, levels, numLevels, sdk, device, level, context); + if (SUCCEEDED(hr) && isCompositor() && device && *device) { + ID3D11DeviceContext* immediate = nullptr; + (*device)->GetImmediateContext(&immediate); // +1 ref, kept by wrap() + if (immediate) wrap(*device, immediate); + } + return hr; +} + +BOOL WINAPI DllMain(HINSTANCE, DWORD reason, LPVOID) { + if (reason == DLL_PROCESS_ATTACH && isCompositor()) realCreateDevice(); // load the real one early + return TRUE; +} diff --git a/scripts/build-studio.sh b/scripts/build-studio.sh index 4ab161d..fda2212 100755 --- a/scripts/build-studio.sh +++ b/scripts/build-studio.sh @@ -60,17 +60,18 @@ build_pc_home() { say "building Frame Home for $target's SteamVR" "$CMAKE" -S "$ROOT" -B "$dir" -DCMAKE_BUILD_TYPE=Release -DFRAME_HOME_PC=ON "$@" > "$dir.configure.log" 2>&1 || { tail -20 "$dir.configure.log"; say "$target: Frame Home configure failed (full log: $dir.configure.log)"; exit 1; } - "$CMAKE" --build "$dir" --target frame-home frame-switcher --parallel + "$CMAKE" --build "$dir" --target frame-home frame-switcher $([ "$target" = windows-x64 ] && echo d3d11.dll) --parallel } -# pc/ beside Studio: Frame Home for this computer's SteamVR and its installer (Studio's Headset tab runs it). -# Windows: SteamVR's home app (install-frame-home-pc.ps1). Linux: the Steam Frame's way, the compositor layer, with +# pc/ beside Studio: Frame Home for this computer's SteamVR and its installer (Studio's Headset tab runs it). Both +# platforms the Steam Frame's way now — the compositor layer draws the home in Aurora's place. Windows: the layer is +# a proxy d3d11.dll (install-frame-home-pc.ps1 puts it beside SteamVR's compositor). Linux: the Vulkan layer, with # the headset's own install.sh. stage_pc() { target=$1 stage=$2 home="$BUILD/$1-home" case "$target" in windows-x64) mkdir -p "$stage/pc" - cp "$home/frame-home.exe" "$home/frame-switcher.exe" "$stage/pc/" + cp "$home/frame-home.exe" "$home/frame-switcher.exe" "$home/d3d11.dll" "$stage/pc/" cp "$ROOT/frame/pc/install-frame-home-pc.ps1" "$ROOT/frame/pc/Install Frame Home on this PC.bat" \ "$ROOT/frame/pc/Remove Frame Home from this PC.bat" "$stage/pc/" ;; linux-x64) mkdir -p "$stage/pc/bin" "$stage/pc/layer" diff --git a/src/env_export.h b/src/env_export.h index 9e9386c..2ac539f 100644 --- a/src/env_export.h +++ b/src/env_export.h @@ -5,22 +5,176 @@ #include "hsl_renderer.h" #ifdef _WIN32 -// A PC has no compositor layer: Frame Home is SteamVR's scene app there, so system mode never starts (init() fails). +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include + +#include +#include +#include + +// Windows system mode: Frame Home renders the home into the compositor's own Direct3D 11 textures (see env_share.h). +// Frame Home owns the shared State (a file mapping) and hands the layer its handle; the layer owns the textures and +// hands Frame Home theirs. createImages() does that hand-over over the layer's named pipe and imports the textures +// into the renderer; after that it mirrors the Linux EnvExport. class EnvExport { public: - bool init() { return false; } - envshare::State* state() { return &state_; } - void release(HslRenderer&) {} - bool connected() const { return false; } - bool createImages(HslRenderer&) { return false; } - bool connect() { return false; } - void poll() {} - int beginFrame() { return 0; } - void publish(int, const float[2][16], const float[2][3]) {} - uint32_t refusedFlags() const { return 0; } - double sinceSampled() const { return 1e9; } + ~EnvExport() { + closeTextures(); + if (pipe_ != INVALID_HANDLE_VALUE) ::CloseHandle(pipe_); + if (state_) ::UnmapViewOfFile(state_); + if (stateMap_) ::CloseHandle(stateMap_); + } + + bool init() { + stateMap_ = ::CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, sizeof(envshare::State), nullptr); + if (!stateMap_) return false; + void* p = ::MapViewOfFile(stateMap_, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(envshare::State)); + if (!p) return false; + state_ = new (p) envshare::State(); + state_->show = 0; + return true; + } + envshare::State* state() { return state_; } + + // (Re)negotiates the compositor's textures at the renderer's current size and imports them. Called at startup + // (where it waits a little for the layer to come up) and on every resize. True once the home can be drawn into + // the compositor's textures. + bool createImages(HslRenderer& r) { + r_ = &r; + const bool first = !everImported_; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(first ? 3 : 0); + do { + if (negotiate(r)) return true; + if (first) ::Sleep(200); + } while (first && std::chrono::steady_clock::now() < deadline); + return false; + } + bool connected() const { return pipe_ != INVALID_HANDLE_VALUE && imported_; } + + // Keeps the hand-over alive: reconnects and re-imports after the compositor restarts (as the Linux connect() + // re-sends its Hello). Cheap when already connected. + bool connect() { + if (connected()) return true; + if (!r_) return false; + return negotiate(*r_); + } + + // Is the layer still there? (the pipe breaks when the compositor goes away) + void poll() { + if (pipe_ == INVALID_HANDLE_VALUE) return; + DWORD avail = 0; + if (!::PeekNamedPipe(pipe_, nullptr, 0, nullptr, &avail, nullptr)) { + const DWORD e = ::GetLastError(); + if (e == ERROR_BROKEN_PIPE || e == ERROR_PIPE_NOT_CONNECTED || e == ERROR_INVALID_HANDLE) { + std::fprintf(stderr, "[frame-home] the compositor went away: reconnecting\n"); + dropPipe(); + } + } + } + + int beginFrame() { const int k = static_cast(next_); next_ = (next_ + 1) % envshare::kSlots; return k; } + + void publish(int k, const float viewProj[2][16], const float eyePos[2][3]) { + envshare::Slot& s = state_->slot[k]; + s.frame = ++frame_; + s.generation = generation_; + std::memcpy(s.viewProj, viewProj, sizeof s.viewProj); + for (int e = 0; e < 2; ++e) { for (int c = 0; c < 3; ++c) s.eyePos[e][c] = eyePos[e][c]; s.eyePos[e][3] = 1; } + state_->published.store(frame_); + state_->latest.store(static_cast(k), std::memory_order_release); + state_->publishNs.store(envshare::nowNs()); + } + + uint32_t refusedFlags() const { return 0; } // (Windows frames carry no Vulkan create flags for the layer to refuse) + + double sinceSampled() const { + const uint64_t s = state_->sampledNs.load(); + return s ? (envshare::nowNs() - s) / 1e9 : 1e9; + } + + void release(HslRenderer&) { dropPipe(); } + private: - envshare::State state_; + void closeTextures() { for (HANDLE& h : texture_) if (h) { ::CloseHandle(h); h = nullptr; } } + void dropPipe() { + if (pipe_ != INVALID_HANDLE_VALUE) { ::CloseHandle(pipe_); pipe_ = INVALID_HANDLE_VALUE; } + imported_ = false; + } + + // Opens the pipe if need be, asks the layer for textures at this size, imports them. Throttled to once a second + // while it can't connect. On success the renderer holds the imported images and connected() is true. + bool negotiate(HslRenderer& r) { + const auto now = std::chrono::steady_clock::now(); + if (pipe_ == INVALID_HANDLE_VALUE) { + if (now - lastTry_ < std::chrono::seconds(1)) return false; + lastTry_ = now; + pipe_ = ::CreateFileA(envshare::kPipeName, GENERIC_READ | GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, 0, nullptr); + if (pipe_ == INVALID_HANDLE_VALUE) { + static bool said = false; + if (!said) { said = true; std::fprintf(stderr, "[frame-home] waiting for the compositor layer (no pipe yet, error %lu)\n", ::GetLastError()); } + return false; + } + DWORD mode = PIPE_READMODE_MESSAGE; + ::SetNamedPipeHandleState(pipe_, &mode, nullptr, nullptr); + } + // The State mapping, duplicated into the layer's (pipe server's) process so it can map it. + DWORD serverPid = 0; + HANDLE serverProc = nullptr, dupState = nullptr; + if (!::GetNamedPipeServerProcessId(pipe_, &serverPid) || + !(serverProc = ::OpenProcess(PROCESS_DUP_HANDLE, FALSE, serverPid)) || + !::DuplicateHandle(::GetCurrentProcess(), stateMap_, serverProc, &dupState, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + if (serverProc) ::CloseHandle(serverProc); + dropPipe(); + return false; + } + ::CloseHandle(serverProc); + + envshare::WinRequest req; + req.width = r.width(); + req.height = r.height(); + req.format = 87; // DXGI_FORMAT_R8G8B8A8_UNORM_SRGB + req.pid = ::GetCurrentProcessId(); + req.generation = generation_ + 1; + req.state = reinterpret_cast(dupState); + req.stateSize = static_cast(sizeof(envshare::State)); + DWORD wrote = 0; + envshare::WinReply reply = {}; + DWORD got = 0; + if (!::WriteFile(pipe_, &req, sizeof req, &wrote, nullptr) || wrote != sizeof req || + !::ReadFile(pipe_, &reply, sizeof reply, &got, nullptr) || got != sizeof reply || + reply.magic != envshare::kMagic || !reply.ok || reply.generation != req.generation) { + std::fprintf(stderr, "[frame-home] the layer didn't hand over textures (wrote %lu read %lu ok %u gen %u/%u)\n", + wrote, got, reply.ok, reply.generation, req.generation); + dropPipe(); + return false; // (the layer now owns dupState: it closes it, or drops it when the pipe breaks) + } + // The compositor's adapter: Frame Home must be on it, or the import silently reads another GPU's memory. + if (const uint8_t* luid = r.deviceLUID(); std::memcmp(luid, reply.deviceLUID, 8) != 0) { + static bool said = false; + if (!said) { said = true; std::fprintf(stderr, "[frame-home] the compositor draws on a different GPU than Frame Home: the home can't be shared with it\n"); } + dropPipe(); + return false; + } + closeTextures(); + for (uint32_t i = 0; i < envshare::kSlots; ++i) texture_[i] = reinterpret_cast(static_cast(reply.texture[i])); + if (!r.importWin32Images(envshare::kSlots, reply.texture, req.width, req.height)) { dropPipe(); return false; } + generation_ = req.generation; + imported_ = everImported_ = true; + std::fprintf(stderr, "[frame-home] the compositor's textures are ours now: Frame Home is the environment (%ux%u per eye)\n", req.width, req.height); + return true; + } + + envshare::State* state_ = nullptr; + HANDLE stateMap_ = nullptr; + HANDLE pipe_ = INVALID_HANDLE_VALUE; + HANDLE texture_[envshare::kSlots] = {}; + HslRenderer* r_ = nullptr; + uint32_t next_ = 0, generation_ = 0; + uint64_t frame_ = 0; + bool imported_ = false, everImported_ = false; + std::chrono::steady_clock::time_point lastTry_; }; #else #include diff --git a/src/env_share.h b/src/env_share.h index 10df7c4..b7612ed 100644 --- a/src/env_share.h +++ b/src/env_share.h @@ -8,9 +8,17 @@ // published frame (rotation reprojection from the pose that frame was rendered with). No frame, or a stale one, // and the layer leaves Aurora alone: Frame Home stopping or crashing never blanks the headset. // -// Hand-over: Frame Home connects to the layer's unix socket (kSocketName in $XDG_RUNTIME_DIR) and sends one Hello -// with SCM_RIGHTS fds: kSlots image memories, then the shared State (a memfd). A new Hello (new size) replaces the -// old images. +// Hand-over, Linux: Frame Home connects to the layer's unix socket (kSocketName in $XDG_RUNTIME_DIR) and sends one +// Hello with SCM_RIGHTS fds: kSlots image memories, then the shared State (a memfd). A new Hello (new size) replaces +// the old images. Frame Home owns the images (Vulkan external memory, opaque fds) and the layer imports them. +// +// Hand-over, Windows: the compositor is Direct3D 11, which cannot open Vulkan's exported memory (only the reverse), +// so the images' ownership flips while the State stays Frame Home's (as on Linux). Frame Home connects to the layer's +// named pipe (kPipeName) and sends a WinRequest: the size and format it wants, and the shared State's file-mapping +// handle duplicated into the layer's process. The layer, inside vrcompositor, creates kSlots shared D3D11 textures, +// duplicates their NT handles into Frame Home's process, and replies with a WinReply carrying the handle values and +// the device LUID. Frame Home imports the textures into Vulkan (VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_BIT) and +// renders the home into them. A new WinRequest (new size) makes the layer build a new generation and reply fresh. #include #include #include @@ -19,13 +27,39 @@ namespace envshare { -constexpr const char* kSocketName = "framehome-env.sock"; +constexpr const char* kSocketName = "framehome-env.sock"; // Linux: in $XDG_RUNTIME_DIR +constexpr const char* kPipeName = "\\\\.\\pipe\\framehome-env"; // Windows: the layer's named pipe +constexpr const char* kStateMapName = "Local\\framehome-env-state"; // Windows: the shared State mapping (+ a + // generation suffix, so a new one is distinct) constexpr uint32_t kMagic = 0x46484556; // "FHEV" constexpr uint32_t kVersion = 5; // 4: Hello says the images' usage (Frame Home draws straight into them) // 5: ...and their create flags (subsampled: the compositor smooths foveation) // Four: the compositor reads the newest while Frame Home publishes the one before it finished and renders the next. constexpr uint32_t kSlots = 4; +// No cross-API GPU lock on the shared textures (as on Linux, where the images carry none): Frame Home publishes a +// slot only once the GPU has finished writing it, and the four slots rotate, so the layer always reads a slot that +// Frame Home finished and isn't touching. A torn read would need the layer to read the very slot being written four +// frames on, which the rotation prevents. +// +// Windows hand-over messages on the pipe (little-endian, fixed size). +struct WinRequest { + uint32_t magic = kMagic, version = kVersion; + uint32_t width = 0, height = 0, layers = 2; + uint32_t format = 0; // DXGI_FORMAT the textures should be (87 = R8G8B8A8_UNORM_SRGB) + uint32_t pid = 0; // Frame Home's process id (the layer duplicates the texture handles into it) + uint32_t generation = 0; // which set Frame Home is asking to (re)create + uint64_t state = 0; // the shared State's file-mapping handle, duplicated into the layer's process + uint32_t stateSize = 0; // sizeof(State) as Frame Home built it (the layer maps that much) +}; +struct WinReply { + uint32_t magic = kMagic, version = kVersion; + uint32_t ok = 0; // 0: the layer couldn't make them (Frame Home keeps waiting) + uint32_t generation = 0; + uint64_t texture[kSlots] = {}; // NT handles (duplicated into Frame Home), as integers + uint8_t deviceLUID[8] = {}; // the compositor's adapter: Frame Home must render on the same one +}; + struct Hello { uint32_t magic = kMagic, version = kVersion; uint32_t width = 0, height = 0, layers = 2; diff --git a/src/home_platform.h b/src/home_platform.h index 38bc77b..23275bc 100644 --- a/src/home_platform.h +++ b/src/home_platform.h @@ -142,6 +142,23 @@ inline int run(const std::vector& args, int nice, bool wait) { CloseHandle(pi.hProcess); return static_cast(code); } +// Starts args[0] detached (its own process, no window) and returns its pid, or 0. appKey, when given, is set as +// STEAMVR_APPKEY for the child so SteamVR takes it as that application (the environment renderer is its own identity). +inline Pid spawn(const std::vector& args, const char* appKey = nullptr) { + std::wstring cl = commandLine(args); + if (appKey) SetEnvironmentVariableA("STEAMVR_APPKEY", appKey); + STARTUPINFOW si = {}; + si.cb = sizeof si; + PROCESS_INFORMATION pi = {}; + const BOOL ok = CreateProcessW(nullptr, cl.data(), nullptr, nullptr, FALSE, CREATE_NO_WINDOW | DETACHED_PROCESS, nullptr, nullptr, &si, &pi); + if (appKey) SetEnvironmentVariableA("STEAMVR_APPKEY", nullptr); + if (!ok) return 0; + const Pid pid = pi.dwProcessId; + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return pid; +} + // This program again with the same arguments (settings that build the renderer differently, a lost GPU); this one // ends. SteamVR knows its home app by its file, so the new one is still the environment. [[noreturn]] inline void restartSelf() { diff --git a/src/hsl_renderer.cpp b/src/hsl_renderer.cpp index 6092e23..dcebb2f 100644 --- a/src/hsl_renderer.cpp +++ b/src/hsl_renderer.cpp @@ -806,6 +806,12 @@ bool HslRenderer::init(uint32_t eyeWidth, uint32_t eyeHeight) { if (astcHdr_) dext.push_back(VK_EXT_TEXTURE_COMPRESSION_ASTC_HDR_EXTENSION_NAME); if (fdmOn_) dext.push_back(VK_EXT_FRAGMENT_DENSITY_MAP_EXTENSION_NAME); if (fdmOffset_) dext.push_back(VK_QCOM_FRAGMENT_DENSITY_MAP_OFFSET_EXTENSION_NAME); +#ifdef _WIN32 + // Windows system mode: import the compositor's Direct3D 11 textures to render the home into them. + if (importWin32) + for (const char* e : {"VK_KHR_external_memory", "VK_KHR_external_memory_win32"}) + if (std::none_of(dext.begin(), dext.end(), [&](const char* d) { return !std::strcmp(d, e); })) dext.push_back(e); +#endif // FH_PERF: the GPU's counters, for the bench (see perfCounterAverages). std::vector perfIdx; VkPhysicalDevicePerformanceQueryFeaturesKHR perfFeat = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PERFORMANCE_QUERY_FEATURES_KHR}; @@ -866,6 +872,15 @@ bool HslRenderer::init(uint32_t eyeWidth, uint32_t eyeHeight) { dci.ppEnabledExtensionNames = dext.data(); if (!VK_OK(vkCreateDevice(physical_, &dci, nullptr, &device_))) return false; vkGetDeviceQueue(device_, queueFamily_, 0, &queue_); +#ifdef _WIN32 + if (importWin32) { // the adapter this device is on, so the compositor can check it's the one it draws with + VkPhysicalDeviceIDProperties idp = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES}; + VkPhysicalDeviceProperties2 pp = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; + pp.pNext = &idp; + vkGetPhysicalDeviceProperties2(physical_, &pp); + if (idp.deviceLUIDValid) std::memcpy(deviceLUID_, idp.deviceLUID, 8); + } +#endif if (!perfIdx.empty()) { VkQueryPoolPerformanceCreateInfoKHR pc = {VK_STRUCTURE_TYPE_QUERY_POOL_PERFORMANCE_CREATE_INFO_KHR}; pc.queueFamilyIndex = queueFamily_; @@ -4187,6 +4202,65 @@ bool HslRenderer::createExportImages(uint32_t n, std::vector& out, return true; } +#ifdef _WIN32 +// Windows system mode: the compositor made kSlots shared Direct3D 11 textures (R8G8B8A8_UNORM_SRGB, `n` layers) at +// this render size and duplicated their NT handles into us. Each becomes a VkImage we render the home into; render() +// copies the eye frame into exportImages_[exportSlot] through the ordinary (non-direct) copy path. No cross-API lock: +// a slot is published only once its frame is retired and the four rotate, as on Linux. +bool HslRenderer::importWin32Images(uint32_t n, const uint64_t handles[], uint32_t w, uint32_t h) { + vkDeviceWaitIdle(device_); // an earlier generation may still be in flight + for (VkImage img : exportImages_) vkDestroyImage(device_, img, nullptr); + for (VkDeviceMemory m : importMem_) vkFreeMemory(device_, m, nullptr); + exportImages_.clear(); + importMem_.clear(); + exportInited_.clear(); + exportSlot = -1; + exportDirect = false; // copied in (the imported textures aren't our render targets) + for (uint32_t i = 0; i < n; ++i) { + VkExternalMemoryImageCreateInfo ext = {VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO}; + ext.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_BIT; + VkImageCreateInfo ci = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + ci.pNext = &ext; + ci.imageType = VK_IMAGE_TYPE_2D; + ci.format = VK_FORMAT_R8G8B8A8_SRGB; // == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB + ci.extent = {w, h, 1}; + ci.mipLevels = 1; + ci.arrayLayers = viewCount; // 2 layers, one per eye + ci.samples = VK_SAMPLE_COUNT_1_BIT; + ci.tiling = VK_IMAGE_TILING_OPTIMAL; + ci.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + ci.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + ci.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + VkImage image = VK_NULL_HANDLE; + if (vkCreateImage(device_, &ci, nullptr, &image) != VK_SUCCESS) { LOG("system mode: import image %u failed", i); return false; } + VkMemoryRequirements req; + vkGetImageMemoryRequirements(device_, image, &req); + VkMemoryDedicatedAllocateInfo ded = {VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO}; + ded.image = image; + VkImportMemoryWin32HandleInfoKHR imp = {VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR}; + imp.pNext = &ded; + imp.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_BIT; + imp.handle = reinterpret_cast(static_cast(handles[i])); + VkMemoryAllocateInfo ai = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + ai.pNext = &imp; + ai.allocationSize = req.size; + ai.memoryTypeIndex = memoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + VkDeviceMemory mem = VK_NULL_HANDLE; + if (vkAllocateMemory(device_, &ai, nullptr, &mem) != VK_SUCCESS || vkBindImageMemory(device_, image, mem, 0) != VK_SUCCESS) { + LOG("system mode: import memory %u failed (adapter mismatch, or the compositor's textures went away)", i); + if (mem) vkFreeMemory(device_, mem, nullptr); + vkDestroyImage(device_, image, nullptr); + return false; + } + exportImages_.push_back(image); + importMem_.push_back(mem); + } + exportInited_.assign(n, false); + LOG("system mode: imported %u compositor textures (%ux%u, %u layers)", n, w, h, viewCount); + return true; +} +#endif + uint32_t HslRenderer::exportUsage() const { return VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | (exportDirect ? VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT : 0u); @@ -4300,6 +4374,12 @@ void HslRenderer::shutdown() { vkDestroyFramebuffer(device_, framebuffer_, nullptr); vkDestroyRenderPass(device_, renderPass_, nullptr); if (exportPass_) vkDestroyRenderPass(device_, exportPass_, nullptr); +#ifdef _WIN32 + for (VkImage img : exportImages_) vkDestroyImage(device_, img, nullptr); // imported compositor textures + for (VkDeviceMemory m : importMem_) vkFreeMemory(device_, m, nullptr); + exportImages_.clear(); + importMem_.clear(); +#endif if (pipelineCache_) { size_t n = 0; if (!pipelineCachePath.empty() && vkGetPipelineCacheData(device_, pipelineCache_, &n, nullptr) == VK_SUCCESS && n) { diff --git a/src/hsl_renderer.h b/src/hsl_renderer.h index 563ca1c..3bea991 100644 --- a/src/hsl_renderer.h +++ b/src/hsl_renderer.h @@ -12,6 +12,9 @@ #include "shader_bake.h" #include "spv_reflect.h" +#ifdef _WIN32 +#define VK_USE_PLATFORM_WIN32_KHR // the Win32 external-memory structs (system mode imports D3D11 textures) +#endif #ifdef FH_USE_VOLK #include // Windows: Vulkan entry points loaded at runtime (no SDK needed) #else @@ -154,6 +157,15 @@ public: uint32_t exportUsage() const; // the export images' usage flags (the layer imports them alike) uint32_t exportCreateFlags() const; // ...and their VkImageCreateFlags (likewise) void destroyExportImages(std::vector& images); +#ifdef _WIN32 + // Windows system mode: the compositor (Direct3D 11) owns the shared textures and hands Frame Home their NT + // handles (env_share.h). Frame Home imports each as a VkImage, and render() copies the eye frame into + // exportImages_[exportSlot] exactly as the Linux copy path does. Set importWin32 before init() so the device is + // created with VK_KHR_external_memory_win32. + bool importWin32 = false; + bool importWin32Images(uint32_t n, const uint64_t handles[], uint32_t width, uint32_t height); + const uint8_t* deviceLUID() const { return deviceLUID_; } // the adapter this device is on (valid after init) +#endif int exportSlot = -1; void waitFrame(); // Frames counted on the GPU (a timeline semaphore, when the device has them): render() signals frameSerial() once @@ -384,6 +396,10 @@ private: void densityTexels(float cx, float cy, std::vector& dens) const; // two per texel: across, down void packDensity(const std::vector& dens, uint8_t* dst) const; // in the map's format std::vector exportInited_; +#ifdef _WIN32 + std::vector importMem_; // importWin32Images: the imported memories, freed on shutdown + uint8_t deviceLUID_[8] = {}; +#endif VkDescriptorPool descPool_ = VK_NULL_HANDLE; // Selection overlay (selection_spirv.h): edges as a line list + faces, solid colour from a 1x1 texture. struct Selection { diff --git a/src/main.cpp b/src/main.cpp index 9712f10..110ea57 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -962,6 +962,26 @@ int runVr(const std::vector& apks, bool system = false) { fs::create_directories(rootDir() / "logs", ec); if (homeos::reopenStderr(rootDir() / "logs" / "frame-home.log", std::getenv("FH_RESTARTED") ? "a" : "w")) setvbuf(stderr, nullptr, _IOLBF, 0); } +#ifdef _WIN32 + // The environment renderer runs one at a time (a restarted panel, or a stray second launch, mustn't make a + // second renderer fighting over the compositor's textures). A named mutex guards it; a pid file lets the panel + // find the running one. Both go when this process ends. + struct SystemGuard { + HANDLE mutex = nullptr; + bool held = false; + fs::path pidFile; + ~SystemGuard() { if (held) { std::error_code ec; fs::remove(pidFile, ec); } if (mutex) CloseHandle(mutex); } + } systemGuard; + if (system) { + systemGuard.mutex = CreateMutexW(nullptr, TRUE, L"Local\\FrameHomeSystemEnv"); + if (systemGuard.mutex && GetLastError() == ERROR_ALREADY_EXISTS) { LOG("another frame-home --system is running: this one steps aside"); return 0; } + systemGuard.held = true; + std::error_code ec; + fs::create_directories(rootDir() / "layer", ec); + systemGuard.pidFile = rootDir() / "layer" / "system.pid"; + if (FILE* f = homeos::openFile(systemGuard.pidFile, "w")) { std::fprintf(f, "%lu\n", static_cast(GetCurrentProcessId())); std::fclose(f); } + } +#endif vr::EVRInitError ierr = vr::VRInitError_None; vr::VR_Init(&ierr, system ? vr::VRApplication_Overlay : vr::VRApplication_Scene); if (ierr != vr::VRInitError_None) { LOG("VR_Init: %s", vr::VR_GetVRInitErrorAsEnglishDescription(ierr)); return 1; } @@ -996,9 +1016,14 @@ int runVr(const std::vector& apks, bool system = false) { std::string dext(dn, '\0'); if (dn) vr::VRCompositor()->GetVulkanDeviceExtensionsRequired(pd, &dext[0], dn); r.deviceExtensions = split(dext.c_str()); - if (system) // the frames go to the compositor's device as external memory + if (system) { // the frames go to the compositor's device as external memory +#ifdef _WIN32 + r.importWin32 = true; // the renderer adds VK_KHR_external_memory_win32 (the compositor owns the textures) +#else for (const char* e : {"VK_KHR_external_memory", "VK_KHR_external_memory_fd"}) if (std::find(r.deviceExtensions.begin(), r.deviceExtensions.end(), e) == r.deviceExtensions.end()) r.deviceExtensions.push_back(e); +#endif + } vkDestroyInstance(probe, nullptr); } r.pipelineCachePath = (rootDir() / "pipelines.bin").string(); diff --git a/src/switcher.cpp b/src/switcher.cpp index 9d66c09..857f2ca 100644 --- a/src/switcher.cpp +++ b/src/switcher.cpp @@ -1551,11 +1551,9 @@ int runPanel() { // game is open. Here the panel makes sure that's on, and that the moments before Frame Home draws (SteamVR // starting, a game quitting) show the home too: SteamVR's background is a 360-degree picture of it // (keepBackground), with the classic environment instead of Aurora. -#ifdef _WIN32 - const bool systemEnv = false; -#else + // The compositor layer is installed (the Frame always; a PC once Frame Home is installed there): its folder + // (rootDir()/layer) is the marker. The layer draws the home in Aurora's place and frame-home --system renders it. const bool systemEnv = fs::exists(rootDir() / "layer"); -#endif // (Frame Home uninstalled from a PC: its manifest is gone, and SteamVR's settings are left as the uninstall put them) const bool homeInstalled = systemEnv || fs::exists(rootDir() / "frame-home.vrmanifest"); if (!systemEnv && homeInstalled) { @@ -1578,9 +1576,12 @@ int runPanel() { } if (systemEnv) { // its own SteamVR identity (hidden: no list, no dock), so its controller bindings are its own const fs::path m = rootDir() / "frame-environment.vrmanifest"; + const std::string lin = (rootDir() / "bin" / "frame-home").string(); + std::string win = (rootDir() / "bin" / "frame-home.exe").string(); + for (size_t i = 0; (i = win.find('\\', i)) != std::string::npos; i += 2) win.replace(i, 1, "\\\\"); // JSON-escape the backslashes std::ofstream(m) << "{\n \"source\": \"builtin\",\n \"applications\": [ {\n \"app_key\": \"framehome.environment\",\n" - " \"launch_type\": \"binary\",\n \"binary_path_linux\": \"" << (rootDir() / "bin" / "frame-home").string() << "\",\n" - " \"binary_path_linux_arm\": \"" << (rootDir() / "bin" / "frame-home").string() << "\",\n \"arguments\": \"--system\",\n" + " \"launch_type\": \"binary\",\n \"binary_path_linux\": \"" << lin << "\",\n" + " \"binary_path_linux_arm\": \"" << lin << "\",\n \"binary_path_windows\": \"" << win << "\",\n \"arguments\": \"--system\",\n" " \"is_dashboard_overlay\": true,\n \"is_hidden\": true,\n" " \"strings\": { \"en_us\": { \"name\": \"Frame Home\", \"description\": \"The environment\" } }\n } ]\n}\n"; const vr::EVRApplicationError ae = vr::VRApplications()->AddApplicationManifest(fs::absolute(m).u8string().c_str(), false); @@ -1632,7 +1633,20 @@ int runPanel() { } if (pid > 0) { systemPid = pid; LOG("started frame-home --system (pid %d)", static_cast(pid)); } #else - (void)now; + // Windows: the panel starts frame-home --system itself and keeps it alive. A pid file (written by the + // renderer, one at a time behind a named mutex) lets a restarted panel find the one already running. + std::error_code ec; + if (systemPid > 0 && homeos::alive(systemPid)) return; + if (fs::exists(rootDir() / "layer" / "off", ec)) return; + { + std::ifstream in(rootDir() / "layer" / "system.pid"); + long p = 0; + if (in >> p && p > 0 && homeos::alive(static_cast(p))) { systemPid = static_cast(p); adopted = true; return; } + } + if (now - lastSpawn < std::chrono::seconds(5)) return; // a crash loop doesn't spin + lastSpawn = now; + systemPid = homeos::spawn({(rootDir() / "bin" / "frame-home.exe").string(), "--system"}, "framehome.environment"); + if (systemPid > 0) LOG("started frame-home --system (pid %lu)", static_cast(systemPid)); #endif }; // The panel's pixels go to SteamVR in Vulkan images of our own (overlay_texture.h): SetOverlayRaw recreated the