steam-frame/its-ven--FrameDepth
1
0 Fork 0
mirror of https://github.com/its-ven/FrameDepth.git synced 2026-10-17 18:02:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
its-ven authored and GitHub committed 2026-10-11 22:06:14 +02:00
1 parent 279867c50a
commit 40c11fdb2f
11 files changed
+2000
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

depth3d.py
+447
View File
@@ -0,0 +1,447 @@
"""
depth3d.py - screen capture + depth estimation core, and a simple side-by-side (SBS) window.
Two roles:
* Library: `Grabber` (Windows Graphics Capture) and `DepthWorker` (Depth Anything V2 on the
GPU, scene-cut aware) are used by depth3d_vr.py, the main SteamVR player.
* Standalone: `python depth3d.py --source-monitor 1` opens a plain SBS window that you can
feed to a 3D-capable viewer (Desktop+ HSBS, Bigscreen...). Simpler and less capable than
the VR player: older warp shader, no menu, no environments.
Controls of the standalone SBS window:
Up / Down depth strength
Left / Right convergence (which depth sits at the screen plane)
X swap eyes
D toggle depth-map debug view
F toggle fullscreen
Esc quit
"""
import argparse
import threading
import time
import glfw
import moderngl
import numpy as np
import torch
import torch.nn.functional as F
from transformers import AutoModelForDepthEstimation
from windows_capture import Frame, InternalCaptureControl, WindowsCapture
MODELS = {
"small": "depth-anything/Depth-Anything-V2-Small-hf",
"base": "depth-anything/Depth-Anything-V2-Base-hf",
"large": "depth-anything/Depth-Anything-V2-Large-hf",
}
# --------------------------------------------------------------------------
# Window listing helper
# --------------------------------------------------------------------------
def list_windows():
import ctypes
from ctypes import wintypes
user32 = ctypes.windll.user32
titles = []
@ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM)
def cb(hwnd, _):
if user32.IsWindowVisible(hwnd):
n = user32.GetWindowTextLengthW(hwnd)
if n:
buf = ctypes.create_unicode_buffer(n + 1)
user32.GetWindowTextW(hwnd, buf, n + 1)
titles.append(buf.value)
return True
user32.EnumWindows(cb, 0)
return titles
# --------------------------------------------------------------------------
# Capture
# --------------------------------------------------------------------------
class Grabber:
"""Keeps only the newest captured frame (BGRA, HxWx4 uint8)."""
def __init__(self, window_name=None, monitor_index=None, cursor=True):
self.lock = threading.Lock()
self.frame = None
self.seq = 0
kwargs = dict(cursor_capture=cursor, draw_border=False)
if window_name:
kwargs["window_name"] = window_name
else:
kwargs["monitor_index"] = monitor_index # 1-based
cap = WindowsCapture(**kwargs)
@cap.event
def on_frame_arrived(frame: Frame, ctrl: InternalCaptureControl):
buf = np.array(frame.frame_buffer, copy=True)
with self.lock:
self.frame = buf
self.seq += 1
@cap.event
def on_closed():
print("Capture session closed.")
self.control = cap.start_free_threaded()
def latest(self):
with self.lock:
return self.frame, self.seq
def stop(self):
try:
self.control.stop()
except Exception:
pass
# --------------------------------------------------------------------------
# Depth estimation worker
# --------------------------------------------------------------------------
class DepthWorker(threading.Thread):
def __init__(self, grabber, width, smooth, model_id, cut_threshold=0.12):
super().__init__(daemon=True)
self.grabber = grabber
self.width = max(112, (width // 14) * 14)
self.smooth = smooth
self.cut_threshold = cut_threshold
self.active = True # set False to idle the model (e.g. when 3D is switched off)
self.frame = None # the captured frame the current depth belongs to
self.stop_evt = threading.Event()
self.lock = threading.Lock()
self.depth = None # float32 HxW in 0..1 (1 = near)
self.seq = 0
self.fps = 0.0
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.dtype = torch.float16 if self.device == "cuda" else torch.float32
if self.device == "cpu":
print("WARNING: CUDA not available, running on CPU (will be slow).")
print(f"Loading {model_id} on {self.device} ...")
self.model = (
AutoModelForDepthEstimation.from_pretrained(model_id)
.to(self.device, self.dtype)
.eval()
)
self.mean = torch.tensor([0.485, 0.456, 0.406], device=self.device).view(1, 3, 1, 1)
self.std = torch.tensor([0.229, 0.224, 0.225], device=self.device).view(1, 3, 1, 1)
def latest(self):
with self.lock:
return self.depth, self.seq
def latest_pair(self):
"""Depth plus the exact captured frame it was computed from (keeps them in sync)."""
with self.lock:
return self.depth, self.frame, self.seq
def run(self):
last_seq = -1
lo = hi = None
prev = None
prev_thumb = None
n, t0 = 0, time.time()
while not self.stop_evt.is_set():
if not self.active:
time.sleep(0.02)
continue
frame, seq = self.grabber.latest()
if frame is None or seq == last_seq:
time.sleep(0.002)
continue
last_seq = seq
with torch.inference_mode():
h, w = frame.shape[:2]
mw = self.width
mh = max(14, int(round(mw * h / w / 14)) * 14)
t = torch.from_numpy(frame).to(self.device, non_blocking=True)
t = t[..., [2, 1, 0]].permute(2, 0, 1).unsqueeze(0).float() / 255.0
t = F.interpolate(t, size=(mh, mw), mode="bilinear",
antialias=True, align_corners=False)
# Tiny luminance thumbnail -> scene-cut / motion detection
thumb = F.adaptive_avg_pool2d(t, (18, 32)).mean(1)
if prev_thumb is None:
diff, cut = 1.0, True
else:
diff = (thumb - prev_thumb).abs().mean().item()
cut = diff > self.cut_threshold
prev_thumb = thumb
if cut: # hard cut: forget everything from the old shot
lo = hi = None
prev = None
t = ((t - self.mean) / self.std).to(self.dtype)
d = self.model(pixel_values=t).predicted_depth[0].float() # HxW, larger = nearer
# Robust, temporally smoothed normalisation (avoids pumping/flicker)
sample = d.flatten()[::7]
q = torch.quantile(sample, torch.tensor([0.02, 0.98], device=self.device))
l, hgh = q[0].item(), q[1].item()
if lo is None:
lo, hi = l, hgh
else:
lo += 0.08 * (l - lo)
hi += 0.08 * (hgh - hi)
dn = ((d - lo) / max(hi - lo, 1e-6)).clamp(0, 1)
if prev is not None and prev.shape == dn.shape:
# calm shot -> smooth heavily; lots of motion -> trust the new frame
alpha = self.smooth + (1 - self.smooth) * min(1.0, diff / 0.06)
dn = prev * (1 - alpha) + dn * alpha
prev = dn
out = dn.cpu().numpy().astype(np.float32)
with self.lock:
self.depth = out
self.frame = frame
self.seq += 1
n += 1
if time.time() - t0 >= 2.0:
self.fps = n / (time.time() - t0)
n, t0 = 0, time.time()
# --------------------------------------------------------------------------
# Rendering
# --------------------------------------------------------------------------
VERT = """
#version 330
in vec2 pos;
out vec2 uv;
void main() {
uv = vec2(pos.x * 0.5 + 0.5, 1.0 - (pos.y * 0.5 + 0.5));
gl_Position = vec4(pos, 0.0, 1.0);
}
"""
FRAG = """
#version 330
uniform sampler2D color_tex;
uniform sampler2D depth_tex;
uniform float strength; // max horizontal shift per eye, as fraction of width
uniform float convergence; // depth value (0..1) that sits at the screen plane
uniform float eye; // -1 = left, +1 = right
uniform int steps;
uniform int show_depth;
in vec2 uv;
out vec4 fragColor;
void main() {
// Backward warp via fixed-point iteration: find the source pixel whose
// own depth-based shift lands on this output pixel.
vec2 src = uv;
for (int i = 0; i < steps; i++) {
float d = texture(depth_tex, src).r;
src.x = uv.x + eye * strength * (d - convergence);
}
if (show_depth == 1) {
fragColor = vec4(vec3(texture(depth_tex, uv).r), 1.0);
} else {
fragColor = vec4(texture(color_tex, src).rgb, 1.0);
}
}
"""
def make_tex(ctx, w, h, comps, dtype, data=None):
tex = ctx.texture((w, h), comps, data, dtype=dtype)
tex.filter = (moderngl.LINEAR, moderngl.LINEAR)
tex.repeat_x = False
tex.repeat_y = False
return tex
def main():
ap = argparse.ArgumentParser(description="Window -> depth -> SBS 3D")
ap.add_argument("--window", help="Title (or part of it) of the window to capture")
ap.add_argument("--source-monitor", type=int, default=None,
help="Capture a whole monitor instead of a window (1 = first monitor)")
ap.add_argument("--list", action="store_true",
help="List visible window titles and monitors, then exit")
ap.add_argument("--model", choices=list(MODELS), default="base",
help="Depth model size (small = fastest, large = best quality)")
ap.add_argument("--depth-width", type=int, default=700,
help="Model input width (lower = faster). Try 518 or 392.")
ap.add_argument("--strength", type=float, default=0.025)
ap.add_argument("--convergence", type=float, default=0.4)
ap.add_argument("--steps", type=int, default=4, help="Warp iterations (quality/cost)")
ap.add_argument("--smooth", type=float, default=0.5,
help="Depth temporal blend, 1 = no smoothing, lower = smoother")
ap.add_argument("--out-monitor", type=int, default=None,
help="Start the SBS output fullscreen on this monitor (0-based). "
"Must NOT be the monitor you are capturing.")
ap.add_argument("--width", type=int, default=1600)
ap.add_argument("--height", type=int, default=900)
args = ap.parse_args()
if args.list:
print("Windows:")
for t in sorted(set(list_windows())):
print(" ", t)
if glfw.init():
print("Monitors (--source-monitor is 1-based, --out-monitor is 0-based):")
for i, m in enumerate(glfw.get_monitors()):
vm = glfw.get_video_mode(m)
x, y = glfw.get_monitor_pos(m)
print(f" out={i} source={i + 1} {glfw.get_monitor_name(m).decode(errors='ignore')}"
f" {vm.size.width}x{vm.size.height} at ({x},{y})")
glfw.terminate()
return
if not args.window and args.source_monitor is None:
ap.error("give --window TITLE or --source-monitor N (use --list to see options)")
if args.window and args.source_monitor is not None:
ap.error("use either --window or --source-monitor, not both")
if args.source_monitor is not None and args.out_monitor is not None \
and args.source_monitor - 1 == args.out_monitor:
ap.error("output monitor and captured monitor are the same: that would feed back on itself")
grabber = Grabber(args.window, args.source_monitor)
worker = DepthWorker(grabber, args.depth_width, args.smooth, MODELS[args.model])
worker.start()
if not glfw.init():
raise RuntimeError("GLFW init failed")
glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 3)
glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 3)
glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE)
monitors = glfw.get_monitors()
mon = None
w, h = args.width, args.height
if args.out_monitor is not None and args.out_monitor < len(monitors):
mon = monitors[args.out_monitor]
mode = glfw.get_video_mode(mon)
w, h = mode.size.width, mode.size.height
win = glfw.create_window(w, h, "depth3d SBS", mon, None)
glfw.make_context_current(win)
glfw.swap_interval(1)
ctx = moderngl.create_context()
prog = ctx.program(vertex_shader=VERT, fragment_shader=FRAG)
vbo = ctx.buffer(np.array([-1, -1, 1, -1, -1, 1, 1, 1], dtype="f4").tobytes())
vao = ctx.vertex_array(prog, [(vbo, "2f", "pos")])
state = {
"strength": args.strength,
"conv": args.convergence,
"swap": False,
"debug": False,
"fullscreen": mon is not None,
"windowed_geom": (100, 100, args.width, args.height),
}
def on_key(window, key, scancode, action, mods):
if action not in (glfw.PRESS, glfw.REPEAT):
return
if key == glfw.KEY_ESCAPE:
glfw.set_window_should_close(window, True)
elif key == glfw.KEY_UP:
state["strength"] = min(0.1, state["strength"] + 0.0025)
elif key == glfw.KEY_DOWN:
state["strength"] = max(0.0, state["strength"] - 0.0025)
elif key == glfw.KEY_RIGHT:
state["conv"] = min(1.0, state["conv"] + 0.02)
elif key == glfw.KEY_LEFT:
state["conv"] = max(0.0, state["conv"] - 0.02)
elif key == glfw.KEY_X and action == glfw.PRESS:
state["swap"] = not state["swap"]
elif key == glfw.KEY_D and action == glfw.PRESS:
state["debug"] = not state["debug"]
elif key == glfw.KEY_F and action == glfw.PRESS:
if state["fullscreen"]:
x, y, ww, hh = state["windowed_geom"]
glfw.set_window_monitor(window, None, x, y, ww, hh, 0)
else:
x, y = glfw.get_window_pos(window)
ww, hh = glfw.get_window_size(window)
state["windowed_geom"] = (x, y, ww, hh)
m = glfw.get_primary_monitor()
vm = glfw.get_video_mode(m)
glfw.set_window_monitor(window, m, 0, 0, vm.size.width, vm.size.height,
vm.refresh_rate)
state["fullscreen"] = not state["fullscreen"]
glfw.set_key_callback(win, on_key)
color_tex = None
color_size = None
depth_tex = make_tex(ctx, 2, 2, 1, "f4", np.full((2, 2), 0.5, dtype="f4").tobytes())
depth_size = (2, 2)
last_color_seq = -1
last_depth_seq = -1
last_title = 0.0
prog["color_tex"].value = 0
prog["depth_tex"].value = 1
prog["steps"].value = args.steps
try:
while not glfw.window_should_close(win):
glfw.poll_events()
frame, cseq = grabber.latest()
if frame is not None and cseq != last_color_seq:
last_color_seq = cseq
fh, fw = frame.shape[:2]
if color_size != (fw, fh):
if color_tex is not None:
color_tex.release()
color_tex = make_tex(ctx, fw, fh, 4, "f1")
color_tex.swizzle = "BGRA"
color_size = (fw, fh)
color_tex.write(np.ascontiguousarray(frame))
depth, dseq = worker.latest()
if depth is not None and dseq != last_depth_seq:
last_depth_seq = dseq
dh, dw = depth.shape
if depth_size != (dw, dh):
depth_tex.release()
depth_tex = make_tex(ctx, dw, dh, 1, "f4")
depth_size = (dw, dh)
depth_tex.write(depth.tobytes())
fbw, fbh = glfw.get_framebuffer_size(win)
ctx.viewport = (0, 0, fbw, fbh)
ctx.clear(0.0, 0.0, 0.0)
if color_tex is not None:
color_tex.use(0)
depth_tex.use(1)
prog["strength"].value = state["strength"]
prog["convergence"].value = state["conv"]
prog["show_depth"].value = 1 if state["debug"] else 0
half = fbw // 2
sign = -1.0 if state["swap"] else 1.0
for i, eye in enumerate((-1.0 * sign, 1.0 * sign)):
ctx.viewport = (i * half, 0, half, fbh)
prog["eye"].value = eye
vao.render(moderngl.TRIANGLE_STRIP)
glfw.swap_buffers(win)
now = time.time()
if now - last_title > 1.0:
last_title = now
glfw.set_window_title(
win,
f"depth3d SBS | depth {worker.fps:.0f} fps | "
f"strength {state['strength']:.3f} | conv {state['conv']:.2f}",
)
finally:
worker.stop_evt.set()
grabber.stop()
glfw.terminate()
if __name__ == "__main__":
main()
depth3d_vr.py
+680
View File
@@ -0,0 +1,680 @@
"""
depth3d_vr.py - SteamVR 2D -> 3D player with an in-VR settings panel and immersive environments.
Captures a monitor (or window), estimates depth with Depth Anything V2, and shows it as a
curved screen in SteamVR with real per-eye depth. It is a normal SteamVR scene app, so it
streams to the Steam Frame like any VR game.
In VR: press MENU or A on a controller to open the settings panel, point with the laser,
trigger = click / drag. Settings are saved to settings.json (Save button, and
automatically on exit).
Hotkeys (hold Ctrl+Alt, work even when another window has focus):
Space open / close settings panel Enter click (fallback if trigger fails)
Up / Down depth strength PgUp/PgDn screen size
Left / Right convergence Home recenter
End swap eyes Insert depth-map debug view
Delete 3D on / off
"""
import argparse
import copy
import ctypes
import json
import math
import time
from pathlib import Path
import glfw
import moderngl
import numpy as np
import openvr
from depth3d import MODELS, DepthWorker, Grabber, list_windows, make_tex
from vr_env import LAYOUTS, EnvManager
from vr_shaders import (DILATE_FRAG, FLAT_FRAG, FLAT_VERT, FS_VERT, PANEL_FRAG, PANEL_VERT,
REFINE_FRAG, SCREEN_FRAG, SCREEN_VERT)
from vr_ui import PANEL_H_M, PANEL_W_M, PH, PW, Panel, ray_panel_hit
HERE = Path(__file__).resolve().parent
SETTINGS_PATH = HERE / "settings.json"
ENV_DIR = HERE / "environments"
SAVE_KEYS = ["depth_cm", "conv", "fov", "distance", "height", "sharpen", "stereo3d", "swap",
"env_name", "env_layouts", "env_brightness", "env_yaw", "env_play", "pointer_tilt"]
# Hotkey limits and adjustment speeds (units per second while held)
LIMITS = dict(depth_cm=(0.0, 30.0), conv=(0.0, 1.0), fov=(20.0, 180.0))
RATES = dict(depth_cm=5.0, conv=0.3, fov=35.0)
UI_DISTANCE = 1.4 # metres in front of you when the panel opens
# ---------------------------------------------------------------- math helpers
def m34(m):
a = np.eye(4, dtype=np.float64)
for r in range(3):
for c in range(4):
a[r, c] = m[r][c]
return a
def m44(m):
return np.array([[m[r][c] for c in range(4)] for r in range(4)], dtype=np.float64)
def rot_y(a):
c, s = math.cos(a), math.sin(a)
return np.array([[c, 0, s, 0], [0, 1, 0, 0], [-s, 0, c, 0], [0, 0, 0, 1]], dtype=np.float64)
def translate(x, y, z):
m = np.eye(4, dtype=np.float64)
m[:3, 3] = (x, y, z)
return m
def head_yaw(head):
"""Yaw (about +Y) that points -Z at where the head is looking, ignoring pitch/roll."""
fwd = -head[:3, 2]
return math.atan2(-fwd[0], -fwd[2])
def build_screen(aspect, dist, fov_deg, segs=96):
"""Curved (cylindrical) screen centred on the viewer. Returns (vertex array, width in m)."""
phi = math.radians(fov_deg)
width = phi * dist
height = width / aspect
v = []
for i in range(segs + 1):
t = i / segs
a = (t - 0.5) * phi
x, z = dist * math.sin(a), -dist * math.cos(a)
v += [x, height / 2, z, t, 0.0]
v += [x, -height / 2, z, t, 1.0]
return np.array(v, dtype="f4"), width
def pointer_ray(pose, tilt_deg):
t = math.radians(tilt_deg)
d = pose[:3, :3] @ np.array([0.0, math.sin(t), -math.cos(t)])
return pose[:3, 3].copy(), d / np.linalg.norm(d)
# ---------------------------------------------------------------- settings
def load_saved():
try:
return json.loads(SETTINGS_PATH.read_text())
except Exception: # noqa: BLE001
return {}
def save_settings(st):
SETTINGS_PATH.write_text(json.dumps({k: st[k] for k in SAVE_KEYS}, indent=2))
# ---------------------------------------------------------------- keyboard (global)
user32 = ctypes.windll.user32 if hasattr(ctypes, "windll") else None
VK = dict(CTRL=0x11, ALT=0x12, SPACE=0x20, PGUP=0x21, PGDN=0x22, END=0x23, HOME=0x24,
LEFT=0x25, UP=0x26, RIGHT=0x27, DOWN=0x28, INS=0x2D, DEL=0x2E, RETURN=0x0D)
def down(name):
return user32 is not None and bool(user32.GetAsyncKeyState(VK[name]) & 0x8000)
# ---------------------------------------------------------------- controllers
class Controllers:
"""Legacy controller polling: pose + trigger + buttons for every connected controller."""
def __init__(self, vrsys, debug):
self.vrsys = vrsys
self.debug = debug
self.devs = []
self.frame = 0
self.prev = {}
def poll(self, poses):
if self.frame % 45 == 0:
self.devs = [i for i in range(openvr.k_unMaxTrackedDeviceCount)
if self.vrsys.getTrackedDeviceClass(i) == openvr.TrackedDeviceClass_Controller]
self.frame += 1
out = []
for i in self.devs:
p = poses[i]
if not (p.bDeviceIsConnected and p.bPoseIsValid):
continue
ok, state = self.vrsys.getControllerState(i)
mask = int(state.ulButtonPressed) if ok else 0
trigger = bool(mask & (1 << openvr.k_EButton_SteamVR_Trigger)) or state.rAxis[1].x > 0.6
new = mask & ~self.prev.get(i, 0)
self.prev[i] = mask
if self.debug and new:
bits = [b for b in range(64) if new >> b & 1]
print(f"\n[input] controller {i} role {self.vrsys.getControllerRoleForTrackedDeviceIndex(i)}"
f" pressed button ids {bits}")
out.append(dict(index=i, role=self.vrsys.getControllerRoleForTrackedDeviceIndex(i),
pose=m34(p.mDeviceToAbsoluteTracking), trigger=trigger, mask=mask, new=new))
return out
# ---------------------------------------------------------------- the app
class App:
def __init__(self, args):
self.args = args
self.base = dict(depth_cm=args.depth_cm, conv=args.convergence, fov=args.fov,
distance=args.distance, height=-args.drop, sharpen=args.sharpen,
stereo3d=True, swap=False, debug=False, env_name="", env_layouts={},
env_brightness=1.0, env_yaw=0.0, env_play=True, pointer_tilt=args.pointer_tilt)
self.st = copy.deepcopy(self.base)
self.st.update({k: v for k, v in load_saved().items() if k in self.base})
self.grabber = Grabber(args.window, args.source_monitor, cursor=not args.no_cursor)
self.worker = DepthWorker(self.grabber, args.depth_width, args.smooth, MODELS[args.model],
args.cut_threshold)
self.worker.start()
self._init_gl()
self._init_vr()
self._init_render()
self.env = EnvManager(self.ctx, ENV_DIR)
self.env.paused = not self.st["env_play"]
if self.st["env_name"] in self.env.names():
self.env.select(self.st["env_name"])
else:
self.st["env_name"] = ""
self.panel = Panel(self.st, self.on_action)
self.ui_tex = self.ctx.texture((PW, PH), 4)
self.ui_tex.filter = (moderngl.LINEAR, moderngl.LINEAR)
self.ui_open = False
self.panel_model = np.eye(4)
self.panel_inv = np.eye(4)
self.rays = []
self.head = np.eye(4)
self.anchor_pos = np.zeros(3)
self.anchor_yaw = 0.0
self.need_recenter = True
self.prev_keys = {}
self.dirty = True
self.running = True
# ------------------------------------------------------------------ setup
def _init_gl(self):
if not glfw.init():
raise RuntimeError("GLFW init failed")
glfw.window_hint(glfw.VISIBLE, glfw.FALSE)
glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 3)
glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 3)
glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE)
self.win = glfw.create_window(64, 64, "depth3d_vr", None, None)
glfw.make_context_current(self.win)
glfw.swap_interval(0)
self.ctx = moderngl.create_context()
def _init_vr(self):
self.vrsys = openvr.init(openvr.VRApplication_Scene)
self.comp = openvr.VRCompositor()
try:
ipd = float(self.vrsys.getFloatTrackedDeviceProperty(
openvr.k_unTrackedDeviceIndex_Hmd, openvr.Prop_UserIpdMeters_Float))
except Exception: # noqa: BLE001
ipd = 0.063
self.ipd = ipd if 0.04 < ipd < 0.09 else 0.063
tw, th = self.vrsys.getRecommendedRenderTargetSize()
self.tw, self.th = int(tw * self.args.render_scale), int(th * self.args.render_scale)
print(f"Per-eye render target: {self.tw}x{self.th}")
self.targets = []
for _ in range(2):
tex = self.ctx.texture((self.tw, self.th), 4)
tex.filter = (moderngl.LINEAR, moderngl.LINEAR)
self.targets.append((tex, self.ctx.framebuffer(color_attachments=[tex])))
eyes = (openvr.Eye_Left, openvr.Eye_Right)
self.proj = [m44(self.vrsys.getProjectionMatrix(e, 0.1, 200.0)) for e in eyes]
self.inv_e2h = [np.linalg.inv(m34(self.vrsys.getEyeToHeadTransform(e))) for e in eyes]
self.controllers = Controllers(self.vrsys, self.args.debug_input)
def _init_render(self):
ctx = self.ctx
self.prog = ctx.program(vertex_shader=SCREEN_VERT, fragment_shader=SCREEN_FRAG)
self.prog["color_tex"].value = 0
self.prog["depth_tex"].value = 1
self.prog["samples"].value = max(4, self.args.samples)
verts, self.screen_w = build_screen(16 / 9, self.st["distance"], self.st["fov"])
self.vbo = ctx.buffer(verts.tobytes())
self.vao = ctx.vertex_array(self.prog, [(self.vbo, "3f 2f", "pos", "tc")])
self.mesh_key = None
# depth refinement passes
quad = ctx.buffer(np.array([-1, -1, 1, -1, -1, 1, 1, 1], dtype="f4").tobytes())
self.prog_ref = ctx.program(vertex_shader=FS_VERT, fragment_shader=REFINE_FRAG)
self.prog_dil = ctx.program(vertex_shader=FS_VERT, fragment_shader=DILATE_FRAG)
self.vao_ref = ctx.vertex_array(self.prog_ref, [(quad, "2f", "pos")])
self.vao_dil = ctx.vertex_array(self.prog_dil, [(quad, "2f", "pos")])
self.prog_ref["color_tex"].value = 0
self.prog_ref["depth_tex"].value = 1
self.prog_ref["sigma_c"].value = self.args.edge_sigma
self.prog_dil["src"].value = 2
self.prog_dil["radius"].value = self.args.dilate
self.S = dict(color_tex=None, color_size=None, depth_size=(2, 2), ref=None)
self.S["depth_tex"] = make_tex(ctx, 2, 2, 1, "f4", np.full((2, 2), 0.5, dtype="f4").tobytes())
self.S["final_depth"] = self.S["depth_tex"]
self.last_cseq = self.last_dseq = -1
# settings panel + laser pointers
self.prog_panel = ctx.program(vertex_shader=PANEL_VERT, fragment_shader=PANEL_FRAG)
self.prog_panel["ui_tex"].value = 4
w, h = PANEL_W_M / 2, PANEL_H_M / 2
pv = np.array([-w, h, 0, 0, 0, -w, -h, 0, 0, 1, w, h, 0, 1, 0, w, -h, 0, 1, 1], dtype="f4")
self.vao_panel = ctx.vertex_array(self.prog_panel,
[(ctx.buffer(pv.tobytes()), "3f 2f", "pos", "tc")])
self.prog_flat = ctx.program(vertex_shader=FLAT_VERT, fragment_shader=FLAT_FRAG)
self.dyn = ctx.buffer(reserve=2048, dynamic=True)
self.vao_flat = ctx.vertex_array(self.prog_flat, [(self.dyn, "3f", "pos")])
# ------------------------------------------------------------------ capture textures
def _make_refine_targets(self, cw, ch):
if self.S["ref"] is not None:
for obj in self.S["ref"]:
obj.release()
self.S["ref"] = None
if self.args.refine_width <= 0:
return
rw = min(cw, self.args.refine_width)
rh = max(1, int(round(ch * rw / cw)))
ta, tb = make_tex(self.ctx, rw, rh, 1, "f4"), make_tex(self.ctx, rw, rh, 1, "f4")
self.S["ref"] = (ta, self.ctx.framebuffer(color_attachments=[ta]),
tb, self.ctx.framebuffer(color_attachments=[tb]))
self.prog_dil["texel"].value = (1.0 / rw, 1.0 / rh)
def upload_color(self, frame):
S = self.S
fh, fw = frame.shape[:2]
if S["color_size"] != (fw, fh):
if S["color_tex"] is not None:
S["color_tex"].release()
t = make_tex(self.ctx, fw, fh, 4, "f1")
t.filter = (moderngl.LINEAR_MIPMAP_LINEAR, moderngl.LINEAR) # crisp text when minified
t.anisotropy = 16.0
t.swizzle = "BGRA"
S["color_tex"], S["color_size"] = t, (fw, fh)
self._make_refine_targets(fw, fh)
S["color_tex"].write(np.ascontiguousarray(frame))
S["color_tex"].build_mipmaps()
def upload_depth(self, depth):
S = self.S
dh, dw = depth.shape
if S["depth_size"] != (dw, dh):
S["depth_tex"].release()
S["depth_tex"] = make_tex(self.ctx, dw, dh, 1, "f4")
S["depth_size"] = (dw, dh)
self.prog_ref["depth_texel"].value = (1.0 / dw, 1.0 / dh)
S["depth_tex"].write(depth.tobytes())
def refine(self):
S = self.S
if S["ref"] is None:
return S["depth_tex"]
ta, fa, tb, fb = S["ref"]
S["color_tex"].use(0)
S["depth_tex"].use(1)
fa.use()
self.vao_ref.render(moderngl.TRIANGLE_STRIP)
if self.args.dilate > 0:
ta.use(2)
fb.use()
self.vao_dil.render(moderngl.TRIANGLE_STRIP)
return tb
return ta
# ------------------------------------------------------------------ actions / UI
def layout_for_env(self):
name = self.st["env_name"]
return self.st["env_layouts"].get(name) or self.env.guessed
def cycle_env(self, step):
names = self.env.names()
i = names.index(self.st["env_name"]) if self.st["env_name"] in names else 0
self.st["env_name"] = names[(i + step) % len(names)]
self.env.select(self.st["env_name"])
self.panel.invalidate()
def on_action(self, name):
if name == "save":
save_settings(self.st)
self.panel.toast("Saved")
elif name == "reset":
self.st.clear()
self.st.update(copy.deepcopy(self.base))
self.env.select("")
self.panel.toast("Reset to launcher defaults (Save to keep)")
self.panel.invalidate()
elif name == "close":
self.ui_open = False
elif name == "recenter":
self.need_recenter = True
elif name == "env_prev":
self.cycle_env(-1)
elif name == "env_next":
self.cycle_env(+1)
elif name == "env_layout":
cur = self.layout_for_env()
nxt = LAYOUTS[(LAYOUTS.index(cur) + 1) % len(LAYOUTS)] if cur in LAYOUTS else LAYOUTS[0]
self.st["env_layouts"][self.st["env_name"]] = nxt
self.panel.invalidate()
elif name == "toggled:env_play":
self.env.paused = not self.st["env_play"]
self.dirty = True
def toggle_ui(self):
self.ui_open = not self.ui_open
if self.ui_open:
yaw = head_yaw(self.head)
fwd = np.array([-math.sin(yaw), 0.0, -math.cos(yaw)])
pos = self.head[:3, 3] + fwd * UI_DISTANCE + np.array([0.0, -0.15, 0.0])
self.panel_model = translate(*pos) @ rot_y(yaw)
self.panel_inv = np.linalg.inv(self.panel_model)
self.panel.invalidate()
def update_ui(self, ctrls):
kb_click = down("CTRL") and down("ALT") and down("RETURN")
best = None
self.rays = []
for c in sorted(ctrls, key=lambda c: c["role"] != openvr.TrackedControllerRole_RightHand):
o, d = pointer_ray(c["pose"], self.st["pointer_tilt"])
trig = c["trigger"] or (kb_click and best is None)
hit = ray_panel_hit(o, d, self.panel_inv)
self.rays.append((o, d, hit[0] if hit else None, trig))
if hit and (best is None or (trig and not best[1])):
best = (hit, trig)
if best:
(_, px, py), trig = best
self.panel.pointer(px, py, trig)
else:
self.panel.pointer(None, None, False)
data = self.panel.render_if_dirty()
if data is not None:
self.ui_tex.write(data)
# ------------------------------------------------------------------ hotkeys
def handle_keys(self, dt):
st = self.st
chord = down("CTRL") and down("ALT")
if chord:
def adjust(name, sign):
lo, hi = LIMITS[name]
st[name] = min(hi, max(lo, st[name] + sign * RATES[name] * dt))
self.dirty = True
for key, name, sign in (("UP", "depth_cm", 1), ("DOWN", "depth_cm", -1),
("RIGHT", "conv", 1), ("LEFT", "conv", -1),
("PGUP", "fov", 1), ("PGDN", "fov", -1)):
if down(key):
adjust(name, sign)
if self.ui_open:
self.panel.invalidate()
for key, action in (("HOME", "recenter"), ("END", "swap"), ("INS", "debug"),
("SPACE", "ui"), ("DEL", "3d")):
cur = chord and down(key)
if cur and not self.prev_keys.get(key):
if action == "recenter":
self.need_recenter = True
elif action == "ui":
self.toggle_ui()
elif action == "3d":
st["stereo3d"] = not st["stereo3d"]
else:
st[action] = not st[action]
self.dirty = True
if self.ui_open:
self.panel.invalidate()
self.prev_keys[key] = cur
# ------------------------------------------------------------------ rendering
def draw_lasers(self, vp):
if not self.rays:
return
head_pos = self.head[:3, 3]
right, up = self.panel_model[:3, 0], self.panel_model[:3, 1]
normal = self.panel_model[:3, 2]
for o, d, t, trig in self.rays:
length = t if t is not None else 1.2
e = o + d * length
side = np.cross(d, head_pos - o)
n = np.linalg.norm(side)
if n < 1e-6:
continue
side = side / n * 0.003
quad = np.array([o + side, o - side, e + side, e - side], dtype="f4")
self.prog_flat["mvp"].write(vp.T.astype("f4").tobytes())
self.prog_flat["color"].value = (1.0, 0.85, 0.3, 0.9) if trig else (0.4, 0.75, 1.0, 0.75)
self.dyn.write(quad.tobytes())
self.vao_flat.render(moderngl.TRIANGLE_STRIP, vertices=4)
if t is not None:
c = e + normal * 0.004
ring = [c] + [c + (math.cos(a) * right + math.sin(a) * up) * 0.013
for a in np.linspace(0, 2 * math.pi, 15)]
self.prog_flat["color"].value = (1.0, 1.0, 1.0, 0.95)
self.dyn.write(np.array(ring, dtype="f4").tobytes())
self.vao_flat.render(moderngl.TRIANGLE_FAN, vertices=len(ring))
def render_eyes(self):
st, S, env = self.st, self.S, self.env
have_color = S["color_tex"] is not None
if have_color:
S["color_tex"].use(0)
S["final_depth"].use(1)
sign = -1.0 if st["swap"] else 1.0
strength = (0.5 * (st["depth_cm"] / 100.0) / self.screen_w) if st["stereo3d"] else 0.0
self.prog["strength"].value = strength
self.prog["convergence"].value = st["conv"]
self.prog["sharpen"].value = st["sharpen"]
self.prog["show_depth"].value = 1 if (st["debug"] and st["stereo3d"]) else 0
screen_model = (translate(self.anchor_pos[0], self.anchor_pos[1] + st["height"], self.anchor_pos[2])
@ rot_y(self.anchor_yaw))
view_base = np.linalg.inv(self.head)
head_rot_inv = np.linalg.inv(self._head_rot())
env_yaw = self.anchor_yaw + math.radians(st["env_yaw"])
env_layout = self.layout_for_env()
for i in range(2):
tex, fbo = self.targets[i]
fbo.use()
self.ctx.viewport = (0, 0, self.tw, self.th)
self.ctx.clear(0.0, 0.0, 0.0)
eye = -1.0 if i == 0 else 1.0
if env.ready:
env.draw(self.proj[i] @ self.inv_e2h[i] @ head_rot_inv, eye, env_layout,
env_yaw, st["env_brightness"])
vp = self.proj[i] @ self.inv_e2h[i] @ view_base
if have_color:
self.prog["mvp"].write((vp @ screen_model).T.astype("f4").tobytes())
self.prog["eye"].value = eye * sign
self.vao.render(moderngl.TRIANGLE_STRIP)
if self.ui_open:
self.ctx.enable(moderngl.BLEND)
self.ui_tex.use(4)
self.prog_panel["mvp"].write((vp @ self.panel_model).T.astype("f4").tobytes())
self.vao_panel.render(moderngl.TRIANGLE_STRIP)
self.draw_lasers(vp)
self.ctx.disable(moderngl.BLEND)
self.ctx.finish()
for i, eye in enumerate((openvr.Eye_Left, openvr.Eye_Right)):
t = openvr.Texture_t()
t.handle = int(self.targets[i][0].glo)
t.eType = openvr.TextureType_OpenGL
t.eColorSpace = openvr.ColorSpace_Gamma
self.comp.submit(eye, t)
def _head_rot(self):
r = self.head.copy()
r[:3, 3] = 0.0
return r
# ------------------------------------------------------------------ main loop
def run(self):
args, st, S = self.args, self.st, self.S
poses = (openvr.TrackedDevicePose_t * openvr.k_unMaxTrackedDeviceCount)()
event = openvr.VREvent_t()
last_t, last_print, mesh_key = time.time(), 0.0, None
print("\nRunning. Press MENU or A on a controller (or hold Ctrl+Alt and press Space) "
"to open the settings panel.")
while self.running:
self.comp.waitGetPoses(poses, None)
while self.vrsys.pollNextEvent(event):
if event.eventType == openvr.VREvent_Quit:
self.running = False
now = time.time()
dt = min(now - last_t, 0.1)
last_t = now
hp = poses[openvr.k_unTrackedDeviceIndex_Hmd]
if hp.bPoseIsValid:
self.head = m34(hp.mDeviceToAbsoluteTracking)
if self.need_recenter:
self.anchor_pos = self.head[:3, 3].copy()
self.anchor_yaw = head_yaw(self.head)
self.need_recenter = False
self.handle_keys(dt)
ctrls = self.controllers.poll(poses)
for c in ctrls:
if any(c["new"] >> b & 1 for b in (openvr.k_EButton_ApplicationMenu, openvr.k_EButton_A)):
self.toggle_ui()
break
if self.ui_open:
self.panel.set_info(env_name=st["env_name"], env_layout=self.layout_for_env(),
env_video=self.env.is_video,
env_status="Loading..." if self.env.loading else self.env.error,
ipd=self.ipd)
self.update_ui(ctrls)
self.worker.active = st["stereo3d"]
self.env.update()
# ---- upload newest color / depth
updated = False
if args.sync and st["stereo3d"]:
depth, frame, dseq = self.worker.latest_pair()
if depth is not None and frame is not None and dseq != self.last_dseq:
self.last_dseq = dseq
self.upload_color(frame)
self.upload_depth(depth)
updated = True
else:
frame, cseq = self.grabber.latest()
if frame is not None and cseq != self.last_cseq:
self.last_cseq = cseq
self.upload_color(frame)
updated = True
depth, dseq = self.worker.latest()
if st["stereo3d"] and depth is not None and dseq != self.last_dseq:
self.last_dseq = dseq
self.upload_depth(depth)
updated = True
if updated and S["color_tex"] is not None and st["stereo3d"]:
S["final_depth"] = self.refine()
# ---- screen mesh (rebuild if aspect / size / distance changed)
aspect = (S["color_size"][0] / S["color_size"][1]) if S["color_size"] else 16 / 9
key = (round(aspect, 4), round(st["fov"], 1), round(st["distance"], 2))
if key != mesh_key:
mesh_key = key
verts, self.screen_w = build_screen(aspect, st["distance"], st["fov"])
self.vbo.write(verts.tobytes())
self.render_eyes()
if self.dirty and now - last_print > 0.4:
last_print, self.dirty = now, False
far_cm = st["depth_cm"] * st["conv"]
warn = (" !! far parts exceed eye spacing - lower depth or convergence"
if far_cm / 100.0 > self.ipd and st["stereo3d"] else "")
print(f"\r3D {'on ' if st['stereo3d'] else 'off'} | depth {st['depth_cm']:.1f} cm | "
f"conv {st['conv']:.2f} | screen {st['fov']:.0f} deg | "
f"depth fps {self.worker.fps:.0f}{warn} ", end="", flush=True)
def shutdown(self):
print("\nShutting down...")
if not self.args.no_autosave:
try:
save_settings(self.st)
except Exception as e: # noqa: BLE001
print("Could not save settings:", e)
self.env.stop()
self.worker.stop_evt.set()
self.grabber.stop()
openvr.shutdown()
glfw.terminate()
def main():
ap = argparse.ArgumentParser(description="SteamVR 2D->3D depth player")
ap.add_argument("--window", help="Title (or part of it) of the window to capture")
ap.add_argument("--source-monitor", type=int, default=None,
help="Capture a whole monitor (1 = main monitor)")
ap.add_argument("--list", action="store_true", help="List window titles and exit")
ap.add_argument("--model", choices=list(MODELS), default="large")
ap.add_argument("--depth-width", type=int, default=700)
ap.add_argument("--smooth", type=float, default=0.5,
help="Depth temporal blend in calm shots (1 = off, lower = smoother)")
ap.add_argument("--cut-threshold", type=float, default=0.12,
help="Scene-cut sensitivity (lower = detects more cuts)")
ap.add_argument("--sync", action=argparse.BooleanOptionalAction, default=True,
help="Show each video frame together with its own depth map (best for movies)")
ap.add_argument("--samples", type=int, default=24,
help="Warp search samples per pixel (quality vs GPU cost)")
ap.add_argument("--refine-width", type=int, default=1920, help="Depth refinement resolution (0 = off)")
ap.add_argument("--edge-sigma", type=float, default=0.10, help="Edge-snap strictness")
ap.add_argument("--dilate", type=float, default=1.0, help="Grow foreground depth by N pixels (0 = off)")
ap.add_argument("--sharpen", type=float, default=0.5, help="Text sharpening 0..1 (initial value)")
ap.add_argument("--depth-cm", type=float, default=6.0, help="Total disparity range in cm (initial)")
ap.add_argument("--convergence", type=float, default=0.4, help="(initial)")
ap.add_argument("--distance", type=float, default=3.0, help="Screen distance in metres (initial)")
ap.add_argument("--fov", type=float, default=70.0, help="Screen width in degrees (initial)")
ap.add_argument("--drop", type=float, default=0.0, help="Lower the screen below eye level, metres")
ap.add_argument("--render-scale", type=float, default=1.0)
ap.add_argument("--pointer-tilt", type=float, default=0.0,
help="Tilt the controller laser up(+)/down(-) in degrees if it points off")
ap.add_argument("--no-cursor", action="store_true", help="Do not capture the mouse cursor")
ap.add_argument("--no-autosave", action="store_true", help="Do not save settings on exit")
ap.add_argument("--debug-input", action="store_true", help="Print controller button ids as you press them")
args = ap.parse_args()
if args.list:
print("Windows:")
for t in sorted(set(list_windows())):
print(" ", t)
if glfw.init():
print("Monitors (use the 'source' number with --source-monitor):")
for i, m in enumerate(glfw.get_monitors()):
vm = glfw.get_video_mode(m)
x, y = glfw.get_monitor_pos(m)
print(f" source={i + 1} {glfw.get_monitor_name(m).decode(errors='ignore')}"
f" {vm.size.width}x{vm.size.height} at ({x},{y})")
glfw.terminate()
return
if not args.window and args.source_monitor is None:
ap.error("give --window TITLE or --source-monitor N")
try:
app = App(args)
except Exception as e: # noqa: BLE001
print("\nCould not start:", e)
print("Start SteamVR and connect the Frame (Steam Link -> PC VR) first.")
raise
try:
app.run()
except KeyboardInterrupt:
pass
finally:
app.shutdown()
if __name__ == "__main__":
main()
pytest.ini
+2
View File
@@ -0,0 +1,2 @@
[pytest]
testpaths = tests
requirements-dev.txt
+8
View File
@@ -0,0 +1,8 @@
# Test / lint dependencies (no GPU, torch or Windows needed - GL tests use Mesa llvmpipe)
numpy
pillow
moderngl
scipy
openvr
pytest
pyflakes
requirements.txt
+10
View File
@@ -0,0 +1,10 @@
# RTX 50-series (Blackwell) needs a CUDA 12.8+ PyTorch build. Install it first:
# pip install torch --index-url https://download.pytorch.org/whl/cu128
transformers
numpy
glfw
moderngl
windows-capture
openvr
pillow
av
run_depth3d_vr.bat
+65
View File
@@ -0,0 +1,65 @@
@echo off
setlocal
cd /d "%~dp0"
REM ================= STARTING VALUES (edit these) =================
REM Settings you save from the in-VR menu (settings.json) override the 3D/screen values below.
REM To go back to these, press Reset + Save in the menu, or delete settings.json.
REM Monitor to convert to 3D (1 = main monitor)
set SOURCE_MONITOR=1
REM Depth model: small / base / large (drop to base if the game stutters)
set MODEL=large
REM Depth model input width in pixels. Higher = crisper depth edges, slower.
set DEPTH_WIDTH=700
REM Starting screen size (degrees), distance (metres), text sharpening (0..1)
set FOV=70
set DISTANCE=3.0
set SHARPEN=0.5
REM Starting 3D look: total depth range in cm, and which depth sits at the screen plane
set DEPTH_CM=6
set CONVERGENCE=0.4
REM --sync = each video frame is shown with its own depth map (best for MOVIES)
REM --no-sync = lowest latency (better for GAMES)
set SYNC_FLAG=--sync
REM Mouse cursor: leave empty to show it, or set to --no-cursor to hide it
set CURSOR_FLAG=
REM Warp quality: samples searched per pixel (24 default; 16 = faster, 32 = best edges)
set SAMPLES=24
REM Depth edge refinement. REFINE_WIDTH 0 = off. EDGE_SIGMA smaller = sharper. DILATE 0 = off.
set REFINE_WIDTH=1920
set EDGE_SIGMA=0.10
set DILATE=1
REM Scene-cut sensitivity. Lower = more cuts detected.
set CUT_THRESHOLD=0.12
REM Render resolution multiplier per eye (1.0 = SteamVR's recommended size)
set RENDER_SCALE=1.0
REM If the menu laser points too high or low, set e.g. 20 or -20 (degrees)
set POINTER_TILT=0
REM Set to --debug-input to print controller button ids (helps if the menu button doesn't respond)
set DEBUG_INPUT_FLAG=
REM =================================================================
echo Start SteamVR and connect the Steam Frame BEFORE running this.
echo Capturing monitor %SOURCE_MONITOR% with the "%MODEL%" model...
echo In VR: press MENU or A on a controller to open the settings panel.
echo Fallback: hold Ctrl+Alt and press Space. Put 360/180 images and videos in the "environments" folder.
echo.
py -3.13 depth3d_vr.py --source-monitor %SOURCE_MONITOR% --model %MODEL% --depth-width %DEPTH_WIDTH% --fov %FOV% --distance %DISTANCE% --sharpen %SHARPEN% --depth-cm %DEPTH_CM% --convergence %CONVERGENCE% %SYNC_FLAG% %CURSOR_FLAG% --samples %SAMPLES% --refine-width %REFINE_WIDTH% --edge-sigma %EDGE_SIGMA% --dilate %DILATE% --cut-threshold %CUT_THRESHOLD% --render-scale %RENDER_SCALE% --pointer-tilt %POINTER_TILT% %DEBUG_INPUT_FLAG%
echo.
echo depth3d_vr exited. If there was an error, it is shown above.
pause
run_sbs_window.bat
+35
View File
@@ -0,0 +1,35 @@
@echo off
setlocal
cd /d "%~dp0"
REM Simple side-by-side window for Desktop+/Bigscreen. The SteamVR player (run_depth3d_vr.bat) is recommended.
REM ================= SETTINGS (edit these) =================
REM Monitor to convert to 3D (1 = main monitor)
set SOURCE_MONITOR=1
REM Depth model: small / base / large
set MODEL=large
REM Depth model input width in pixels. Lower = faster. Try 518 or 392 if fps drops.
set DEPTH_WIDTH=700
REM Size of the side-by-side output window. Keep it at your source aspect ratio.
REM 2580x1080 = 21:9 in Half SBS (Desktop+ "HSBS" or Bigscreen), ~1290 px per eye
REM For a 16:9 monitor use 1920x1080 instead.
set WIN_W=2580
set WIN_H=1080
REM 3D look. Can also be changed live with the arrow keys.
set STRENGTH=0.025
set CONVERGENCE=0.4
REM =========================================================
echo Capturing monitor %SOURCE_MONITOR% with the "%MODEL%" model...
echo Drag the "depth3d SBS" window to ANOTHER monitor, or it will capture itself.
echo.
py -3.13 depth3d.py --source-monitor %SOURCE_MONITOR% --model %MODEL% --depth-width %DEPTH_WIDTH% --width %WIN_W% --height %WIN_H% --strength %STRENGTH% --convergence %CONVERGENCE%
echo.
echo depth3d (SBS window) exited. If there was an error, it is shown above.
pause
setup.bat
+23
View File
@@ -0,0 +1,23 @@
@echo off
REM Run this ONCE to install everything depth3d needs (Python 3.13).
cd /d "%~dp0"
echo Installing PyTorch (CUDA 12.8 build, required for RTX 50-series)...
py -3.13 -m pip install --upgrade pip
py -3.13 -m pip install torch --index-url https://download.pytorch.org/whl/cu128
if errorlevel 1 goto fail
echo Installing the rest...
py -3.13 -m pip install transformers numpy glfw moderngl windows-capture openvr pillow av
if errorlevel 1 goto fail
echo.
echo Done. You can now double-click run_depth3d.bat
pause
exit /b 0
:fail
echo.
echo Install failed. Check the error above.
pause
exit /b 1
vr_env.py
+247
View File
@@ -0,0 +1,247 @@
"""
Immersive environments for depth3d_vr: 360/180 degree images and videos,
mono or stereo (side-by-side / over-under), shown on a big sphere around you.
Drop files into the `environments` folder next to the script. Name hints
(optional, the in-VR menu can also cycle the layout):
"..._180_sbs.mp4" VR180 side-by-side "..._360_ou.jpg" 360 over-under
"..._360.jpg" 360 mono "..._tb", "_lr", "_3dh", "_3dv" also work
"""
import re
import threading
import time
from pathlib import Path
import moderngl
import numpy as np
from PIL import Image
from vr_shaders import ENV_FRAG, ENV_VERT
Image.MAX_IMAGE_PIXELS = None
IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"}
VIDEO_EXT = {".mp4", ".mkv", ".webm", ".mov", ".m4v", ".avi"}
LAYOUTS = ["360 mono", "360 OU", "360 SBS", "180 mono", "180 SBS", "180 OU"]
MAX_IMAGE_DIM = 8192
MAX_VIDEO_WIDTH = 4096
def scan(folder):
folder = Path(folder)
if not folder.is_dir():
return []
files = [p for p in folder.iterdir() if p.suffix.lower() in IMAGE_EXT | VIDEO_EXT]
return sorted(files, key=lambda p: p.name.lower())
def _tag(name, *words):
pat = r"(^|[^a-z0-9])(" + "|".join(words) + r")([^a-z0-9]|$)"
return re.search(pat, name) is not None
def guess_layout(name, w, h):
"""Best-effort layout from the file name, falling back to the aspect ratio."""
n = name.lower()
fov = 180 if "180" in n else (360 if "360" in n else None)
if _tag(n, "sbs", "lr", "3dh", "lr180", "sbs180"):
stereo = "SBS"
elif _tag(n, "ou", "tb", "3dv", "overunder", "over_under", "topbottom"):
stereo = "OU"
elif _tag(n, "mono"):
stereo = "mono"
else:
stereo = None
a = w / max(h, 1)
if fov is None:
fov = 360
if stereo is None:
if fov == 180:
stereo = "SBS" if a > 1.5 else "mono"
else:
stereo = "OU" if 0.75 < a < 1.35 else ("SBS" if a > 3.0 else "mono")
return f"{fov} {stereo}"
def parse_layout(layout):
fov180 = 1 if layout.startswith("180") else 0
stereo = 1 if "SBS" in layout else (2 if "OU" in layout else 0)
return fov180, stereo
def build_sphere(lat_n=48, lon_n=96):
verts = []
for i in range(lat_n + 1):
lat = np.pi * (i / lat_n - 0.5)
for j in range(lon_n + 1):
lon = 2 * np.pi * (j / lon_n - 0.5)
verts.append((np.cos(lat) * np.sin(lon), np.sin(lat), -np.cos(lat) * np.cos(lon)))
idx = []
for i in range(lat_n):
for j in range(lon_n):
a = i * (lon_n + 1) + j
b = a + lon_n + 1
idx += [a, b, a + 1, a + 1, b, b + 1]
return np.array(verts, "f4"), np.array(idx, "i4")
class EnvManager:
def __init__(self, ctx, folder):
self.ctx = ctx
self.folder = Path(folder)
self.folder.mkdir(exist_ok=True)
self.files = scan(self.folder)
self.prog = ctx.program(vertex_shader=ENV_VERT, fragment_shader=ENV_FRAG)
v, i = build_sphere()
self.vao = ctx.vertex_array(self.prog, [(ctx.buffer(v.tobytes()), "3f", "pos")],
index_buffer=ctx.buffer(i.tobytes()))
self.prog["env_tex"].value = 3
self.tex = None
self.size = None
self.name = ""
self.is_video = False
self.guessed = "360 mono"
self.loading = False
self.error = ""
self.paused = False
self._lock = threading.Lock()
self._pending = None
self._pending_info = None
self._gen = 0
# ---------------------------------------------------------------- selection
def names(self):
return [""] + [p.name for p in self.files]
def rescan(self):
self.files = scan(self.folder)
def select(self, name):
"""Start loading `name` ('' = no environment). Loading happens on a worker thread."""
self._gen += 1
gen = self._gen
self.error = ""
with self._lock:
self._pending = None
if not name:
self.name = ""
self.loading = False
self.is_video = False
self._release()
return
path = self.folder / name
if not path.exists():
self.name = ""
self.error = f"{name} not found"
return
self.name = name
self.loading = True
self.is_video = path.suffix.lower() in VIDEO_EXT
target = self._video_loop if self.is_video else self._load_image
threading.Thread(target=target, args=(path, gen), daemon=True).start()
def _release(self):
if self.tex is not None:
self.tex.release()
self.tex = None
self.size = None
# ---------------------------------------------------------------- worker threads
def _load_image(self, path, gen):
try:
img = Image.open(path)
img.load()
img = img.convert("RGB")
w, h = img.size
s = min(1.0, MAX_IMAGE_DIM / max(w, h))
if s < 1.0:
img = img.resize((max(1, int(w * s)), max(1, int(h * s))), Image.LANCZOS)
arr = np.ascontiguousarray(np.asarray(img))
with self._lock:
if gen == self._gen:
self._pending = arr
except Exception as e: # noqa: BLE001
if gen == self._gen:
self.error = f"Could not load: {e}"
self.loading = False
def _video_loop(self, path, gen):
try:
import av
except ImportError:
self.error = "Videos need the 'av' package (run setup.bat)"
self.loading = False
return
try:
while gen == self._gen:
container = av.open(str(path))
stream = container.streams.video[0]
stream.thread_type = "AUTO"
fps = float(stream.average_rate or 30)
t0, n = time.time(), 0
for frame in container.decode(stream):
if gen != self._gen:
break
while self.paused and gen == self._gen:
time.sleep(0.05)
t0 = time.time() - n / fps
w, h = frame.width, frame.height
s = min(1.0, MAX_VIDEO_WIDTH / w)
arr = frame.reformat(width=int(w * s) // 2 * 2, height=int(h * s) // 2 * 2,
format="rgb24").to_ndarray()
with self._lock:
if gen == self._gen:
self._pending = arr
n += 1
delay = t0 + n / fps - time.time()
if delay > 0:
time.sleep(delay)
elif delay < -1.0: # fell far behind: resync, don't spiral
t0, n = time.time(), 0
container.close()
except Exception as e: # noqa: BLE001
if gen == self._gen:
self.error = f"Video error: {e}"
self.loading = False
# ---------------------------------------------------------------- GL thread
def update(self):
"""Upload a newly decoded image / video frame (call once per VR frame)."""
with self._lock:
arr, self._pending = self._pending, None
if arr is None:
return
h, w = arr.shape[:2]
if self.size != (w, h):
self._release()
self.tex = self.ctx.texture((w, h), 3, dtype="f1")
self.tex.filter = (moderngl.LINEAR_MIPMAP_LINEAR, moderngl.LINEAR)
self.tex.repeat_x = True
self.tex.repeat_y = False
self.tex.anisotropy = 8.0
self.size = (w, h)
self.guessed = guess_layout(self.name, w, h)
self.tex.write(arr)
self.tex.build_mipmaps()
self.loading = False
@property
def ready(self):
return self.tex is not None and bool(self.name)
def draw(self, mvp, eye, layout, yaw, brightness):
fov180, stereo = parse_layout(layout)
self.tex.use(3)
self.prog["mvp"].write(mvp.T.astype("f4").tobytes())
self.prog["stereo"].value = stereo
self.prog["fov180"].value = fov180
self.prog["eye"].value = eye
self.prog["yaw"].value = yaw
self.prog["brightness"].value = brightness
self.vao.render()
def stop(self):
self._gen += 1
vr_shaders.py
+216
View File
@@ -0,0 +1,216 @@
"""GLSL shaders for depth3d_vr.py (kept separate so they can be tested on their own)."""
# ---------------------------------------------------------------- the 3D screen
SCREEN_VERT = """
#version 330
uniform mat4 mvp;
in vec3 pos;
in vec2 tc;
out vec2 uv;
void main() {
uv = tc;
gl_Position = mvp * vec4(pos, 1.0);
}
"""
# For every output pixel, SEARCH along the scan line for the source pixel that lands here
# (nearest one wins, like a z-buffer). Revealed gaps are filled from the far side.
# Mip-mapped + anisotropic sampling keeps text readable; optional sharpening on top.
SCREEN_FRAG = """
#version 330
uniform sampler2D color_tex;
uniform sampler2D depth_tex;
uniform float strength; // per-eye uv shift per unit of (depth - convergence)
uniform float convergence;
uniform float eye; // -1 left, +1 right
uniform float sharpen;
uniform int samples;
uniform int show_depth;
in vec2 uv;
out vec4 fragColor;
void main() {
vec2 gx = dFdx(uv); // footprint of one output pixel -> picks the right mip level
vec2 gy = dFdy(uv);
if (show_depth == 1) {
fragColor = vec4(vec3(textureLod(depth_tex, uv, 0.0).r), 1.0);
return;
}
float sxs = uv.x;
if (strength > 1e-7) {
float R = strength * max(convergence, 1.0 - convergence);
float stp = 2.0 * R / float(samples - 1);
float tol = 0.75 * stp;
float gapmax = strength;
float best_d = -1.0, fb_score = 1e9, fb_sx = uv.x;
for (int i = 0; i < samples; i++) {
float sx = uv.x - R + stp * float(i);
float d = textureLod(depth_tex, vec2(sx, uv.y), 0.0).r;
float r = abs(sx - eye * strength * (d - convergence) - uv.x);
if (r <= tol && d > best_d) best_d = d;
float score = r / gapmax + 2.0 * d; // in a gap: nearest FAR sample
if (r <= gapmax && score < fb_score) { fb_score = score; fb_sx = sx; }
}
sxs = (best_d >= 0.0) ? uv.x + eye * strength * (best_d - convergence) : fb_sx;
}
vec2 p = vec2(sxs, uv.y);
vec3 col = textureGrad(color_tex, p, gx, gy).rgb;
if (sharpen > 0.0) {
vec3 blur = 0.25 * (textureGrad(color_tex, p + gx, gx, gy).rgb +
textureGrad(color_tex, p - gx, gx, gy).rgb +
textureGrad(color_tex, p + gy, gx, gy).rgb +
textureGrad(color_tex, p - gy, gx, gy).rgb);
col = clamp(col + sharpen * (col - blur), 0.0, 1.0);
}
fragColor = vec4(col, 1.0);
}
"""
# ---------------------------------------------------------------- depth refinement passes
FS_VERT = """
#version 330
in vec2 pos;
out vec2 uv;
void main() {
uv = pos * 0.5 + 0.5;
gl_Position = vec4(pos, 0.0, 1.0);
}
"""
# Joint-bilateral upsampling: snaps soft low-res depth edges onto the real colour edges.
REFINE_FRAG = """
#version 330
uniform sampler2D color_tex;
uniform sampler2D depth_tex;
uniform vec2 depth_texel; // 1 / raw depth size
uniform float sigma_c; // colour similarity (smaller = harder edges)
in vec2 uv;
out vec4 fragColor;
void main() {
vec3 c0 = texture(color_tex, uv).rgb;
float acc = 0.0, wsum = 0.0;
for (int j = -2; j <= 2; j++) {
for (int i = -2; i <= 2; i++) {
vec2 p = uv + vec2(float(i), float(j)) * depth_texel;
vec3 dc = texture(color_tex, p).rgb - c0;
float ws = exp(-float(i * i + j * j) / 4.5);
float wc = exp(-dot(dc, dc) / (2.0 * sigma_c * sigma_c));
float w = ws * wc;
acc += w * texture(depth_tex, p).r;
wsum += w;
}
}
fragColor = vec4(acc / max(wsum, 1e-5), 0.0, 0.0, 1.0);
}
"""
# Tiny foreground dilation so the nearest depth fully covers an object's rim.
DILATE_FRAG = """
#version 330
uniform sampler2D src;
uniform vec2 texel;
uniform float radius; // in pixels of the refined map
in vec2 uv;
out vec4 fragColor;
void main() {
float m = texture(src, uv).r;
for (int j = -1; j <= 1; j++) {
for (int i = -1; i <= 1; i++) {
m = max(m, texture(src, uv + vec2(float(i), float(j)) * texel * radius).r);
}
}
fragColor = vec4(m, 0.0, 0.0, 1.0);
}
"""
# ---------------------------------------------------------------- immersive environment
ENV_VERT = """
#version 330
uniform mat4 mvp;
in vec3 pos;
out vec3 dir;
void main() {
dir = pos;
gl_Position = mvp * vec4(pos * 80.0, 1.0);
}
"""
# Equirectangular 360 / 180 viewer, mono or stereo (side-by-side / over-under).
ENV_FRAG = """
#version 330
uniform sampler2D env_tex;
uniform int stereo; // 0 mono, 1 side-by-side, 2 over-under
uniform int fov180; // 0 = 360, 1 = 180 (front half only)
uniform float eye; // -1 left, +1 right
uniform float yaw; // radians: rotation of the environment about the vertical axis
uniform float brightness;
in vec3 dir;
out vec4 fragColor;
const float PI = 3.14159265358979;
void main() {
vec3 d = normalize(dir);
float c = cos(yaw), s = sin(yaw);
vec3 e = vec3(d.x * c - d.z * s, d.y, d.x * s + d.z * c); // world -> environment frame
float lon = atan(e.x, -e.z);
float lat = asin(clamp(e.y, -1.0, 1.0));
float v = 0.5 - lat / PI;
vec2 uvA = vec2(lon / (2.0 * PI) + 0.5, v);
vec2 uvB = vec2(fract(lon / (2.0 * PI)), v);
vec2 gx = dFdx(uvA), gy = dFdy(uvA);
vec2 gxb = dFdx(uvB), gyb = dFdy(uvB);
if (abs(gxb.x) + abs(gyb.x) < abs(gx.x) + abs(gy.x)) { gx = gxb; gy = gyb; } // hide the wrap seam
vec2 uv = uvA;
float inside = 1.0;
if (fov180 == 1) {
uv.x = lon / PI + 0.5;
gx.x *= 2.0; gy.x *= 2.0;
inside = 1.0 - smoothstep(0.5 * PI - 0.03, 0.5 * PI, abs(lon));
}
vec2 scale = vec2(1.0), off = vec2(0.0);
float right = step(0.0, eye);
if (stereo == 1) { scale.x = 0.5; off.x = 0.5 * right; }
if (stereo == 2) { scale.y = 0.5; off.y = 0.5 * right; }
vec3 col = textureGrad(env_tex, uv * scale + off, gx * scale, gy * scale).rgb;
fragColor = vec4(col * brightness * inside, 1.0);
}
"""
# ---------------------------------------------------------------- settings panel + laser pointers
PANEL_VERT = """
#version 330
uniform mat4 mvp;
in vec3 pos;
in vec2 tc;
out vec2 uv;
void main() {
uv = tc;
gl_Position = mvp * vec4(pos, 1.0);
}
"""
PANEL_FRAG = """
#version 330
uniform sampler2D ui_tex;
in vec2 uv;
out vec4 fragColor;
void main() {
fragColor = texture(ui_tex, uv);
}
"""
FLAT_VERT = """
#version 330
uniform mat4 mvp;
in vec3 pos;
void main() {
gl_Position = mvp * vec4(pos, 1.0);
}
"""
FLAT_FRAG = """
#version 330
uniform vec4 color;
out vec4 fragColor;
void main() {
fragColor = color;
}
"""
vr_ui.py
+267
View File
@@ -0,0 +1,267 @@
"""
Floating settings panel for depth3d_vr. Drawn with Pillow into a texture and operated
with VR controller laser pointers (trigger = click / drag).
"""
import time
import numpy as np
from PIL import Image, ImageDraw, ImageFont
PW, PH = 1280, 800 # panel texture size in pixels
PANEL_W_M = 1.6 # physical size in the VR world (metres)
PANEL_H_M = PANEL_W_M * PH / PW
COL_BG = (16, 18, 26, 240)
COL_TEXT = (235, 238, 245, 255)
COL_DIM = (150, 156, 172, 255)
COL_ACCENT = (64, 156, 255, 255)
COL_WIDGET = (48, 52, 68, 255)
COL_HOVER = (72, 78, 100, 255)
COL_WARN = (255, 190, 80, 255)
TABS = ["3D", "Screen", "Environment"]
_fonts = {}
def font(size, bold=False):
key = (size, bold)
if key in _fonts:
return _fonts[key]
names = (["segoeuib.ttf", "arialbd.ttf", "DejaVuSans-Bold.ttf"] if bold
else ["segoeui.ttf", "arial.ttf", "DejaVuSans.ttf"])
f = None
for n in names:
try:
f = ImageFont.truetype(n, size)
break
except OSError:
continue
if f is None:
f = ImageFont.load_default(size)
_fonts[key] = f
return f
def ray_panel_hit(origin, direction, inv_model):
"""Intersect a world-space ray with the panel. Returns (t, px, py) or None."""
o = inv_model @ np.append(origin, 1.0)
d = inv_model[:3, :3] @ direction
if abs(d[2]) < 1e-6:
return None
t = -o[2] / d[2]
if t <= 0:
return None
x, y = o[0] + t * d[0], o[1] + t * d[1]
if abs(x) > PANEL_W_M / 2 or abs(y) > PANEL_H_M / 2:
return None
return t, (x / PANEL_W_M + 0.5) * PW, (0.5 - y / PANEL_H_M) * PH
class Panel:
def __init__(self, st, on_action):
self.st = st
self.on_action = on_action
self.tab = 0
self.info = dict(env_name="", env_layout="", env_video=False, env_status="", ipd=0.063)
self.widgets = []
self.hover = None
self.active = None
self._down = False
self.dirty = True
self._toast = ("", 0.0)
self._layout()
# ------------------------------------------------------------------ state
def set_info(self, **kw):
if any(self.info.get(k) != v for k, v in kw.items()):
self.info.update(kw)
self.dirty = True
self._layout()
def toast(self, text, seconds=2.5):
self._toast = (text, time.time() + seconds)
self.dirty = True
def invalidate(self):
self.dirty = True
self._layout()
# ------------------------------------------------------------------ layout
def _layout(self):
W = []
for i, name in enumerate(TABS):
W.append(dict(t="tab", idx=i, label=name, rect=(60 + i * 230, 92, 60 + i * 230 + 210, 148)))
y = 190
row = 84
def slider(key, label, lo, hi, fmt):
nonlocal y
W.append(dict(t="slider", key=key, label=label, lo=lo, hi=hi, fmt=fmt,
rect=(500, y - 28, 1010, y + 28), track=(520, 990), y=y))
y += row
def toggle(key, label):
nonlocal y
W.append(dict(t="toggle", key=key, label=label, rect=(500, y - 28, 640, y + 28), y=y))
y += row
def button(action, label, x0=60, x1=480, big=False):
W.append(dict(t="button", action=action, label=label, rect=(x0, y - 30, x1, y + 30), y=y))
return y
if self.tab == 0:
slider("depth_cm", "3D depth", 0.0, 30.0, "{:.1f} cm")
slider("conv", "Convergence", 0.0, 1.0, "{:.2f}")
toggle("stereo3d", "3D enabled")
toggle("swap", "Swap eyes")
toggle("debug", "Show depth map")
elif self.tab == 1:
slider("fov", "Screen size", 20.0, 180.0, "{:.0f}\u00b0")
slider("distance", "Distance", 1.0, 10.0, "{:.1f} m")
slider("height", "Height", -1.5, 1.5, "{:+.2f} m")
slider("sharpen", "Text sharpening", 0.0, 1.0, "{:.2f}")
button("recenter", "Recenter screen", 500, 900)
y += row
else:
W.append(dict(t="button", action="env_prev", label="\u25c0", rect=(60, y - 30, 160, y + 30), y=y))
W.append(dict(t="button", action="env_next", label="\u25b6", rect=(1120, y - 30, 1220, y + 30), y=y))
y += row
W.append(dict(t="button", action="env_layout", label="Layout: " + (self.info["env_layout"] or "-"),
rect=(60, y - 30, 600, y + 30), y=y))
y += row
slider("env_brightness", "Brightness", 0.0, 1.5, "{:.2f}")
slider("env_yaw", "Rotation", -180.0, 180.0, "{:+.0f}\u00b0")
if self.info["env_video"]:
toggle("env_play", "Play video")
fy = 742
for i, (label, action) in enumerate((("Save", "save"), ("Reset", "reset"), ("Close", "close"))):
W.append(dict(t="button", action=action, label=label,
rect=(60 + i * 240, fy - 32, 60 + i * 240 + 210, fy + 32), y=fy))
self.widgets = W
self.dirty = True
# ------------------------------------------------------------------ interaction
def _widget_at(self, px, py):
for w in self.widgets:
x0, y0, x1, y1 = w["rect"]
if x0 <= px <= x1 and y0 <= py <= y1:
return w
return None
def _set_slider(self, w, px):
t0, t1 = w["track"]
t = min(1.0, max(0.0, (px - t0) / (t1 - t0)))
self.st[w["key"]] = w["lo"] + t * (w["hi"] - w["lo"])
self.dirty = True
def pointer(self, px, py, down):
"""Feed the pointer position in panel pixels (None = not pointing at the panel)."""
press = down and not self._down
self._down = down
if px is None:
if self.hover is not None:
self.hover = None
self.dirty = True
if not down:
self.active = None
return
w = self._widget_at(px, py)
if w is not self.hover:
self.hover = w
self.dirty = True
if not down:
self.active = None
return
if self.active is not None:
self._set_slider(self.active, px)
return
if not press or w is None:
return
if w["t"] == "slider":
self.active = w
self._set_slider(w, px)
elif w["t"] == "toggle":
self.st[w["key"]] = not self.st[w["key"]]
self.dirty = True
self.on_action("toggled:" + w["key"])
elif w["t"] == "button":
self.on_action(w["action"])
elif w["t"] == "tab":
self.tab = w["idx"]
self.hover = None
self._layout()
# ------------------------------------------------------------------ drawing
def render_if_dirty(self):
if self._toast[0] and time.time() > self._toast[1]:
self._toast = ("", 0.0)
self.dirty = True
if not self.dirty:
return None
self.dirty = False
return self.render()
def render(self):
img = Image.new("RGBA", (PW, PH), (0, 0, 0, 0))
d = ImageDraw.Draw(img)
d.rounded_rectangle((0, 0, PW - 1, PH - 1), radius=40, fill=COL_BG)
d.text((60, 46), "Depth 3D", font=font(44, True), fill=COL_TEXT, anchor="lm")
d.text((PW - 60, 46), "trigger: click / drag", font=font(24), fill=COL_DIM, anchor="rm")
f_label, f_val = font(32), font(32, True)
for w in self.widgets:
hov = w is self.hover or w is self.active
x0, y0, x1, y1 = w["rect"]
if w["t"] == "tab":
sel = w["idx"] == self.tab
d.rounded_rectangle(w["rect"], radius=18,
fill=COL_ACCENT if sel else (COL_HOVER if hov else COL_WIDGET))
d.text(((x0 + x1) / 2, (y0 + y1) / 2), w["label"], font=font(30, True),
fill=COL_TEXT, anchor="mm")
elif w["t"] == "slider":
d.text((60, w["y"]), w["label"], font=f_label, fill=COL_TEXT, anchor="lm")
t0, t1 = w["track"]
val = float(self.st[w["key"]])
frac = min(1.0, max(0.0, (val - w["lo"]) / (w["hi"] - w["lo"])))
kx = t0 + frac * (t1 - t0)
d.rounded_rectangle((t0, w["y"] - 6, t1, w["y"] + 6), radius=6, fill=COL_WIDGET)
d.rounded_rectangle((t0, w["y"] - 6, kx, w["y"] + 6), radius=6, fill=COL_ACCENT)
d.ellipse((kx - 20, w["y"] - 20, kx + 20, w["y"] + 20),
fill=(255, 255, 255, 255) if hov else COL_TEXT)
d.text((1040, w["y"]), w["fmt"].format(val), font=f_val, fill=COL_TEXT, anchor="lm")
elif w["t"] == "toggle":
d.text((60, w["y"]), w["label"], font=f_label, fill=COL_TEXT, anchor="lm")
on = bool(self.st[w["key"]])
d.rounded_rectangle(w["rect"], radius=28,
fill=COL_ACCENT if on else (COL_HOVER if hov else COL_WIDGET))
kx = x1 - 28 if on else x0 + 28
d.ellipse((kx - 22, w["y"] - 22, kx + 22, w["y"] + 22), fill=COL_TEXT)
elif w["t"] == "button":
d.rounded_rectangle(w["rect"], radius=20, fill=COL_HOVER if hov else COL_WIDGET)
d.text(((x0 + x1) / 2, (y0 + y1) / 2), w["label"], font=font(30, True),
fill=COL_TEXT, anchor="mm")
if self.tab == 0:
far_cm = self.st["depth_cm"] * self.st["conv"]
if far_cm / 100.0 > self.info["ipd"]:
d.text((60, 640), "Far parts exceed your eye spacing - lower depth or convergence.",
font=font(26), fill=COL_WARN, anchor="lm")
else:
d.text((60, 640), "Tip: keep depth x convergence under about 6 for comfort.",
font=font(26), fill=COL_DIM, anchor="lm")
elif self.tab == 2:
name = self.info["env_name"] or "None (black void)"
d.text((640, 190), name, font=font(32, True), fill=COL_TEXT, anchor="mm")
status = self.info["env_status"]
if status:
d.text((60, 640), status, font=font(26), fill=COL_WARN, anchor="lm")
else:
d.text((60, 640), "Put 360/180 images or videos in the 'environments' folder.",
font=font(26), fill=COL_DIM, anchor="lm")
if self._toast[0]:
d.text((PW - 60, 742), self._toast[0], font=font(30, True), fill=COL_ACCENT, anchor="rm")
return img.tobytes()