From 40c11fdb2f526fc7002081e1542e2c9ee8f62fa4 Mon Sep 17 00:00:00 2001 From: its-ven <134007790+its-ven@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:06:14 +0200 Subject: [PATCH] Add files via upload --- depth3d.py | 447 ++++++++++++++++++++++++++++ depth3d_vr.py | 680 +++++++++++++++++++++++++++++++++++++++++++ pytest.ini | 2 + requirements-dev.txt | 8 + requirements.txt | 10 + run_depth3d_vr.bat | 65 +++++ run_sbs_window.bat | 35 +++ setup.bat | 23 ++ vr_env.py | 247 ++++++++++++++++ vr_shaders.py | 216 ++++++++++++++ vr_ui.py | 267 +++++++++++++++++ 11 files changed, 2000 insertions(+) create mode 100644 depth3d.py create mode 100644 depth3d_vr.py create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 run_depth3d_vr.bat create mode 100644 run_sbs_window.bat create mode 100644 setup.bat create mode 100644 vr_env.py create mode 100644 vr_shaders.py create mode 100644 vr_ui.py diff --git a/depth3d.py b/depth3d.py new file mode 100644 index 0000000..1492879 --- /dev/null +++ b/depth3d.py @@ -0,0 +1,447 @@ +""" +depth3d.py - screen capture + depth estimation core, and a simple side-by-side (SBS) window. + +Two roles: + * Library: `Grabber` (Windows Graphics Capture) and `DepthWorker` (Depth Anything V2 on the + GPU, scene-cut aware) are used by depth3d_vr.py, the main SteamVR player. + * Standalone: `python depth3d.py --source-monitor 1` opens a plain SBS window that you can + feed to a 3D-capable viewer (Desktop+ HSBS, Bigscreen...). Simpler and less capable than + the VR player: older warp shader, no menu, no environments. + +Controls of the standalone SBS window: + Up / Down depth strength + Left / Right convergence (which depth sits at the screen plane) + X swap eyes + D toggle depth-map debug view + F toggle fullscreen + Esc quit +""" +import argparse +import threading +import time + +import glfw +import moderngl +import numpy as np +import torch +import torch.nn.functional as F +from transformers import AutoModelForDepthEstimation +from windows_capture import Frame, InternalCaptureControl, WindowsCapture + +MODELS = { + "small": "depth-anything/Depth-Anything-V2-Small-hf", + "base": "depth-anything/Depth-Anything-V2-Base-hf", + "large": "depth-anything/Depth-Anything-V2-Large-hf", +} + + +# -------------------------------------------------------------------------- +# Window listing helper +# -------------------------------------------------------------------------- +def list_windows(): + import ctypes + from ctypes import wintypes + + user32 = ctypes.windll.user32 + titles = [] + + @ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM) + def cb(hwnd, _): + if user32.IsWindowVisible(hwnd): + n = user32.GetWindowTextLengthW(hwnd) + if n: + buf = ctypes.create_unicode_buffer(n + 1) + user32.GetWindowTextW(hwnd, buf, n + 1) + titles.append(buf.value) + return True + + user32.EnumWindows(cb, 0) + return titles + + +# -------------------------------------------------------------------------- +# Capture +# -------------------------------------------------------------------------- +class Grabber: + """Keeps only the newest captured frame (BGRA, HxWx4 uint8).""" + + def __init__(self, window_name=None, monitor_index=None, cursor=True): + self.lock = threading.Lock() + self.frame = None + self.seq = 0 + kwargs = dict(cursor_capture=cursor, draw_border=False) + if window_name: + kwargs["window_name"] = window_name + else: + kwargs["monitor_index"] = monitor_index # 1-based + cap = WindowsCapture(**kwargs) + + @cap.event + def on_frame_arrived(frame: Frame, ctrl: InternalCaptureControl): + buf = np.array(frame.frame_buffer, copy=True) + with self.lock: + self.frame = buf + self.seq += 1 + + @cap.event + def on_closed(): + print("Capture session closed.") + + self.control = cap.start_free_threaded() + + def latest(self): + with self.lock: + return self.frame, self.seq + + def stop(self): + try: + self.control.stop() + except Exception: + pass + + +# -------------------------------------------------------------------------- +# Depth estimation worker +# -------------------------------------------------------------------------- +class DepthWorker(threading.Thread): + def __init__(self, grabber, width, smooth, model_id, cut_threshold=0.12): + super().__init__(daemon=True) + self.grabber = grabber + self.width = max(112, (width // 14) * 14) + self.smooth = smooth + self.cut_threshold = cut_threshold + self.active = True # set False to idle the model (e.g. when 3D is switched off) + self.frame = None # the captured frame the current depth belongs to + self.stop_evt = threading.Event() + self.lock = threading.Lock() + self.depth = None # float32 HxW in 0..1 (1 = near) + self.seq = 0 + self.fps = 0.0 + + self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.dtype = torch.float16 if self.device == "cuda" else torch.float32 + if self.device == "cpu": + print("WARNING: CUDA not available, running on CPU (will be slow).") + print(f"Loading {model_id} on {self.device} ...") + self.model = ( + AutoModelForDepthEstimation.from_pretrained(model_id) + .to(self.device, self.dtype) + .eval() + ) + self.mean = torch.tensor([0.485, 0.456, 0.406], device=self.device).view(1, 3, 1, 1) + self.std = torch.tensor([0.229, 0.224, 0.225], device=self.device).view(1, 3, 1, 1) + + def latest(self): + with self.lock: + return self.depth, self.seq + + def latest_pair(self): + """Depth plus the exact captured frame it was computed from (keeps them in sync).""" + with self.lock: + return self.depth, self.frame, self.seq + + def run(self): + last_seq = -1 + lo = hi = None + prev = None + prev_thumb = None + n, t0 = 0, time.time() + while not self.stop_evt.is_set(): + if not self.active: + time.sleep(0.02) + continue + frame, seq = self.grabber.latest() + if frame is None or seq == last_seq: + time.sleep(0.002) + continue + last_seq = seq + + with torch.inference_mode(): + h, w = frame.shape[:2] + mw = self.width + mh = max(14, int(round(mw * h / w / 14)) * 14) + + t = torch.from_numpy(frame).to(self.device, non_blocking=True) + t = t[..., [2, 1, 0]].permute(2, 0, 1).unsqueeze(0).float() / 255.0 + t = F.interpolate(t, size=(mh, mw), mode="bilinear", + antialias=True, align_corners=False) + + # Tiny luminance thumbnail -> scene-cut / motion detection + thumb = F.adaptive_avg_pool2d(t, (18, 32)).mean(1) + if prev_thumb is None: + diff, cut = 1.0, True + else: + diff = (thumb - prev_thumb).abs().mean().item() + cut = diff > self.cut_threshold + prev_thumb = thumb + if cut: # hard cut: forget everything from the old shot + lo = hi = None + prev = None + + t = ((t - self.mean) / self.std).to(self.dtype) + + d = self.model(pixel_values=t).predicted_depth[0].float() # HxW, larger = nearer + + # Robust, temporally smoothed normalisation (avoids pumping/flicker) + sample = d.flatten()[::7] + q = torch.quantile(sample, torch.tensor([0.02, 0.98], device=self.device)) + l, hgh = q[0].item(), q[1].item() + if lo is None: + lo, hi = l, hgh + else: + lo += 0.08 * (l - lo) + hi += 0.08 * (hgh - hi) + dn = ((d - lo) / max(hi - lo, 1e-6)).clamp(0, 1) + + if prev is not None and prev.shape == dn.shape: + # calm shot -> smooth heavily; lots of motion -> trust the new frame + alpha = self.smooth + (1 - self.smooth) * min(1.0, diff / 0.06) + dn = prev * (1 - alpha) + dn * alpha + prev = dn + out = dn.cpu().numpy().astype(np.float32) + + with self.lock: + self.depth = out + self.frame = frame + self.seq += 1 + + n += 1 + if time.time() - t0 >= 2.0: + self.fps = n / (time.time() - t0) + n, t0 = 0, time.time() + + +# -------------------------------------------------------------------------- +# Rendering +# -------------------------------------------------------------------------- +VERT = """ +#version 330 +in vec2 pos; +out vec2 uv; +void main() { + uv = vec2(pos.x * 0.5 + 0.5, 1.0 - (pos.y * 0.5 + 0.5)); + gl_Position = vec4(pos, 0.0, 1.0); +} +""" + +FRAG = """ +#version 330 +uniform sampler2D color_tex; +uniform sampler2D depth_tex; +uniform float strength; // max horizontal shift per eye, as fraction of width +uniform float convergence; // depth value (0..1) that sits at the screen plane +uniform float eye; // -1 = left, +1 = right +uniform int steps; +uniform int show_depth; +in vec2 uv; +out vec4 fragColor; +void main() { + // Backward warp via fixed-point iteration: find the source pixel whose + // own depth-based shift lands on this output pixel. + vec2 src = uv; + for (int i = 0; i < steps; i++) { + float d = texture(depth_tex, src).r; + src.x = uv.x + eye * strength * (d - convergence); + } + if (show_depth == 1) { + fragColor = vec4(vec3(texture(depth_tex, uv).r), 1.0); + } else { + fragColor = vec4(texture(color_tex, src).rgb, 1.0); + } +} +""" + + +def make_tex(ctx, w, h, comps, dtype, data=None): + tex = ctx.texture((w, h), comps, data, dtype=dtype) + tex.filter = (moderngl.LINEAR, moderngl.LINEAR) + tex.repeat_x = False + tex.repeat_y = False + return tex + + +def main(): + ap = argparse.ArgumentParser(description="Window -> depth -> SBS 3D") + ap.add_argument("--window", help="Title (or part of it) of the window to capture") + ap.add_argument("--source-monitor", type=int, default=None, + help="Capture a whole monitor instead of a window (1 = first monitor)") + ap.add_argument("--list", action="store_true", + help="List visible window titles and monitors, then exit") + ap.add_argument("--model", choices=list(MODELS), default="base", + help="Depth model size (small = fastest, large = best quality)") + ap.add_argument("--depth-width", type=int, default=700, + help="Model input width (lower = faster). Try 518 or 392.") + ap.add_argument("--strength", type=float, default=0.025) + ap.add_argument("--convergence", type=float, default=0.4) + ap.add_argument("--steps", type=int, default=4, help="Warp iterations (quality/cost)") + ap.add_argument("--smooth", type=float, default=0.5, + help="Depth temporal blend, 1 = no smoothing, lower = smoother") + ap.add_argument("--out-monitor", type=int, default=None, + help="Start the SBS output fullscreen on this monitor (0-based). " + "Must NOT be the monitor you are capturing.") + ap.add_argument("--width", type=int, default=1600) + ap.add_argument("--height", type=int, default=900) + args = ap.parse_args() + + if args.list: + print("Windows:") + for t in sorted(set(list_windows())): + print(" ", t) + if glfw.init(): + print("Monitors (--source-monitor is 1-based, --out-monitor is 0-based):") + for i, m in enumerate(glfw.get_monitors()): + vm = glfw.get_video_mode(m) + x, y = glfw.get_monitor_pos(m) + print(f" out={i} source={i + 1} {glfw.get_monitor_name(m).decode(errors='ignore')}" + f" {vm.size.width}x{vm.size.height} at ({x},{y})") + glfw.terminate() + return + if not args.window and args.source_monitor is None: + ap.error("give --window TITLE or --source-monitor N (use --list to see options)") + if args.window and args.source_monitor is not None: + ap.error("use either --window or --source-monitor, not both") + if args.source_monitor is not None and args.out_monitor is not None \ + and args.source_monitor - 1 == args.out_monitor: + ap.error("output monitor and captured monitor are the same: that would feed back on itself") + + grabber = Grabber(args.window, args.source_monitor) + worker = DepthWorker(grabber, args.depth_width, args.smooth, MODELS[args.model]) + worker.start() + + if not glfw.init(): + raise RuntimeError("GLFW init failed") + glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 3) + glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 3) + glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) + + monitors = glfw.get_monitors() + mon = None + w, h = args.width, args.height + if args.out_monitor is not None and args.out_monitor < len(monitors): + mon = monitors[args.out_monitor] + mode = glfw.get_video_mode(mon) + w, h = mode.size.width, mode.size.height + win = glfw.create_window(w, h, "depth3d SBS", mon, None) + glfw.make_context_current(win) + glfw.swap_interval(1) + + ctx = moderngl.create_context() + prog = ctx.program(vertex_shader=VERT, fragment_shader=FRAG) + vbo = ctx.buffer(np.array([-1, -1, 1, -1, -1, 1, 1, 1], dtype="f4").tobytes()) + vao = ctx.vertex_array(prog, [(vbo, "2f", "pos")]) + + state = { + "strength": args.strength, + "conv": args.convergence, + "swap": False, + "debug": False, + "fullscreen": mon is not None, + "windowed_geom": (100, 100, args.width, args.height), + } + + def on_key(window, key, scancode, action, mods): + if action not in (glfw.PRESS, glfw.REPEAT): + return + if key == glfw.KEY_ESCAPE: + glfw.set_window_should_close(window, True) + elif key == glfw.KEY_UP: + state["strength"] = min(0.1, state["strength"] + 0.0025) + elif key == glfw.KEY_DOWN: + state["strength"] = max(0.0, state["strength"] - 0.0025) + elif key == glfw.KEY_RIGHT: + state["conv"] = min(1.0, state["conv"] + 0.02) + elif key == glfw.KEY_LEFT: + state["conv"] = max(0.0, state["conv"] - 0.02) + elif key == glfw.KEY_X and action == glfw.PRESS: + state["swap"] = not state["swap"] + elif key == glfw.KEY_D and action == glfw.PRESS: + state["debug"] = not state["debug"] + elif key == glfw.KEY_F and action == glfw.PRESS: + if state["fullscreen"]: + x, y, ww, hh = state["windowed_geom"] + glfw.set_window_monitor(window, None, x, y, ww, hh, 0) + else: + x, y = glfw.get_window_pos(window) + ww, hh = glfw.get_window_size(window) + state["windowed_geom"] = (x, y, ww, hh) + m = glfw.get_primary_monitor() + vm = glfw.get_video_mode(m) + glfw.set_window_monitor(window, m, 0, 0, vm.size.width, vm.size.height, + vm.refresh_rate) + state["fullscreen"] = not state["fullscreen"] + + glfw.set_key_callback(win, on_key) + + color_tex = None + color_size = None + depth_tex = make_tex(ctx, 2, 2, 1, "f4", np.full((2, 2), 0.5, dtype="f4").tobytes()) + depth_size = (2, 2) + last_color_seq = -1 + last_depth_seq = -1 + last_title = 0.0 + + prog["color_tex"].value = 0 + prog["depth_tex"].value = 1 + prog["steps"].value = args.steps + + try: + while not glfw.window_should_close(win): + glfw.poll_events() + + frame, cseq = grabber.latest() + if frame is not None and cseq != last_color_seq: + last_color_seq = cseq + fh, fw = frame.shape[:2] + if color_size != (fw, fh): + if color_tex is not None: + color_tex.release() + color_tex = make_tex(ctx, fw, fh, 4, "f1") + color_tex.swizzle = "BGRA" + color_size = (fw, fh) + color_tex.write(np.ascontiguousarray(frame)) + + depth, dseq = worker.latest() + if depth is not None and dseq != last_depth_seq: + last_depth_seq = dseq + dh, dw = depth.shape + if depth_size != (dw, dh): + depth_tex.release() + depth_tex = make_tex(ctx, dw, dh, 1, "f4") + depth_size = (dw, dh) + depth_tex.write(depth.tobytes()) + + fbw, fbh = glfw.get_framebuffer_size(win) + ctx.viewport = (0, 0, fbw, fbh) + ctx.clear(0.0, 0.0, 0.0) + + if color_tex is not None: + color_tex.use(0) + depth_tex.use(1) + prog["strength"].value = state["strength"] + prog["convergence"].value = state["conv"] + prog["show_depth"].value = 1 if state["debug"] else 0 + half = fbw // 2 + sign = -1.0 if state["swap"] else 1.0 + for i, eye in enumerate((-1.0 * sign, 1.0 * sign)): + ctx.viewport = (i * half, 0, half, fbh) + prog["eye"].value = eye + vao.render(moderngl.TRIANGLE_STRIP) + + glfw.swap_buffers(win) + + now = time.time() + if now - last_title > 1.0: + last_title = now + glfw.set_window_title( + win, + f"depth3d SBS | depth {worker.fps:.0f} fps | " + f"strength {state['strength']:.3f} | conv {state['conv']:.2f}", + ) + finally: + worker.stop_evt.set() + grabber.stop() + glfw.terminate() + + +if __name__ == "__main__": + main() diff --git a/depth3d_vr.py b/depth3d_vr.py new file mode 100644 index 0000000..851e709 --- /dev/null +++ b/depth3d_vr.py @@ -0,0 +1,680 @@ +""" +depth3d_vr.py - SteamVR 2D -> 3D player with an in-VR settings panel and immersive environments. + +Captures a monitor (or window), estimates depth with Depth Anything V2, and shows it as a +curved screen in SteamVR with real per-eye depth. It is a normal SteamVR scene app, so it +streams to the Steam Frame like any VR game. + +In VR: press MENU or A on a controller to open the settings panel, point with the laser, + trigger = click / drag. Settings are saved to settings.json (Save button, and + automatically on exit). +Hotkeys (hold Ctrl+Alt, work even when another window has focus): + Space open / close settings panel Enter click (fallback if trigger fails) + Up / Down depth strength PgUp/PgDn screen size + Left / Right convergence Home recenter + End swap eyes Insert depth-map debug view + Delete 3D on / off +""" +import argparse +import copy +import ctypes +import json +import math +import time +from pathlib import Path + +import glfw +import moderngl +import numpy as np +import openvr + +from depth3d import MODELS, DepthWorker, Grabber, list_windows, make_tex +from vr_env import LAYOUTS, EnvManager +from vr_shaders import (DILATE_FRAG, FLAT_FRAG, FLAT_VERT, FS_VERT, PANEL_FRAG, PANEL_VERT, + REFINE_FRAG, SCREEN_FRAG, SCREEN_VERT) +from vr_ui import PANEL_H_M, PANEL_W_M, PH, PW, Panel, ray_panel_hit + +HERE = Path(__file__).resolve().parent +SETTINGS_PATH = HERE / "settings.json" +ENV_DIR = HERE / "environments" + +SAVE_KEYS = ["depth_cm", "conv", "fov", "distance", "height", "sharpen", "stereo3d", "swap", + "env_name", "env_layouts", "env_brightness", "env_yaw", "env_play", "pointer_tilt"] + +# Hotkey limits and adjustment speeds (units per second while held) +LIMITS = dict(depth_cm=(0.0, 30.0), conv=(0.0, 1.0), fov=(20.0, 180.0)) +RATES = dict(depth_cm=5.0, conv=0.3, fov=35.0) + +UI_DISTANCE = 1.4 # metres in front of you when the panel opens + + +# ---------------------------------------------------------------- math helpers +def m34(m): + a = np.eye(4, dtype=np.float64) + for r in range(3): + for c in range(4): + a[r, c] = m[r][c] + return a + + +def m44(m): + return np.array([[m[r][c] for c in range(4)] for r in range(4)], dtype=np.float64) + + +def rot_y(a): + c, s = math.cos(a), math.sin(a) + return np.array([[c, 0, s, 0], [0, 1, 0, 0], [-s, 0, c, 0], [0, 0, 0, 1]], dtype=np.float64) + + +def translate(x, y, z): + m = np.eye(4, dtype=np.float64) + m[:3, 3] = (x, y, z) + return m + + +def head_yaw(head): + """Yaw (about +Y) that points -Z at where the head is looking, ignoring pitch/roll.""" + fwd = -head[:3, 2] + return math.atan2(-fwd[0], -fwd[2]) + + +def build_screen(aspect, dist, fov_deg, segs=96): + """Curved (cylindrical) screen centred on the viewer. Returns (vertex array, width in m).""" + phi = math.radians(fov_deg) + width = phi * dist + height = width / aspect + v = [] + for i in range(segs + 1): + t = i / segs + a = (t - 0.5) * phi + x, z = dist * math.sin(a), -dist * math.cos(a) + v += [x, height / 2, z, t, 0.0] + v += [x, -height / 2, z, t, 1.0] + return np.array(v, dtype="f4"), width + + +def pointer_ray(pose, tilt_deg): + t = math.radians(tilt_deg) + d = pose[:3, :3] @ np.array([0.0, math.sin(t), -math.cos(t)]) + return pose[:3, 3].copy(), d / np.linalg.norm(d) + + +# ---------------------------------------------------------------- settings +def load_saved(): + try: + return json.loads(SETTINGS_PATH.read_text()) + except Exception: # noqa: BLE001 + return {} + + +def save_settings(st): + SETTINGS_PATH.write_text(json.dumps({k: st[k] for k in SAVE_KEYS}, indent=2)) + + +# ---------------------------------------------------------------- keyboard (global) +user32 = ctypes.windll.user32 if hasattr(ctypes, "windll") else None +VK = dict(CTRL=0x11, ALT=0x12, SPACE=0x20, PGUP=0x21, PGDN=0x22, END=0x23, HOME=0x24, + LEFT=0x25, UP=0x26, RIGHT=0x27, DOWN=0x28, INS=0x2D, DEL=0x2E, RETURN=0x0D) + + +def down(name): + return user32 is not None and bool(user32.GetAsyncKeyState(VK[name]) & 0x8000) + + +# ---------------------------------------------------------------- controllers +class Controllers: + """Legacy controller polling: pose + trigger + buttons for every connected controller.""" + + def __init__(self, vrsys, debug): + self.vrsys = vrsys + self.debug = debug + self.devs = [] + self.frame = 0 + self.prev = {} + + def poll(self, poses): + if self.frame % 45 == 0: + self.devs = [i for i in range(openvr.k_unMaxTrackedDeviceCount) + if self.vrsys.getTrackedDeviceClass(i) == openvr.TrackedDeviceClass_Controller] + self.frame += 1 + out = [] + for i in self.devs: + p = poses[i] + if not (p.bDeviceIsConnected and p.bPoseIsValid): + continue + ok, state = self.vrsys.getControllerState(i) + mask = int(state.ulButtonPressed) if ok else 0 + trigger = bool(mask & (1 << openvr.k_EButton_SteamVR_Trigger)) or state.rAxis[1].x > 0.6 + new = mask & ~self.prev.get(i, 0) + self.prev[i] = mask + if self.debug and new: + bits = [b for b in range(64) if new >> b & 1] + print(f"\n[input] controller {i} role {self.vrsys.getControllerRoleForTrackedDeviceIndex(i)}" + f" pressed button ids {bits}") + out.append(dict(index=i, role=self.vrsys.getControllerRoleForTrackedDeviceIndex(i), + pose=m34(p.mDeviceToAbsoluteTracking), trigger=trigger, mask=mask, new=new)) + return out + + +# ---------------------------------------------------------------- the app +class App: + def __init__(self, args): + self.args = args + self.base = dict(depth_cm=args.depth_cm, conv=args.convergence, fov=args.fov, + distance=args.distance, height=-args.drop, sharpen=args.sharpen, + stereo3d=True, swap=False, debug=False, env_name="", env_layouts={}, + env_brightness=1.0, env_yaw=0.0, env_play=True, pointer_tilt=args.pointer_tilt) + self.st = copy.deepcopy(self.base) + self.st.update({k: v for k, v in load_saved().items() if k in self.base}) + + self.grabber = Grabber(args.window, args.source_monitor, cursor=not args.no_cursor) + self.worker = DepthWorker(self.grabber, args.depth_width, args.smooth, MODELS[args.model], + args.cut_threshold) + self.worker.start() + + self._init_gl() + self._init_vr() + self._init_render() + + self.env = EnvManager(self.ctx, ENV_DIR) + self.env.paused = not self.st["env_play"] + if self.st["env_name"] in self.env.names(): + self.env.select(self.st["env_name"]) + else: + self.st["env_name"] = "" + + self.panel = Panel(self.st, self.on_action) + self.ui_tex = self.ctx.texture((PW, PH), 4) + self.ui_tex.filter = (moderngl.LINEAR, moderngl.LINEAR) + self.ui_open = False + self.panel_model = np.eye(4) + self.panel_inv = np.eye(4) + self.rays = [] + + self.head = np.eye(4) + self.anchor_pos = np.zeros(3) + self.anchor_yaw = 0.0 + self.need_recenter = True + self.prev_keys = {} + self.dirty = True + self.running = True + + # ------------------------------------------------------------------ setup + def _init_gl(self): + if not glfw.init(): + raise RuntimeError("GLFW init failed") + glfw.window_hint(glfw.VISIBLE, glfw.FALSE) + glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 3) + glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 3) + glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE) + self.win = glfw.create_window(64, 64, "depth3d_vr", None, None) + glfw.make_context_current(self.win) + glfw.swap_interval(0) + self.ctx = moderngl.create_context() + + def _init_vr(self): + self.vrsys = openvr.init(openvr.VRApplication_Scene) + self.comp = openvr.VRCompositor() + try: + ipd = float(self.vrsys.getFloatTrackedDeviceProperty( + openvr.k_unTrackedDeviceIndex_Hmd, openvr.Prop_UserIpdMeters_Float)) + except Exception: # noqa: BLE001 + ipd = 0.063 + self.ipd = ipd if 0.04 < ipd < 0.09 else 0.063 + + tw, th = self.vrsys.getRecommendedRenderTargetSize() + self.tw, self.th = int(tw * self.args.render_scale), int(th * self.args.render_scale) + print(f"Per-eye render target: {self.tw}x{self.th}") + self.targets = [] + for _ in range(2): + tex = self.ctx.texture((self.tw, self.th), 4) + tex.filter = (moderngl.LINEAR, moderngl.LINEAR) + self.targets.append((tex, self.ctx.framebuffer(color_attachments=[tex]))) + + eyes = (openvr.Eye_Left, openvr.Eye_Right) + self.proj = [m44(self.vrsys.getProjectionMatrix(e, 0.1, 200.0)) for e in eyes] + self.inv_e2h = [np.linalg.inv(m34(self.vrsys.getEyeToHeadTransform(e))) for e in eyes] + self.controllers = Controllers(self.vrsys, self.args.debug_input) + + def _init_render(self): + ctx = self.ctx + self.prog = ctx.program(vertex_shader=SCREEN_VERT, fragment_shader=SCREEN_FRAG) + self.prog["color_tex"].value = 0 + self.prog["depth_tex"].value = 1 + self.prog["samples"].value = max(4, self.args.samples) + verts, self.screen_w = build_screen(16 / 9, self.st["distance"], self.st["fov"]) + self.vbo = ctx.buffer(verts.tobytes()) + self.vao = ctx.vertex_array(self.prog, [(self.vbo, "3f 2f", "pos", "tc")]) + self.mesh_key = None + + # depth refinement passes + quad = ctx.buffer(np.array([-1, -1, 1, -1, -1, 1, 1, 1], dtype="f4").tobytes()) + self.prog_ref = ctx.program(vertex_shader=FS_VERT, fragment_shader=REFINE_FRAG) + self.prog_dil = ctx.program(vertex_shader=FS_VERT, fragment_shader=DILATE_FRAG) + self.vao_ref = ctx.vertex_array(self.prog_ref, [(quad, "2f", "pos")]) + self.vao_dil = ctx.vertex_array(self.prog_dil, [(quad, "2f", "pos")]) + self.prog_ref["color_tex"].value = 0 + self.prog_ref["depth_tex"].value = 1 + self.prog_ref["sigma_c"].value = self.args.edge_sigma + self.prog_dil["src"].value = 2 + self.prog_dil["radius"].value = self.args.dilate + + self.S = dict(color_tex=None, color_size=None, depth_size=(2, 2), ref=None) + self.S["depth_tex"] = make_tex(ctx, 2, 2, 1, "f4", np.full((2, 2), 0.5, dtype="f4").tobytes()) + self.S["final_depth"] = self.S["depth_tex"] + self.last_cseq = self.last_dseq = -1 + + # settings panel + laser pointers + self.prog_panel = ctx.program(vertex_shader=PANEL_VERT, fragment_shader=PANEL_FRAG) + self.prog_panel["ui_tex"].value = 4 + w, h = PANEL_W_M / 2, PANEL_H_M / 2 + pv = np.array([-w, h, 0, 0, 0, -w, -h, 0, 0, 1, w, h, 0, 1, 0, w, -h, 0, 1, 1], dtype="f4") + self.vao_panel = ctx.vertex_array(self.prog_panel, + [(ctx.buffer(pv.tobytes()), "3f 2f", "pos", "tc")]) + self.prog_flat = ctx.program(vertex_shader=FLAT_VERT, fragment_shader=FLAT_FRAG) + self.dyn = ctx.buffer(reserve=2048, dynamic=True) + self.vao_flat = ctx.vertex_array(self.prog_flat, [(self.dyn, "3f", "pos")]) + + # ------------------------------------------------------------------ capture textures + def _make_refine_targets(self, cw, ch): + if self.S["ref"] is not None: + for obj in self.S["ref"]: + obj.release() + self.S["ref"] = None + if self.args.refine_width <= 0: + return + rw = min(cw, self.args.refine_width) + rh = max(1, int(round(ch * rw / cw))) + ta, tb = make_tex(self.ctx, rw, rh, 1, "f4"), make_tex(self.ctx, rw, rh, 1, "f4") + self.S["ref"] = (ta, self.ctx.framebuffer(color_attachments=[ta]), + tb, self.ctx.framebuffer(color_attachments=[tb])) + self.prog_dil["texel"].value = (1.0 / rw, 1.0 / rh) + + def upload_color(self, frame): + S = self.S + fh, fw = frame.shape[:2] + if S["color_size"] != (fw, fh): + if S["color_tex"] is not None: + S["color_tex"].release() + t = make_tex(self.ctx, fw, fh, 4, "f1") + t.filter = (moderngl.LINEAR_MIPMAP_LINEAR, moderngl.LINEAR) # crisp text when minified + t.anisotropy = 16.0 + t.swizzle = "BGRA" + S["color_tex"], S["color_size"] = t, (fw, fh) + self._make_refine_targets(fw, fh) + S["color_tex"].write(np.ascontiguousarray(frame)) + S["color_tex"].build_mipmaps() + + def upload_depth(self, depth): + S = self.S + dh, dw = depth.shape + if S["depth_size"] != (dw, dh): + S["depth_tex"].release() + S["depth_tex"] = make_tex(self.ctx, dw, dh, 1, "f4") + S["depth_size"] = (dw, dh) + self.prog_ref["depth_texel"].value = (1.0 / dw, 1.0 / dh) + S["depth_tex"].write(depth.tobytes()) + + def refine(self): + S = self.S + if S["ref"] is None: + return S["depth_tex"] + ta, fa, tb, fb = S["ref"] + S["color_tex"].use(0) + S["depth_tex"].use(1) + fa.use() + self.vao_ref.render(moderngl.TRIANGLE_STRIP) + if self.args.dilate > 0: + ta.use(2) + fb.use() + self.vao_dil.render(moderngl.TRIANGLE_STRIP) + return tb + return ta + + # ------------------------------------------------------------------ actions / UI + def layout_for_env(self): + name = self.st["env_name"] + return self.st["env_layouts"].get(name) or self.env.guessed + + def cycle_env(self, step): + names = self.env.names() + i = names.index(self.st["env_name"]) if self.st["env_name"] in names else 0 + self.st["env_name"] = names[(i + step) % len(names)] + self.env.select(self.st["env_name"]) + self.panel.invalidate() + + def on_action(self, name): + if name == "save": + save_settings(self.st) + self.panel.toast("Saved") + elif name == "reset": + self.st.clear() + self.st.update(copy.deepcopy(self.base)) + self.env.select("") + self.panel.toast("Reset to launcher defaults (Save to keep)") + self.panel.invalidate() + elif name == "close": + self.ui_open = False + elif name == "recenter": + self.need_recenter = True + elif name == "env_prev": + self.cycle_env(-1) + elif name == "env_next": + self.cycle_env(+1) + elif name == "env_layout": + cur = self.layout_for_env() + nxt = LAYOUTS[(LAYOUTS.index(cur) + 1) % len(LAYOUTS)] if cur in LAYOUTS else LAYOUTS[0] + self.st["env_layouts"][self.st["env_name"]] = nxt + self.panel.invalidate() + elif name == "toggled:env_play": + self.env.paused = not self.st["env_play"] + self.dirty = True + + def toggle_ui(self): + self.ui_open = not self.ui_open + if self.ui_open: + yaw = head_yaw(self.head) + fwd = np.array([-math.sin(yaw), 0.0, -math.cos(yaw)]) + pos = self.head[:3, 3] + fwd * UI_DISTANCE + np.array([0.0, -0.15, 0.0]) + self.panel_model = translate(*pos) @ rot_y(yaw) + self.panel_inv = np.linalg.inv(self.panel_model) + self.panel.invalidate() + + def update_ui(self, ctrls): + kb_click = down("CTRL") and down("ALT") and down("RETURN") + best = None + self.rays = [] + for c in sorted(ctrls, key=lambda c: c["role"] != openvr.TrackedControllerRole_RightHand): + o, d = pointer_ray(c["pose"], self.st["pointer_tilt"]) + trig = c["trigger"] or (kb_click and best is None) + hit = ray_panel_hit(o, d, self.panel_inv) + self.rays.append((o, d, hit[0] if hit else None, trig)) + if hit and (best is None or (trig and not best[1])): + best = (hit, trig) + if best: + (_, px, py), trig = best + self.panel.pointer(px, py, trig) + else: + self.panel.pointer(None, None, False) + data = self.panel.render_if_dirty() + if data is not None: + self.ui_tex.write(data) + + # ------------------------------------------------------------------ hotkeys + def handle_keys(self, dt): + st = self.st + chord = down("CTRL") and down("ALT") + if chord: + def adjust(name, sign): + lo, hi = LIMITS[name] + st[name] = min(hi, max(lo, st[name] + sign * RATES[name] * dt)) + self.dirty = True + for key, name, sign in (("UP", "depth_cm", 1), ("DOWN", "depth_cm", -1), + ("RIGHT", "conv", 1), ("LEFT", "conv", -1), + ("PGUP", "fov", 1), ("PGDN", "fov", -1)): + if down(key): + adjust(name, sign) + if self.ui_open: + self.panel.invalidate() + for key, action in (("HOME", "recenter"), ("END", "swap"), ("INS", "debug"), + ("SPACE", "ui"), ("DEL", "3d")): + cur = chord and down(key) + if cur and not self.prev_keys.get(key): + if action == "recenter": + self.need_recenter = True + elif action == "ui": + self.toggle_ui() + elif action == "3d": + st["stereo3d"] = not st["stereo3d"] + else: + st[action] = not st[action] + self.dirty = True + if self.ui_open: + self.panel.invalidate() + self.prev_keys[key] = cur + + # ------------------------------------------------------------------ rendering + def draw_lasers(self, vp): + if not self.rays: + return + head_pos = self.head[:3, 3] + right, up = self.panel_model[:3, 0], self.panel_model[:3, 1] + normal = self.panel_model[:3, 2] + for o, d, t, trig in self.rays: + length = t if t is not None else 1.2 + e = o + d * length + side = np.cross(d, head_pos - o) + n = np.linalg.norm(side) + if n < 1e-6: + continue + side = side / n * 0.003 + quad = np.array([o + side, o - side, e + side, e - side], dtype="f4") + self.prog_flat["mvp"].write(vp.T.astype("f4").tobytes()) + self.prog_flat["color"].value = (1.0, 0.85, 0.3, 0.9) if trig else (0.4, 0.75, 1.0, 0.75) + self.dyn.write(quad.tobytes()) + self.vao_flat.render(moderngl.TRIANGLE_STRIP, vertices=4) + if t is not None: + c = e + normal * 0.004 + ring = [c] + [c + (math.cos(a) * right + math.sin(a) * up) * 0.013 + for a in np.linspace(0, 2 * math.pi, 15)] + self.prog_flat["color"].value = (1.0, 1.0, 1.0, 0.95) + self.dyn.write(np.array(ring, dtype="f4").tobytes()) + self.vao_flat.render(moderngl.TRIANGLE_FAN, vertices=len(ring)) + + def render_eyes(self): + st, S, env = self.st, self.S, self.env + have_color = S["color_tex"] is not None + if have_color: + S["color_tex"].use(0) + S["final_depth"].use(1) + sign = -1.0 if st["swap"] else 1.0 + strength = (0.5 * (st["depth_cm"] / 100.0) / self.screen_w) if st["stereo3d"] else 0.0 + self.prog["strength"].value = strength + self.prog["convergence"].value = st["conv"] + self.prog["sharpen"].value = st["sharpen"] + self.prog["show_depth"].value = 1 if (st["debug"] and st["stereo3d"]) else 0 + + screen_model = (translate(self.anchor_pos[0], self.anchor_pos[1] + st["height"], self.anchor_pos[2]) + @ rot_y(self.anchor_yaw)) + view_base = np.linalg.inv(self.head) + head_rot_inv = np.linalg.inv(self._head_rot()) + env_yaw = self.anchor_yaw + math.radians(st["env_yaw"]) + env_layout = self.layout_for_env() + + for i in range(2): + tex, fbo = self.targets[i] + fbo.use() + self.ctx.viewport = (0, 0, self.tw, self.th) + self.ctx.clear(0.0, 0.0, 0.0) + eye = -1.0 if i == 0 else 1.0 + if env.ready: + env.draw(self.proj[i] @ self.inv_e2h[i] @ head_rot_inv, eye, env_layout, + env_yaw, st["env_brightness"]) + vp = self.proj[i] @ self.inv_e2h[i] @ view_base + if have_color: + self.prog["mvp"].write((vp @ screen_model).T.astype("f4").tobytes()) + self.prog["eye"].value = eye * sign + self.vao.render(moderngl.TRIANGLE_STRIP) + if self.ui_open: + self.ctx.enable(moderngl.BLEND) + self.ui_tex.use(4) + self.prog_panel["mvp"].write((vp @ self.panel_model).T.astype("f4").tobytes()) + self.vao_panel.render(moderngl.TRIANGLE_STRIP) + self.draw_lasers(vp) + self.ctx.disable(moderngl.BLEND) + self.ctx.finish() + for i, eye in enumerate((openvr.Eye_Left, openvr.Eye_Right)): + t = openvr.Texture_t() + t.handle = int(self.targets[i][0].glo) + t.eType = openvr.TextureType_OpenGL + t.eColorSpace = openvr.ColorSpace_Gamma + self.comp.submit(eye, t) + + def _head_rot(self): + r = self.head.copy() + r[:3, 3] = 0.0 + return r + + # ------------------------------------------------------------------ main loop + def run(self): + args, st, S = self.args, self.st, self.S + poses = (openvr.TrackedDevicePose_t * openvr.k_unMaxTrackedDeviceCount)() + event = openvr.VREvent_t() + last_t, last_print, mesh_key = time.time(), 0.0, None + print("\nRunning. Press MENU or A on a controller (or hold Ctrl+Alt and press Space) " + "to open the settings panel.") + while self.running: + self.comp.waitGetPoses(poses, None) + while self.vrsys.pollNextEvent(event): + if event.eventType == openvr.VREvent_Quit: + self.running = False + + now = time.time() + dt = min(now - last_t, 0.1) + last_t = now + + hp = poses[openvr.k_unTrackedDeviceIndex_Hmd] + if hp.bPoseIsValid: + self.head = m34(hp.mDeviceToAbsoluteTracking) + if self.need_recenter: + self.anchor_pos = self.head[:3, 3].copy() + self.anchor_yaw = head_yaw(self.head) + self.need_recenter = False + + self.handle_keys(dt) + ctrls = self.controllers.poll(poses) + for c in ctrls: + if any(c["new"] >> b & 1 for b in (openvr.k_EButton_ApplicationMenu, openvr.k_EButton_A)): + self.toggle_ui() + break + if self.ui_open: + self.panel.set_info(env_name=st["env_name"], env_layout=self.layout_for_env(), + env_video=self.env.is_video, + env_status="Loading..." if self.env.loading else self.env.error, + ipd=self.ipd) + self.update_ui(ctrls) + + self.worker.active = st["stereo3d"] + self.env.update() + + # ---- upload newest color / depth + updated = False + if args.sync and st["stereo3d"]: + depth, frame, dseq = self.worker.latest_pair() + if depth is not None and frame is not None and dseq != self.last_dseq: + self.last_dseq = dseq + self.upload_color(frame) + self.upload_depth(depth) + updated = True + else: + frame, cseq = self.grabber.latest() + if frame is not None and cseq != self.last_cseq: + self.last_cseq = cseq + self.upload_color(frame) + updated = True + depth, dseq = self.worker.latest() + if st["stereo3d"] and depth is not None and dseq != self.last_dseq: + self.last_dseq = dseq + self.upload_depth(depth) + updated = True + if updated and S["color_tex"] is not None and st["stereo3d"]: + S["final_depth"] = self.refine() + + # ---- screen mesh (rebuild if aspect / size / distance changed) + aspect = (S["color_size"][0] / S["color_size"][1]) if S["color_size"] else 16 / 9 + key = (round(aspect, 4), round(st["fov"], 1), round(st["distance"], 2)) + if key != mesh_key: + mesh_key = key + verts, self.screen_w = build_screen(aspect, st["distance"], st["fov"]) + self.vbo.write(verts.tobytes()) + + self.render_eyes() + + if self.dirty and now - last_print > 0.4: + last_print, self.dirty = now, False + far_cm = st["depth_cm"] * st["conv"] + warn = (" !! far parts exceed eye spacing - lower depth or convergence" + if far_cm / 100.0 > self.ipd and st["stereo3d"] else "") + print(f"\r3D {'on ' if st['stereo3d'] else 'off'} | depth {st['depth_cm']:.1f} cm | " + f"conv {st['conv']:.2f} | screen {st['fov']:.0f} deg | " + f"depth fps {self.worker.fps:.0f}{warn} ", end="", flush=True) + + def shutdown(self): + print("\nShutting down...") + if not self.args.no_autosave: + try: + save_settings(self.st) + except Exception as e: # noqa: BLE001 + print("Could not save settings:", e) + self.env.stop() + self.worker.stop_evt.set() + self.grabber.stop() + openvr.shutdown() + glfw.terminate() + + +def main(): + ap = argparse.ArgumentParser(description="SteamVR 2D->3D depth player") + ap.add_argument("--window", help="Title (or part of it) of the window to capture") + ap.add_argument("--source-monitor", type=int, default=None, + help="Capture a whole monitor (1 = main monitor)") + ap.add_argument("--list", action="store_true", help="List window titles and exit") + ap.add_argument("--model", choices=list(MODELS), default="large") + ap.add_argument("--depth-width", type=int, default=700) + ap.add_argument("--smooth", type=float, default=0.5, + help="Depth temporal blend in calm shots (1 = off, lower = smoother)") + ap.add_argument("--cut-threshold", type=float, default=0.12, + help="Scene-cut sensitivity (lower = detects more cuts)") + ap.add_argument("--sync", action=argparse.BooleanOptionalAction, default=True, + help="Show each video frame together with its own depth map (best for movies)") + ap.add_argument("--samples", type=int, default=24, + help="Warp search samples per pixel (quality vs GPU cost)") + ap.add_argument("--refine-width", type=int, default=1920, help="Depth refinement resolution (0 = off)") + ap.add_argument("--edge-sigma", type=float, default=0.10, help="Edge-snap strictness") + ap.add_argument("--dilate", type=float, default=1.0, help="Grow foreground depth by N pixels (0 = off)") + ap.add_argument("--sharpen", type=float, default=0.5, help="Text sharpening 0..1 (initial value)") + ap.add_argument("--depth-cm", type=float, default=6.0, help="Total disparity range in cm (initial)") + ap.add_argument("--convergence", type=float, default=0.4, help="(initial)") + ap.add_argument("--distance", type=float, default=3.0, help="Screen distance in metres (initial)") + ap.add_argument("--fov", type=float, default=70.0, help="Screen width in degrees (initial)") + ap.add_argument("--drop", type=float, default=0.0, help="Lower the screen below eye level, metres") + ap.add_argument("--render-scale", type=float, default=1.0) + ap.add_argument("--pointer-tilt", type=float, default=0.0, + help="Tilt the controller laser up(+)/down(-) in degrees if it points off") + ap.add_argument("--no-cursor", action="store_true", help="Do not capture the mouse cursor") + ap.add_argument("--no-autosave", action="store_true", help="Do not save settings on exit") + ap.add_argument("--debug-input", action="store_true", help="Print controller button ids as you press them") + args = ap.parse_args() + + if args.list: + print("Windows:") + for t in sorted(set(list_windows())): + print(" ", t) + if glfw.init(): + print("Monitors (use the 'source' number with --source-monitor):") + for i, m in enumerate(glfw.get_monitors()): + vm = glfw.get_video_mode(m) + x, y = glfw.get_monitor_pos(m) + print(f" source={i + 1} {glfw.get_monitor_name(m).decode(errors='ignore')}" + f" {vm.size.width}x{vm.size.height} at ({x},{y})") + glfw.terminate() + return + if not args.window and args.source_monitor is None: + ap.error("give --window TITLE or --source-monitor N") + + try: + app = App(args) + except Exception as e: # noqa: BLE001 + print("\nCould not start:", e) + print("Start SteamVR and connect the Frame (Steam Link -> PC VR) first.") + raise + try: + app.run() + except KeyboardInterrupt: + pass + finally: + app.shutdown() + + +if __name__ == "__main__": + main() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..5ee6477 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +testpaths = tests diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..4429409 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,8 @@ +# Test / lint dependencies (no GPU, torch or Windows needed - GL tests use Mesa llvmpipe) +numpy +pillow +moderngl +scipy +openvr +pytest +pyflakes diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6ee0c33 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# RTX 50-series (Blackwell) needs a CUDA 12.8+ PyTorch build. Install it first: +# pip install torch --index-url https://download.pytorch.org/whl/cu128 +transformers +numpy +glfw +moderngl +windows-capture +openvr +pillow +av diff --git a/run_depth3d_vr.bat b/run_depth3d_vr.bat new file mode 100644 index 0000000..ff170ef --- /dev/null +++ b/run_depth3d_vr.bat @@ -0,0 +1,65 @@ +@echo off +setlocal +cd /d "%~dp0" + +REM ================= STARTING VALUES (edit these) ================= +REM Settings you save from the in-VR menu (settings.json) override the 3D/screen values below. +REM To go back to these, press Reset + Save in the menu, or delete settings.json. + +REM Monitor to convert to 3D (1 = main monitor) +set SOURCE_MONITOR=1 + +REM Depth model: small / base / large (drop to base if the game stutters) +set MODEL=large + +REM Depth model input width in pixels. Higher = crisper depth edges, slower. +set DEPTH_WIDTH=700 + +REM Starting screen size (degrees), distance (metres), text sharpening (0..1) +set FOV=70 +set DISTANCE=3.0 +set SHARPEN=0.5 + +REM Starting 3D look: total depth range in cm, and which depth sits at the screen plane +set DEPTH_CM=6 +set CONVERGENCE=0.4 + +REM --sync = each video frame is shown with its own depth map (best for MOVIES) +REM --no-sync = lowest latency (better for GAMES) +set SYNC_FLAG=--sync + +REM Mouse cursor: leave empty to show it, or set to --no-cursor to hide it +set CURSOR_FLAG= + +REM Warp quality: samples searched per pixel (24 default; 16 = faster, 32 = best edges) +set SAMPLES=24 + +REM Depth edge refinement. REFINE_WIDTH 0 = off. EDGE_SIGMA smaller = sharper. DILATE 0 = off. +set REFINE_WIDTH=1920 +set EDGE_SIGMA=0.10 +set DILATE=1 + +REM Scene-cut sensitivity. Lower = more cuts detected. +set CUT_THRESHOLD=0.12 + +REM Render resolution multiplier per eye (1.0 = SteamVR's recommended size) +set RENDER_SCALE=1.0 + +REM If the menu laser points too high or low, set e.g. 20 or -20 (degrees) +set POINTER_TILT=0 + +REM Set to --debug-input to print controller button ids (helps if the menu button doesn't respond) +set DEBUG_INPUT_FLAG= +REM ================================================================= + +echo Start SteamVR and connect the Steam Frame BEFORE running this. +echo Capturing monitor %SOURCE_MONITOR% with the "%MODEL%" model... +echo In VR: press MENU or A on a controller to open the settings panel. +echo Fallback: hold Ctrl+Alt and press Space. Put 360/180 images and videos in the "environments" folder. +echo. + +py -3.13 depth3d_vr.py --source-monitor %SOURCE_MONITOR% --model %MODEL% --depth-width %DEPTH_WIDTH% --fov %FOV% --distance %DISTANCE% --sharpen %SHARPEN% --depth-cm %DEPTH_CM% --convergence %CONVERGENCE% %SYNC_FLAG% %CURSOR_FLAG% --samples %SAMPLES% --refine-width %REFINE_WIDTH% --edge-sigma %EDGE_SIGMA% --dilate %DILATE% --cut-threshold %CUT_THRESHOLD% --render-scale %RENDER_SCALE% --pointer-tilt %POINTER_TILT% %DEBUG_INPUT_FLAG% + +echo. +echo depth3d_vr exited. If there was an error, it is shown above. +pause diff --git a/run_sbs_window.bat b/run_sbs_window.bat new file mode 100644 index 0000000..27cf1ee --- /dev/null +++ b/run_sbs_window.bat @@ -0,0 +1,35 @@ +@echo off +setlocal +cd /d "%~dp0" + +REM Simple side-by-side window for Desktop+/Bigscreen. The SteamVR player (run_depth3d_vr.bat) is recommended. +REM ================= SETTINGS (edit these) ================= +REM Monitor to convert to 3D (1 = main monitor) +set SOURCE_MONITOR=1 + +REM Depth model: small / base / large +set MODEL=large + +REM Depth model input width in pixels. Lower = faster. Try 518 or 392 if fps drops. +set DEPTH_WIDTH=700 + +REM Size of the side-by-side output window. Keep it at your source aspect ratio. +REM 2580x1080 = 21:9 in Half SBS (Desktop+ "HSBS" or Bigscreen), ~1290 px per eye +REM For a 16:9 monitor use 1920x1080 instead. +set WIN_W=2580 +set WIN_H=1080 + +REM 3D look. Can also be changed live with the arrow keys. +set STRENGTH=0.025 +set CONVERGENCE=0.4 +REM ========================================================= + +echo Capturing monitor %SOURCE_MONITOR% with the "%MODEL%" model... +echo Drag the "depth3d SBS" window to ANOTHER monitor, or it will capture itself. +echo. + +py -3.13 depth3d.py --source-monitor %SOURCE_MONITOR% --model %MODEL% --depth-width %DEPTH_WIDTH% --width %WIN_W% --height %WIN_H% --strength %STRENGTH% --convergence %CONVERGENCE% + +echo. +echo depth3d (SBS window) exited. If there was an error, it is shown above. +pause diff --git a/setup.bat b/setup.bat new file mode 100644 index 0000000..a9b4995 --- /dev/null +++ b/setup.bat @@ -0,0 +1,23 @@ +@echo off +REM Run this ONCE to install everything depth3d needs (Python 3.13). +cd /d "%~dp0" + +echo Installing PyTorch (CUDA 12.8 build, required for RTX 50-series)... +py -3.13 -m pip install --upgrade pip +py -3.13 -m pip install torch --index-url https://download.pytorch.org/whl/cu128 +if errorlevel 1 goto fail + +echo Installing the rest... +py -3.13 -m pip install transformers numpy glfw moderngl windows-capture openvr pillow av +if errorlevel 1 goto fail + +echo. +echo Done. You can now double-click run_depth3d.bat +pause +exit /b 0 + +:fail +echo. +echo Install failed. Check the error above. +pause +exit /b 1 diff --git a/vr_env.py b/vr_env.py new file mode 100644 index 0000000..fb00d3a --- /dev/null +++ b/vr_env.py @@ -0,0 +1,247 @@ +""" +Immersive environments for depth3d_vr: 360/180 degree images and videos, +mono or stereo (side-by-side / over-under), shown on a big sphere around you. + +Drop files into the `environments` folder next to the script. Name hints +(optional, the in-VR menu can also cycle the layout): + "..._180_sbs.mp4" VR180 side-by-side "..._360_ou.jpg" 360 over-under + "..._360.jpg" 360 mono "..._tb", "_lr", "_3dh", "_3dv" also work +""" +import re +import threading +import time +from pathlib import Path + +import moderngl +import numpy as np +from PIL import Image + +from vr_shaders import ENV_FRAG, ENV_VERT + +Image.MAX_IMAGE_PIXELS = None + +IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"} +VIDEO_EXT = {".mp4", ".mkv", ".webm", ".mov", ".m4v", ".avi"} +LAYOUTS = ["360 mono", "360 OU", "360 SBS", "180 mono", "180 SBS", "180 OU"] +MAX_IMAGE_DIM = 8192 +MAX_VIDEO_WIDTH = 4096 + + +def scan(folder): + folder = Path(folder) + if not folder.is_dir(): + return [] + files = [p for p in folder.iterdir() if p.suffix.lower() in IMAGE_EXT | VIDEO_EXT] + return sorted(files, key=lambda p: p.name.lower()) + + +def _tag(name, *words): + pat = r"(^|[^a-z0-9])(" + "|".join(words) + r")([^a-z0-9]|$)" + return re.search(pat, name) is not None + + +def guess_layout(name, w, h): + """Best-effort layout from the file name, falling back to the aspect ratio.""" + n = name.lower() + fov = 180 if "180" in n else (360 if "360" in n else None) + if _tag(n, "sbs", "lr", "3dh", "lr180", "sbs180"): + stereo = "SBS" + elif _tag(n, "ou", "tb", "3dv", "overunder", "over_under", "topbottom"): + stereo = "OU" + elif _tag(n, "mono"): + stereo = "mono" + else: + stereo = None + a = w / max(h, 1) + if fov is None: + fov = 360 + if stereo is None: + if fov == 180: + stereo = "SBS" if a > 1.5 else "mono" + else: + stereo = "OU" if 0.75 < a < 1.35 else ("SBS" if a > 3.0 else "mono") + return f"{fov} {stereo}" + + +def parse_layout(layout): + fov180 = 1 if layout.startswith("180") else 0 + stereo = 1 if "SBS" in layout else (2 if "OU" in layout else 0) + return fov180, stereo + + +def build_sphere(lat_n=48, lon_n=96): + verts = [] + for i in range(lat_n + 1): + lat = np.pi * (i / lat_n - 0.5) + for j in range(lon_n + 1): + lon = 2 * np.pi * (j / lon_n - 0.5) + verts.append((np.cos(lat) * np.sin(lon), np.sin(lat), -np.cos(lat) * np.cos(lon))) + idx = [] + for i in range(lat_n): + for j in range(lon_n): + a = i * (lon_n + 1) + j + b = a + lon_n + 1 + idx += [a, b, a + 1, a + 1, b, b + 1] + return np.array(verts, "f4"), np.array(idx, "i4") + + +class EnvManager: + def __init__(self, ctx, folder): + self.ctx = ctx + self.folder = Path(folder) + self.folder.mkdir(exist_ok=True) + self.files = scan(self.folder) + + self.prog = ctx.program(vertex_shader=ENV_VERT, fragment_shader=ENV_FRAG) + v, i = build_sphere() + self.vao = ctx.vertex_array(self.prog, [(ctx.buffer(v.tobytes()), "3f", "pos")], + index_buffer=ctx.buffer(i.tobytes())) + self.prog["env_tex"].value = 3 + + self.tex = None + self.size = None + self.name = "" + self.is_video = False + self.guessed = "360 mono" + self.loading = False + self.error = "" + self.paused = False + + self._lock = threading.Lock() + self._pending = None + self._pending_info = None + self._gen = 0 + + # ---------------------------------------------------------------- selection + def names(self): + return [""] + [p.name for p in self.files] + + def rescan(self): + self.files = scan(self.folder) + + def select(self, name): + """Start loading `name` ('' = no environment). Loading happens on a worker thread.""" + self._gen += 1 + gen = self._gen + self.error = "" + with self._lock: + self._pending = None + if not name: + self.name = "" + self.loading = False + self.is_video = False + self._release() + return + path = self.folder / name + if not path.exists(): + self.name = "" + self.error = f"{name} not found" + return + self.name = name + self.loading = True + self.is_video = path.suffix.lower() in VIDEO_EXT + target = self._video_loop if self.is_video else self._load_image + threading.Thread(target=target, args=(path, gen), daemon=True).start() + + def _release(self): + if self.tex is not None: + self.tex.release() + self.tex = None + self.size = None + + # ---------------------------------------------------------------- worker threads + def _load_image(self, path, gen): + try: + img = Image.open(path) + img.load() + img = img.convert("RGB") + w, h = img.size + s = min(1.0, MAX_IMAGE_DIM / max(w, h)) + if s < 1.0: + img = img.resize((max(1, int(w * s)), max(1, int(h * s))), Image.LANCZOS) + arr = np.ascontiguousarray(np.asarray(img)) + with self._lock: + if gen == self._gen: + self._pending = arr + except Exception as e: # noqa: BLE001 + if gen == self._gen: + self.error = f"Could not load: {e}" + self.loading = False + + def _video_loop(self, path, gen): + try: + import av + except ImportError: + self.error = "Videos need the 'av' package (run setup.bat)" + self.loading = False + return + try: + while gen == self._gen: + container = av.open(str(path)) + stream = container.streams.video[0] + stream.thread_type = "AUTO" + fps = float(stream.average_rate or 30) + t0, n = time.time(), 0 + for frame in container.decode(stream): + if gen != self._gen: + break + while self.paused and gen == self._gen: + time.sleep(0.05) + t0 = time.time() - n / fps + w, h = frame.width, frame.height + s = min(1.0, MAX_VIDEO_WIDTH / w) + arr = frame.reformat(width=int(w * s) // 2 * 2, height=int(h * s) // 2 * 2, + format="rgb24").to_ndarray() + with self._lock: + if gen == self._gen: + self._pending = arr + n += 1 + delay = t0 + n / fps - time.time() + if delay > 0: + time.sleep(delay) + elif delay < -1.0: # fell far behind: resync, don't spiral + t0, n = time.time(), 0 + container.close() + except Exception as e: # noqa: BLE001 + if gen == self._gen: + self.error = f"Video error: {e}" + self.loading = False + + # ---------------------------------------------------------------- GL thread + def update(self): + """Upload a newly decoded image / video frame (call once per VR frame).""" + with self._lock: + arr, self._pending = self._pending, None + if arr is None: + return + h, w = arr.shape[:2] + if self.size != (w, h): + self._release() + self.tex = self.ctx.texture((w, h), 3, dtype="f1") + self.tex.filter = (moderngl.LINEAR_MIPMAP_LINEAR, moderngl.LINEAR) + self.tex.repeat_x = True + self.tex.repeat_y = False + self.tex.anisotropy = 8.0 + self.size = (w, h) + self.guessed = guess_layout(self.name, w, h) + self.tex.write(arr) + self.tex.build_mipmaps() + self.loading = False + + @property + def ready(self): + return self.tex is not None and bool(self.name) + + def draw(self, mvp, eye, layout, yaw, brightness): + fov180, stereo = parse_layout(layout) + self.tex.use(3) + self.prog["mvp"].write(mvp.T.astype("f4").tobytes()) + self.prog["stereo"].value = stereo + self.prog["fov180"].value = fov180 + self.prog["eye"].value = eye + self.prog["yaw"].value = yaw + self.prog["brightness"].value = brightness + self.vao.render() + + def stop(self): + self._gen += 1 diff --git a/vr_shaders.py b/vr_shaders.py new file mode 100644 index 0000000..16c2643 --- /dev/null +++ b/vr_shaders.py @@ -0,0 +1,216 @@ +"""GLSL shaders for depth3d_vr.py (kept separate so they can be tested on their own).""" + +# ---------------------------------------------------------------- the 3D screen +SCREEN_VERT = """ +#version 330 +uniform mat4 mvp; +in vec3 pos; +in vec2 tc; +out vec2 uv; +void main() { + uv = tc; + gl_Position = mvp * vec4(pos, 1.0); +} +""" + +# For every output pixel, SEARCH along the scan line for the source pixel that lands here +# (nearest one wins, like a z-buffer). Revealed gaps are filled from the far side. +# Mip-mapped + anisotropic sampling keeps text readable; optional sharpening on top. +SCREEN_FRAG = """ +#version 330 +uniform sampler2D color_tex; +uniform sampler2D depth_tex; +uniform float strength; // per-eye uv shift per unit of (depth - convergence) +uniform float convergence; +uniform float eye; // -1 left, +1 right +uniform float sharpen; +uniform int samples; +uniform int show_depth; +in vec2 uv; +out vec4 fragColor; +void main() { + vec2 gx = dFdx(uv); // footprint of one output pixel -> picks the right mip level + vec2 gy = dFdy(uv); + if (show_depth == 1) { + fragColor = vec4(vec3(textureLod(depth_tex, uv, 0.0).r), 1.0); + return; + } + float sxs = uv.x; + if (strength > 1e-7) { + float R = strength * max(convergence, 1.0 - convergence); + float stp = 2.0 * R / float(samples - 1); + float tol = 0.75 * stp; + float gapmax = strength; + float best_d = -1.0, fb_score = 1e9, fb_sx = uv.x; + for (int i = 0; i < samples; i++) { + float sx = uv.x - R + stp * float(i); + float d = textureLod(depth_tex, vec2(sx, uv.y), 0.0).r; + float r = abs(sx - eye * strength * (d - convergence) - uv.x); + if (r <= tol && d > best_d) best_d = d; + float score = r / gapmax + 2.0 * d; // in a gap: nearest FAR sample + if (r <= gapmax && score < fb_score) { fb_score = score; fb_sx = sx; } + } + sxs = (best_d >= 0.0) ? uv.x + eye * strength * (best_d - convergence) : fb_sx; + } + vec2 p = vec2(sxs, uv.y); + vec3 col = textureGrad(color_tex, p, gx, gy).rgb; + if (sharpen > 0.0) { + vec3 blur = 0.25 * (textureGrad(color_tex, p + gx, gx, gy).rgb + + textureGrad(color_tex, p - gx, gx, gy).rgb + + textureGrad(color_tex, p + gy, gx, gy).rgb + + textureGrad(color_tex, p - gy, gx, gy).rgb); + col = clamp(col + sharpen * (col - blur), 0.0, 1.0); + } + fragColor = vec4(col, 1.0); +} +""" + +# ---------------------------------------------------------------- depth refinement passes +FS_VERT = """ +#version 330 +in vec2 pos; +out vec2 uv; +void main() { + uv = pos * 0.5 + 0.5; + gl_Position = vec4(pos, 0.0, 1.0); +} +""" + +# Joint-bilateral upsampling: snaps soft low-res depth edges onto the real colour edges. +REFINE_FRAG = """ +#version 330 +uniform sampler2D color_tex; +uniform sampler2D depth_tex; +uniform vec2 depth_texel; // 1 / raw depth size +uniform float sigma_c; // colour similarity (smaller = harder edges) +in vec2 uv; +out vec4 fragColor; +void main() { + vec3 c0 = texture(color_tex, uv).rgb; + float acc = 0.0, wsum = 0.0; + for (int j = -2; j <= 2; j++) { + for (int i = -2; i <= 2; i++) { + vec2 p = uv + vec2(float(i), float(j)) * depth_texel; + vec3 dc = texture(color_tex, p).rgb - c0; + float ws = exp(-float(i * i + j * j) / 4.5); + float wc = exp(-dot(dc, dc) / (2.0 * sigma_c * sigma_c)); + float w = ws * wc; + acc += w * texture(depth_tex, p).r; + wsum += w; + } + } + fragColor = vec4(acc / max(wsum, 1e-5), 0.0, 0.0, 1.0); +} +""" + +# Tiny foreground dilation so the nearest depth fully covers an object's rim. +DILATE_FRAG = """ +#version 330 +uniform sampler2D src; +uniform vec2 texel; +uniform float radius; // in pixels of the refined map +in vec2 uv; +out vec4 fragColor; +void main() { + float m = texture(src, uv).r; + for (int j = -1; j <= 1; j++) { + for (int i = -1; i <= 1; i++) { + m = max(m, texture(src, uv + vec2(float(i), float(j)) * texel * radius).r); + } + } + fragColor = vec4(m, 0.0, 0.0, 1.0); +} +""" + +# ---------------------------------------------------------------- immersive environment +ENV_VERT = """ +#version 330 +uniform mat4 mvp; +in vec3 pos; +out vec3 dir; +void main() { + dir = pos; + gl_Position = mvp * vec4(pos * 80.0, 1.0); +} +""" + +# Equirectangular 360 / 180 viewer, mono or stereo (side-by-side / over-under). +ENV_FRAG = """ +#version 330 +uniform sampler2D env_tex; +uniform int stereo; // 0 mono, 1 side-by-side, 2 over-under +uniform int fov180; // 0 = 360, 1 = 180 (front half only) +uniform float eye; // -1 left, +1 right +uniform float yaw; // radians: rotation of the environment about the vertical axis +uniform float brightness; +in vec3 dir; +out vec4 fragColor; +const float PI = 3.14159265358979; +void main() { + vec3 d = normalize(dir); + float c = cos(yaw), s = sin(yaw); + vec3 e = vec3(d.x * c - d.z * s, d.y, d.x * s + d.z * c); // world -> environment frame + float lon = atan(e.x, -e.z); + float lat = asin(clamp(e.y, -1.0, 1.0)); + float v = 0.5 - lat / PI; + vec2 uvA = vec2(lon / (2.0 * PI) + 0.5, v); + vec2 uvB = vec2(fract(lon / (2.0 * PI)), v); + vec2 gx = dFdx(uvA), gy = dFdy(uvA); + vec2 gxb = dFdx(uvB), gyb = dFdy(uvB); + if (abs(gxb.x) + abs(gyb.x) < abs(gx.x) + abs(gy.x)) { gx = gxb; gy = gyb; } // hide the wrap seam + vec2 uv = uvA; + float inside = 1.0; + if (fov180 == 1) { + uv.x = lon / PI + 0.5; + gx.x *= 2.0; gy.x *= 2.0; + inside = 1.0 - smoothstep(0.5 * PI - 0.03, 0.5 * PI, abs(lon)); + } + vec2 scale = vec2(1.0), off = vec2(0.0); + float right = step(0.0, eye); + if (stereo == 1) { scale.x = 0.5; off.x = 0.5 * right; } + if (stereo == 2) { scale.y = 0.5; off.y = 0.5 * right; } + vec3 col = textureGrad(env_tex, uv * scale + off, gx * scale, gy * scale).rgb; + fragColor = vec4(col * brightness * inside, 1.0); +} +""" + +# ---------------------------------------------------------------- settings panel + laser pointers +PANEL_VERT = """ +#version 330 +uniform mat4 mvp; +in vec3 pos; +in vec2 tc; +out vec2 uv; +void main() { + uv = tc; + gl_Position = mvp * vec4(pos, 1.0); +} +""" + +PANEL_FRAG = """ +#version 330 +uniform sampler2D ui_tex; +in vec2 uv; +out vec4 fragColor; +void main() { + fragColor = texture(ui_tex, uv); +} +""" + +FLAT_VERT = """ +#version 330 +uniform mat4 mvp; +in vec3 pos; +void main() { + gl_Position = mvp * vec4(pos, 1.0); +} +""" + +FLAT_FRAG = """ +#version 330 +uniform vec4 color; +out vec4 fragColor; +void main() { + fragColor = color; +} +""" diff --git a/vr_ui.py b/vr_ui.py new file mode 100644 index 0000000..f42c4f8 --- /dev/null +++ b/vr_ui.py @@ -0,0 +1,267 @@ +""" +Floating settings panel for depth3d_vr. Drawn with Pillow into a texture and operated +with VR controller laser pointers (trigger = click / drag). +""" +import time + +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +PW, PH = 1280, 800 # panel texture size in pixels +PANEL_W_M = 1.6 # physical size in the VR world (metres) +PANEL_H_M = PANEL_W_M * PH / PW + +COL_BG = (16, 18, 26, 240) +COL_TEXT = (235, 238, 245, 255) +COL_DIM = (150, 156, 172, 255) +COL_ACCENT = (64, 156, 255, 255) +COL_WIDGET = (48, 52, 68, 255) +COL_HOVER = (72, 78, 100, 255) +COL_WARN = (255, 190, 80, 255) + +TABS = ["3D", "Screen", "Environment"] + +_fonts = {} + + +def font(size, bold=False): + key = (size, bold) + if key in _fonts: + return _fonts[key] + names = (["segoeuib.ttf", "arialbd.ttf", "DejaVuSans-Bold.ttf"] if bold + else ["segoeui.ttf", "arial.ttf", "DejaVuSans.ttf"]) + f = None + for n in names: + try: + f = ImageFont.truetype(n, size) + break + except OSError: + continue + if f is None: + f = ImageFont.load_default(size) + _fonts[key] = f + return f + + +def ray_panel_hit(origin, direction, inv_model): + """Intersect a world-space ray with the panel. Returns (t, px, py) or None.""" + o = inv_model @ np.append(origin, 1.0) + d = inv_model[:3, :3] @ direction + if abs(d[2]) < 1e-6: + return None + t = -o[2] / d[2] + if t <= 0: + return None + x, y = o[0] + t * d[0], o[1] + t * d[1] + if abs(x) > PANEL_W_M / 2 or abs(y) > PANEL_H_M / 2: + return None + return t, (x / PANEL_W_M + 0.5) * PW, (0.5 - y / PANEL_H_M) * PH + + +class Panel: + def __init__(self, st, on_action): + self.st = st + self.on_action = on_action + self.tab = 0 + self.info = dict(env_name="", env_layout="", env_video=False, env_status="", ipd=0.063) + self.widgets = [] + self.hover = None + self.active = None + self._down = False + self.dirty = True + self._toast = ("", 0.0) + self._layout() + + # ------------------------------------------------------------------ state + def set_info(self, **kw): + if any(self.info.get(k) != v for k, v in kw.items()): + self.info.update(kw) + self.dirty = True + self._layout() + + def toast(self, text, seconds=2.5): + self._toast = (text, time.time() + seconds) + self.dirty = True + + def invalidate(self): + self.dirty = True + self._layout() + + # ------------------------------------------------------------------ layout + def _layout(self): + W = [] + for i, name in enumerate(TABS): + W.append(dict(t="tab", idx=i, label=name, rect=(60 + i * 230, 92, 60 + i * 230 + 210, 148))) + + y = 190 + row = 84 + + def slider(key, label, lo, hi, fmt): + nonlocal y + W.append(dict(t="slider", key=key, label=label, lo=lo, hi=hi, fmt=fmt, + rect=(500, y - 28, 1010, y + 28), track=(520, 990), y=y)) + y += row + + def toggle(key, label): + nonlocal y + W.append(dict(t="toggle", key=key, label=label, rect=(500, y - 28, 640, y + 28), y=y)) + y += row + + def button(action, label, x0=60, x1=480, big=False): + W.append(dict(t="button", action=action, label=label, rect=(x0, y - 30, x1, y + 30), y=y)) + return y + + if self.tab == 0: + slider("depth_cm", "3D depth", 0.0, 30.0, "{:.1f} cm") + slider("conv", "Convergence", 0.0, 1.0, "{:.2f}") + toggle("stereo3d", "3D enabled") + toggle("swap", "Swap eyes") + toggle("debug", "Show depth map") + elif self.tab == 1: + slider("fov", "Screen size", 20.0, 180.0, "{:.0f}\u00b0") + slider("distance", "Distance", 1.0, 10.0, "{:.1f} m") + slider("height", "Height", -1.5, 1.5, "{:+.2f} m") + slider("sharpen", "Text sharpening", 0.0, 1.0, "{:.2f}") + button("recenter", "Recenter screen", 500, 900) + y += row + else: + W.append(dict(t="button", action="env_prev", label="\u25c0", rect=(60, y - 30, 160, y + 30), y=y)) + W.append(dict(t="button", action="env_next", label="\u25b6", rect=(1120, y - 30, 1220, y + 30), y=y)) + y += row + W.append(dict(t="button", action="env_layout", label="Layout: " + (self.info["env_layout"] or "-"), + rect=(60, y - 30, 600, y + 30), y=y)) + y += row + slider("env_brightness", "Brightness", 0.0, 1.5, "{:.2f}") + slider("env_yaw", "Rotation", -180.0, 180.0, "{:+.0f}\u00b0") + if self.info["env_video"]: + toggle("env_play", "Play video") + + fy = 742 + for i, (label, action) in enumerate((("Save", "save"), ("Reset", "reset"), ("Close", "close"))): + W.append(dict(t="button", action=action, label=label, + rect=(60 + i * 240, fy - 32, 60 + i * 240 + 210, fy + 32), y=fy)) + self.widgets = W + self.dirty = True + + # ------------------------------------------------------------------ interaction + def _widget_at(self, px, py): + for w in self.widgets: + x0, y0, x1, y1 = w["rect"] + if x0 <= px <= x1 and y0 <= py <= y1: + return w + return None + + def _set_slider(self, w, px): + t0, t1 = w["track"] + t = min(1.0, max(0.0, (px - t0) / (t1 - t0))) + self.st[w["key"]] = w["lo"] + t * (w["hi"] - w["lo"]) + self.dirty = True + + def pointer(self, px, py, down): + """Feed the pointer position in panel pixels (None = not pointing at the panel).""" + press = down and not self._down + self._down = down + if px is None: + if self.hover is not None: + self.hover = None + self.dirty = True + if not down: + self.active = None + return + w = self._widget_at(px, py) + if w is not self.hover: + self.hover = w + self.dirty = True + if not down: + self.active = None + return + if self.active is not None: + self._set_slider(self.active, px) + return + if not press or w is None: + return + if w["t"] == "slider": + self.active = w + self._set_slider(w, px) + elif w["t"] == "toggle": + self.st[w["key"]] = not self.st[w["key"]] + self.dirty = True + self.on_action("toggled:" + w["key"]) + elif w["t"] == "button": + self.on_action(w["action"]) + elif w["t"] == "tab": + self.tab = w["idx"] + self.hover = None + self._layout() + + # ------------------------------------------------------------------ drawing + def render_if_dirty(self): + if self._toast[0] and time.time() > self._toast[1]: + self._toast = ("", 0.0) + self.dirty = True + if not self.dirty: + return None + self.dirty = False + return self.render() + + def render(self): + img = Image.new("RGBA", (PW, PH), (0, 0, 0, 0)) + d = ImageDraw.Draw(img) + d.rounded_rectangle((0, 0, PW - 1, PH - 1), radius=40, fill=COL_BG) + d.text((60, 46), "Depth 3D", font=font(44, True), fill=COL_TEXT, anchor="lm") + d.text((PW - 60, 46), "trigger: click / drag", font=font(24), fill=COL_DIM, anchor="rm") + + f_label, f_val = font(32), font(32, True) + for w in self.widgets: + hov = w is self.hover or w is self.active + x0, y0, x1, y1 = w["rect"] + if w["t"] == "tab": + sel = w["idx"] == self.tab + d.rounded_rectangle(w["rect"], radius=18, + fill=COL_ACCENT if sel else (COL_HOVER if hov else COL_WIDGET)) + d.text(((x0 + x1) / 2, (y0 + y1) / 2), w["label"], font=font(30, True), + fill=COL_TEXT, anchor="mm") + elif w["t"] == "slider": + d.text((60, w["y"]), w["label"], font=f_label, fill=COL_TEXT, anchor="lm") + t0, t1 = w["track"] + val = float(self.st[w["key"]]) + frac = min(1.0, max(0.0, (val - w["lo"]) / (w["hi"] - w["lo"]))) + kx = t0 + frac * (t1 - t0) + d.rounded_rectangle((t0, w["y"] - 6, t1, w["y"] + 6), radius=6, fill=COL_WIDGET) + d.rounded_rectangle((t0, w["y"] - 6, kx, w["y"] + 6), radius=6, fill=COL_ACCENT) + d.ellipse((kx - 20, w["y"] - 20, kx + 20, w["y"] + 20), + fill=(255, 255, 255, 255) if hov else COL_TEXT) + d.text((1040, w["y"]), w["fmt"].format(val), font=f_val, fill=COL_TEXT, anchor="lm") + elif w["t"] == "toggle": + d.text((60, w["y"]), w["label"], font=f_label, fill=COL_TEXT, anchor="lm") + on = bool(self.st[w["key"]]) + d.rounded_rectangle(w["rect"], radius=28, + fill=COL_ACCENT if on else (COL_HOVER if hov else COL_WIDGET)) + kx = x1 - 28 if on else x0 + 28 + d.ellipse((kx - 22, w["y"] - 22, kx + 22, w["y"] + 22), fill=COL_TEXT) + elif w["t"] == "button": + d.rounded_rectangle(w["rect"], radius=20, fill=COL_HOVER if hov else COL_WIDGET) + d.text(((x0 + x1) / 2, (y0 + y1) / 2), w["label"], font=font(30, True), + fill=COL_TEXT, anchor="mm") + + if self.tab == 0: + far_cm = self.st["depth_cm"] * self.st["conv"] + if far_cm / 100.0 > self.info["ipd"]: + d.text((60, 640), "Far parts exceed your eye spacing - lower depth or convergence.", + font=font(26), fill=COL_WARN, anchor="lm") + else: + d.text((60, 640), "Tip: keep depth x convergence under about 6 for comfort.", + font=font(26), fill=COL_DIM, anchor="lm") + elif self.tab == 2: + name = self.info["env_name"] or "None (black void)" + d.text((640, 190), name, font=font(32, True), fill=COL_TEXT, anchor="mm") + status = self.info["env_status"] + if status: + d.text((60, 640), status, font=font(26), fill=COL_WARN, anchor="lm") + else: + d.text((60, 640), "Put 360/180 images or videos in the 'environments' folder.", + font=font(26), fill=COL_DIM, anchor="lm") + + if self._toast[0]: + d.text((PW - 60, 742), self._toast[0], font=font(30, True), fill=COL_ACCENT, anchor="rm") + return img.tobytes()