Show real controller models alongside the laser pointers

project.godot

@@ -15,8 +15,8 @@ compatibility/default_parent_skeleton_in_mesh_instance_3d=true
 [application]
 
 config/name="Jellyframe"
-config/version="0.1.1"
 config/description="A Jellyfin cinema for standalone VR."
+config/version="0.1.1"
 run/main_scene="res://scenes/main.tscn"
 config/use_custom_user_dir=true
 config/features=PackedStringArray("4.7", "GL Compatibility")
@@ -39,8 +39,6 @@ textures/vram_compression/import_etc2_astc=true
 environment/defaults/default_clear_color=Color(0.018, 0.029, 0.046, 1)
 renderer/rendering_method.windows="mobile"
 renderer/rendering_method.linux="mobile"
-rendering_device/driver.windows="vulkan"
-rendering_device/driver.linuxbsd="vulkan"
 rendering_device/fallback_to_d3d12.windows=false
 rendering_device/fallback_to_opengl3.windows=true
 rendering_device/fallback_to_opengl3.linux=true
@@ -54,5 +52,6 @@ rendering_device/staging_buffer/texture_upload_region_size_px.linux=512
 [xr]
 
 openxr/enabled=true
+openxr/extensions/render_model=true
 shaders/enabled=true
 openxr/extensions/meta/passthrough=true

scripts/ui/player_loading.gd

@@ -33,9 +33,9 @@ func _ready() -> void:
 	var material := StandardMaterial3D.new()
 	material.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
 	material.transparency = BaseMaterial3D.TRANSPARENCY_ALPHA
-	material.no_depth_test = true
 	material.render_priority = 1
 	material.albedo_texture = viewport.get_texture()
+	VrOverlayMaterial.apply_depth(material)
 	material_override = material
 	var panel := PanelContainer.new()
 	panel.theme = Widgets.theme()

scripts/ui/player_overlay.gd

@@ -25,6 +25,7 @@ func _ready() -> void:
 	material.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
 	material.transparency = BaseMaterial3D.TRANSPARENCY_ALPHA
 	material.albedo_texture = viewport.get_texture()
+	VrOverlayMaterial.apply_depth(material)
 	material_override = material
 	var panel := PanelContainer.new()
 	panel.theme = Widgets.theme()

scripts/vr/overlay_material.gd

@@ -0,0 +1,10 @@
+class_name VrOverlayMaterial
+extends RefCounted
+
+const DEPTH_SCALE := 0.01
+
+static func apply_depth(material: BaseMaterial3D) -> void:
+	# Menus and controllers share a depth range in front of the room.
+	material.use_z_clip_scale = true
+	material.z_clip_scale = DEPTH_SCALE
+	material.no_depth_test = false

scripts/vr/overlay_material.gd.uid

@@ -0,0 +1 @@
+uid://bf207wru26usd

scripts/vr/pointer.gd

@@ -50,7 +50,7 @@ func _process(delta: float) -> void:
 		if is_instance_valid(active_pointer) and active_pointer != self:
 			active_pointer._cancel()
 		active_pointer = self
-	if (is_instance_valid(active_pointer) and active_pointer != self) or not get_is_active() or not panel.is_visible_in_tree():
+	if not is_visible_in_tree() or (is_instance_valid(active_pointer) and active_pointer != self) or not get_is_active() or not panel.is_visible_in_tree():
 		_cancel()
 		return
 	var inverse := panel.global_transform.affine_inverse()

scripts/vr/world.gd

@@ -16,6 +16,7 @@ var passthrough := PassthroughEnvironment.new()
 var player_overlay: PlayerOverlay
 var player_loading: PlayerLoading
 var pointers: Array[LaserPointer] = []
+var controller_models: OpenXRRenderModelManager
 
 func setup() -> void:
 	var xr := XRServer.find_interface("OpenXR")
@@ -53,8 +54,8 @@ func setup() -> void:
 	ui_material.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
 	ui_material.albedo_texture = viewport.get_texture()
 	ui_material.transparency = BaseMaterial3D.TRANSPARENCY_ALPHA
-	ui_material.no_depth_test = true
 	ui_material.render_priority = 100
+	VrOverlayMaterial.apply_depth(ui_material)
 	panel.material_override = ui_material
 	add_child(panel)
 	screen = MeshInstance3D.new()
@@ -74,6 +75,9 @@ func setup() -> void:
 	screen.add_child(player_loading)
 	_build_theatre()
 	if immersive:
+		controller_models = OpenXRRenderModelManager.new()
+		controller_models.render_model_added.connect(_prepare_controller_model)
+		origin.add_child(controller_models)
 		for hand in ["left_hand", "right_hand"]:
 			var pointer := LaserPointer.new()
 			pointer.tracker = hand
@@ -101,7 +105,13 @@ func _process(_delta: float) -> void:
 		panorama.visible = show_panorama
 		apply_environment()
 	if show_panorama: panorama.global_position = camera.global_position
+	var loading_menu_visible := player_loading.is_visible_in_tree() and player_loading.cancel_button.visible
+	var menu_visible := panel.is_visible_in_tree() or player_overlay.is_visible_in_tree() or loading_menu_visible
+	var controllers_visible := not screen.visible or menu_visible
+	if controller_models != null:
+		controller_models.visible = controllers_visible
 	for pointer in pointers:
+		pointer.visible = controllers_visible
 		if player_loading.is_visible_in_tree() and player_loading.cancel_button.visible:
 			pointer.set_target(player_loading, player_loading.viewport, player_loading.panel_size)
 		elif player_overlay.is_visible_in_tree():
@@ -109,6 +119,18 @@ func _process(_delta: float) -> void:
 		else:
 			pointer.set_target(panel, viewport, Vector2(4.4, 2.75))
 
+func _prepare_controller_model(node: Node) -> void:
+	if node is MeshInstance3D and node.mesh != null:
+		for surface in range(node.mesh.get_surface_count()):
+			var source := node.get_active_material(surface) as BaseMaterial3D
+			if source == null: continue
+			var unshaded := source.duplicate() as BaseMaterial3D
+			unshaded.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
+			VrOverlayMaterial.apply_depth(unshaded)
+			node.set_surface_override_material(surface, unshaded)
+	for child in node.get_children():
+		_prepare_controller_model(child)
+
 func _build_theatre() -> void:
 	theatre = TheatreEnvironment.new()
 	add_child(theatre)

tests/test_controller_depth.gd

@@ -0,0 +1,70 @@
+extends SceneTree
+
+var failures := 0
+
+func _initialize() -> void:
+	call_deferred("run_checks")
+
+func run_checks() -> void:
+	if DisplayServer.get_name() == "headless":
+		push_error("Controller depth checks require a rendering window.")
+		quit(1)
+		return
+	var app: Node = load("res://scenes/main.tscn").instantiate()
+	root.add_child(app)
+	app.set_process(false)
+	var view := SubViewport.new()
+	view.size = Vector2i(64, 64)
+	view.own_world_3d = true
+	view.render_target_update_mode = SubViewport.UPDATE_ALWAYS
+	root.add_child(view)
+	view.add_child(Camera3D.new())
+	var room := MeshInstance3D.new()
+	room.mesh = QuadMesh.new()
+	room.position.z = -0.4
+	var blue := StandardMaterial3D.new()
+	blue.shading_mode = BaseMaterial3D.SHADING_MODE_UNSHADED
+	blue.albedo_color = Color.BLUE
+	room.material_override = blue
+	view.add_child(room)
+	var panel := MeshInstance3D.new()
+	panel.mesh = QuadMesh.new()
+	panel.position.z = -2.0
+	view.add_child(panel)
+	var controller := MeshInstance3D.new()
+	controller.mesh = BoxMesh.new()
+	controller.scale = Vector3.ONE * 0.2
+	var red := StandardMaterial3D.new()
+	red.albedo_color = Color.RED
+	controller.mesh.material = red
+	app.world._prepare_controller_model(controller)
+	view.add_child(controller)
+	var panels := [app.world.panel, app.world.player_loading, app.player_overlay]
+	for source in panels:
+		var material: StandardMaterial3D = source.material_override.duplicate()
+		material.albedo_texture = null
+		material.albedo_color = Color.GREEN
+		panel.material_override = material
+		for room_visible in [false, true]:
+			room.visible = room_visible
+			for depth in [-1.0, -3.0]:
+				controller.position.z = depth
+				await process_frame
+				await RenderingServer.frame_post_draw
+				var pixels := view.get_texture().get_image()
+				var center := pixels.get_pixel(32, 32)
+				var expected := Color.RED if depth == -1.0 else Color.GREEN
+				check(center.is_equal_approx(expected), "Controller depth is preserved with room visibility %s at %s: %s" % [room_visible, depth, center])
+				check(pixels.get_pixel(23, 32).is_equal_approx(Color.GREEN), "The menu draws over closer room geometry")
+				if room_visible:
+					check(pixels.get_pixel(1, 1).is_equal_approx(Color.BLUE), "Room geometry outside the menu remains visible")
+	app.queue_free()
+	view.queue_free()
+	await process_frame
+	print("Rendered controller depth checks: ", "PASS" if failures == 0 else str(failures) + " failures")
+	quit(1 if failures else 0)
+
+func check(condition: bool, message: String) -> void:
+	if not condition:
+		failures += 1
+		push_error(message)

tests/test_controller_depth.gd.uid

@@ -0,0 +1 @@
+uid://cn6nx2tgy8qbx

tests/test_controller_visibility.gd

@@ -0,0 +1,76 @@
+extends SceneTree
+
+func _initialize() -> void:
+	call_deferred("run_checks")
+
+func run_checks() -> void:
+	var app: Node = load("res://scenes/main.tscn").instantiate()
+	root.add_child(app)
+	app.set_process(false)
+	var world: ViewingWorld = app.world
+	check_controller_materials(world)
+	var origin := world.camera.get_parent()
+	var pointer := LaserPointer.new()
+	pointer.panel = world.panel
+	pointer.viewport = world.viewport
+	origin.add_child(pointer)
+	world.pointers.append(pointer)
+	assert(ProjectSettings.get_setting("xr/openxr/extensions/render_model"), "OpenXR render models are enabled")
+	world._process(0.0)
+	assert(pointer.visible, "Controllers are visible in the library")
+	app.playing = true
+	app.screen.show()
+	app._show_playback()
+	app._video_prepared()
+	app._player_action("hide")
+	world._process(0.0)
+	assert(not pointer.visible, "Video without menus hides the laser pointer")
+	assert(pointer.is_processing(), "Hidden controllers keep processing input")
+	app._show_playback()
+	world._process(0.0)
+	assert(pointer.visible, "Opening playback controls restores controllers")
+	app._screen_settings()
+	world.panel.show()
+	world._process(0.0)
+	assert(pointer.visible, "Settings keep controllers visible during playback")
+	app._show_playback()
+	app._player_action("hide")
+	world.player_loading.show_buffering()
+	world._process(0.0)
+	assert(not pointer.visible, "Buffering without buttons keeps controllers hidden")
+	world.player_loading.show_countdown("Next episode", 5)
+	world._process(0.0)
+	assert(pointer.visible, "Autoplay cancellation keeps controllers visible")
+	world.player_loading.hide()
+	world.set_panoramic(true)
+	world._process(0.0)
+	assert(not pointer.visible, "Panoramic playback also hides controllers")
+	app.playing = false
+	app.screen.hide()
+	world._process(0.0)
+	assert(pointer.visible, "Stopping playback restores controllers")
+	app.queue_free()
+	await process_frame
+	print("Controller visibility checks: PASS")
+	quit()
+
+func check_controller_materials(world: ViewingWorld) -> void:
+	var model := Node3D.new()
+	var component := Node3D.new()
+	model.add_child(component)
+	var mesh := MeshInstance3D.new()
+	mesh.mesh = BoxMesh.new()
+	var source := StandardMaterial3D.new()
+	source.albedo_color = Color("b7c4d6")
+	source.albedo_texture = GradientTexture2D.new()
+	source.metallic = 0.5
+	mesh.mesh.material = source
+	component.add_child(mesh)
+	world._prepare_controller_model(model)
+	var result := mesh.get_active_material(0) as BaseMaterial3D
+	assert(result.shading_mode == BaseMaterial3D.SHADING_MODE_UNSHADED, "Nested controller meshes do not require scene lighting")
+	assert(result.albedo_color == source.albedo_color and result.albedo_texture == source.albedo_texture, "Controller colors and textures are preserved")
+	assert(result.metallic == source.metallic, "Other controller material properties are preserved")
+	assert(result.use_z_clip_scale and is_equal_approx(result.z_clip_scale, VrOverlayMaterial.DEPTH_SCALE), "Controllers share the menu depth range")
+	assert(source.shading_mode == BaseMaterial3D.SHADING_MODE_PER_PIXEL, "Shared source materials remain unchanged")
+	model.free()

tests/test_controller_visibility.gd.uid

@@ -0,0 +1 @@
+uid://lndphkh6u8to

tests/test_player_loading.gd

@@ -32,7 +32,7 @@ func run_checks() -> void:
 	assert(indicator.label.text == "Buffering video...", "Buffering identifies why playback is waiting")
 	assert(indicator.viewport.render_target_update_mode == SubViewport.UPDATE_ALWAYS, "The visible popup renders independently of video frames")
 	var popup_material: StandardMaterial3D = indicator.material_override
-	assert(popup_material.no_depth_test and popup_material.render_priority > 0, "The loading popup draws over video and playback controls")
+	assert(not popup_material.no_depth_test and popup_material.render_priority > 0, "The loading popup respects closer geometry while drawing over playback controls")
 	app.screen.hide()
 	assert(indicator.viewport.render_target_update_mode == SubViewport.UPDATE_DISABLED, "Hiding the video also stops popup rendering")
 	app.screen.show()

tests/test_recenter.gd

@@ -51,7 +51,7 @@ func run_checks() -> void:
 	app._screen_settings()
 	assert(app.panel.visible and not app.player_overlay.visible, "Settings open in the main window")
 	var menu_material: StandardMaterial3D = app.panel.material_override
-	assert(menu_material.no_depth_test and menu_material.render_priority > 0, "Main-window pages draw over a nearer movie screen")
+	assert(not menu_material.no_depth_test, "Main-window pages respect controller depth")
 	app._show_playback()
 	assert(not app.panel.visible and app.player_overlay.visible, "Returning from settings restores the overlay")
 	var before_reset: Transform3D = app.screen.global_transform
@@ -64,8 +64,19 @@ func run_checks() -> void:
 	assert(app.screen.global_basis.is_equal_approx(app.camera.global_basis), "Headset recenter uses the current viewing direction")
 	assert(app.panel.global_position.is_equal_approx(app.camera.global_position - app.camera.global_basis.z * 2.7), "Headset recenter also moves the library menu")
 	assert(not app.panel.visible and app.player_overlay.visible, "Headset recenter preserves menu visibility")
+	check_menu_placement(app)
 	app.playing = false
 	root.use_xr = false
 	if added_interface: XRServer.remove_interface(xr)
 	print("Startup recenter checks: PASS")
 	quit()
+
+func check_menu_placement(app: Node) -> void:
+	for environment in ["Theatre", "Cinema", "Void"]:
+		app.settings.environment = environment
+		app._change_environment()
+		for pitch in [-70.0, -30.0, 0.0]:
+			app.camera.rotation = Vector3(deg_to_rad(pitch), 0.4, 0.3)
+			app._recenter()
+			assert(app.panel.global_basis.is_equal_approx(app.camera.global_basis), "Menu recenter preserves the viewing direction")
+			assert(app.panel.global_position.is_equal_approx(app.camera.global_position - app.camera.global_basis.z * 2.7), "Room geometry does not shift the menu position")