diff --git a/README.md b/README.md index 924533f..1c06c4a 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,16 @@ MOCKTAIL_VR_RUNTIME=alvr MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan ./build/mocktai If you previously set `XR_RUNTIME_JSON`, unset it before switching runtimes. Use `--no-vr` for desktop mode. +### Typing in VR + +When a text box takes focus (chat, search), a keyboard opens in front of you, +below eye level. It is a full US layout without the function row or +Ctrl/Alt. Point at a key and pull the trigger. While you point at the keyboard, +the trigger types and does not click the game behind it. Hold Backspace to +delete repeatedly. Shift applies to the next key and selects its upper symbol; +Caps Lock stays on until you tap it again. Close dismisses the keyboard and +leaves the text box. Set `MOCKTAIL_VR_KEYBOARD=0` to turn the keyboard off. + ## Known limitations Nightcap does not currently work with `hardened_malloc`. Using diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index 783f35d..5ca55d3 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -11,11 +11,18 @@ add_library(mocktail_vr STATIC src/vr/vr_perf.cc src/vr/vr_mirror.cc src/vr/xr_controller.cc - src/vr/roblox_vr_device_bridge.cc) + src/vr/roblox_vr_device_bridge.cc + src/vr/virtual_keyboard.cc + src/vr/virtual_keyboard_raster.cc) add_library(Mocktail::VR ALIAS mocktail_vr) target_include_directories(mocktail_vr PUBLIC "${CMAKE_SOURCE_DIR}/include") target_include_directories(mocktail_vr PRIVATE "${CMAKE_SOURCE_DIR}/src") target_link_libraries(mocktail_vr PUBLIC Vulkan::Headers PRIVATE nlohmann_json::nlohmann_json) +# The in-headset keyboard draws its labels with the same SDL3_ttf/fontconfig +# stack as the TextBox overlay. MocktailInput/MocktailAudio find these +# packages later in the same directory scope; links resolve at generate time. +target_link_libraries(mocktail_vr PRIVATE SDL3_ttf::SDL3_ttf SDL3::SDL3 + PkgConfig::FONTCONFIG) # shm_open for the desktop eye mirror lives in librt on glibc. find_library(MOCKTAIL_RT_LIBRARY rt) if(MOCKTAIL_RT_LIBRARY) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 3a5f294..c044892 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -10,7 +10,10 @@ #include #include +#include + #include "mocktail/status.h" +#include "mocktail/vr/virtual_keyboard.h" #include "mocktail/vr/vr_mirror.h" #include "mocktail/vr/xr_actions.h" #include "mocktail/vr/xr_controller.h" @@ -308,6 +311,14 @@ class OpenXrBackend final { // readbacks) into one command buffer, submits once and waits on one fence. // Replaces the previous per-eye submit+fence pair. bool CopyEyesIntoSwapchains(VkQueue queue); + // In-headset keyboard: show/hide from the host's TextBox request, pointer + // input, and trigger masking. Runs inside the controller sync. + void UpdateVirtualKeyboardLocked(); + // Rasterizes and uploads the panel when it changed. Returns true when the + // keyboard swapchain holds a released image ready for a quad layer. + bool PrepareVirtualKeyboardLayerLocked(VkQueue queue); + bool UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image); + void DestroyVirtualKeyboardResources(); bool CaptureEyeEvidence(VkQueue queue, int eye); ScriptedPoseSample ComputeScriptedPose(std::uint64_t frame) const; void EnsureMirrorForExtent(std::uint32_t width, std::uint32_t height); @@ -337,6 +348,21 @@ class OpenXrBackend final { std::uint32_t xr_eye_width_[2] = {}; std::uint32_t xr_eye_height_[2] = {}; EyeSwapchain eye_swapchains_[2]; + VirtualKeyboard keyboard_; + std::unique_ptr keyboard_raster_; + EyeSwapchain keyboard_swapchain_; + bool keyboard_swapchain_failed_ = false; + bool keyboard_image_ready_ = false; + std::uint64_t keyboard_uploaded_revision_ = 0; + // A new showKeyboard request may open the panel; Close disarms it until + // Roblox drops that request. + bool keyboard_armed_ = true; + pose::Pose keyboard_head_{}; + bool keyboard_head_valid_ = false; + std::vector keyboard_pixels_; + VkBuffer keyboard_staging_ = VK_NULL_HANDLE; + VkDeviceMemory keyboard_staging_memory_ = VK_NULL_HANDLE; + void* keyboard_staging_mapped_ = nullptr; int xr_blend_mode_ = 0; // Runtime name captured at xrCreateInstance so the bind-time diagnostics line // can report it without re-entering runtime_name() under the held mutex. diff --git a/include/mocktail/vr/virtual_keyboard.h b/include/mocktail/vr/virtual_keyboard.h new file mode 100644 index 0000000..a9cb230 --- /dev/null +++ b/include/mocktail/vr/virtual_keyboard.h @@ -0,0 +1,188 @@ +#ifndef MOCKTAIL_VR_VIRTUAL_KEYBOARD_H_ +#define MOCKTAIL_VR_VIRTUAL_KEYBOARD_H_ + +#include +#include +#include + +#include "mocktail/vr/vr_pose_math.h" + +namespace mocktail::vr { + +// Host side of the in-headset keyboard. The window module owns Roblox's +// TextBox focus and the SDL event path; main wires these callbacks so the VR +// library never links the window library. Every callback may be called from +// the render thread and must be thread-safe. +struct VirtualKeyboardHost { + void* context = nullptr; + // True while Roblox has asked for an on-screen keyboard (showKeyboard). + bool (*input_requested)(void* context) = nullptr; + // Commits UTF-8 text exactly like an SDL_EVENT_TEXT_INPUT. + void (*commit_text)(void* context, const char* utf8) = nullptr; + // Presses and releases one SDL scancode exactly like a physical key. + void (*press_key)(void* context, std::uint32_t scancode) = nullptr; +}; + +// Installs the process-wide host. Passing nullptr clears it. +void SetVirtualKeyboardHost(const VirtualKeyboardHost* host); +VirtualKeyboardHost CurrentVirtualKeyboardHost(); + +// The focused TextBox's current contents, shown in the panel's text field. +// The desktop draws TextBox text as a window overlay that never reaches the +// eye images, so the headset would otherwise show an empty box. Safe from any +// thread; the text is sensitive and is never logged. +struct VirtualKeyboardText { + std::string utf8; + int cursor_utf16 = 0; + bool password = false; + std::uint64_t revision = 0; +}; +void SetVirtualKeyboardText(const char* utf8, std::size_t size, + int cursor_utf16, bool password); +void ClearVirtualKeyboardText(); +VirtualKeyboardText CurrentVirtualKeyboardText(); + +enum class VirtualKeyAction { + kText, // commits text, or shift_text under Shift/Caps Lock + kBackspace, + kEnter, + kTab, + kShift, // one-shot + kCapsLock, + kSpace, + kClose, // Escape: leaves the TextBox and hides the panel +}; + +struct VirtualKey { + std::string label; + std::string text; // committed text for kText + std::string shift_text; // committed under Shift (Caps Lock for letters) + VirtualKeyAction action = VirtualKeyAction::kText; + // Panel UV rectangle, origin top-left, both axes 0..1. + float u0 = 0.f, v0 = 0.f, u1 = 0.f, v1 = 0.f; +}; + +// One controller's pointing state for a frame. Positions are metres in the +// OpenXR LOCAL space; the ray points along the aim pose's -Z axis. +struct VirtualKeyboardPointer { + bool valid = false; + pose::Pose aim{}; + float trigger = 0.f; // 0..1 +}; + +// What the keyboard asks the host to do after an Update. +struct VirtualKeyboardOutput { + std::vector text; // committed in order + std::vector scancodes; // pressed in order, after text + bool closed = false; // user pressed Close +}; + +// Ray/quad intersection. The quad is centred on `quad` with its face along +// +Z, `width` x `height` metres. Returns false when the ray misses or points +// away. u/v are panel UVs (origin top-left). +bool IntersectVirtualKeyboardPanel(const pose::Pose& quad, float width, + float height, const pose::Pose& aim, + float* u, float* v, float* distance); + +// Pure keyboard state machine: layout, paging, shift, hover, trigger edges and +// backspace auto-repeat. No OpenXR, Vulkan or SDL types, so it is unit-tested +// directly. +class VirtualKeyboard { + public: + // A text field over 15 x 5 normal-key units, 4 cm per unit. + static constexpr float kPanelWidthMetres = 0.60f; + static constexpr float kPanelHeightMetres = 0.248f; + // Texture aspect matches the panel so keys are not stretched. + static constexpr int kTextureWidth = 1500; + static constexpr int kTextureHeight = 620; + // Top share of the panel holding the text field; keys fill the rest. + static constexpr float kFieldFraction = 120.f / 620.f; + + VirtualKeyboard(); + + // Opens the panel in front of `head` (LOCAL space), below eye level and + // tilted up toward the viewer. Resets Shift and hover; keeps Caps Lock. + void Show(const pose::Pose& head); + void Hide(); + bool visible() const { return visible_; } + const pose::Pose& panel_pose() const { return panel_; } + + // Text shown in the field. Bumps the revision only when it changed. + void SetText(const VirtualKeyboardText& text); + const std::string& text() const { return text_.utf8; } + int text_cursor_utf16() const { return text_.cursor_utf16; } + bool text_password() const { return text_.password; } + + // Processes one frame of both pointers. now_ns is monotonic. + VirtualKeyboardOutput Update(std::uint64_t now_ns, + const VirtualKeyboardPointer (&pointers)[2]); + + // True when this hand's trigger belongs to the keyboard this frame (aimed + // at the panel, or a press that began on it is still held). The backend + // withholds that trigger from Roblox so a key press is not also a click. + bool CapturesTrigger(int hand) const; + + // Current layout, with Shift/Caps Lock applied to the labels. + const std::vector& keys() const; + // -1 when the hand is not over a key. + int hovered_key(int hand) const { return hands_[hand & 1].hovered; } + int pressed_key(int hand) const { return hands_[hand & 1].pressed; } + // Pointer position on the panel, when the hand is over it. + bool cursor(int hand, float* u, float* v) const; + bool shifted() const { return shift_; } + bool caps_lock() const { return caps_lock_; } + + // Advances whenever anything visible changes; the backend re-rasterizes + // and re-uploads only when this moves. + std::uint64_t revision() const { return revision_; } + + private: + struct Hand { + int hovered = -1; + int pressed = -1; // key under an active press, -1 otherwise + bool trigger_down = false; + bool captured = false; + bool over_panel = false; + float u = 0.f, v = 0.f; + std::uint64_t repeat_at_ns = 0; + }; + + void RebuildLayout(); + int KeyAt(float u, float v) const; + std::string Output(const VirtualKey& key) const; + void Activate(const VirtualKey& key, VirtualKeyboardOutput* output); + void Touch() { ++revision_; } + + bool visible_ = false; + bool shift_ = false; + bool caps_lock_ = false; + pose::Pose panel_{}; + VirtualKeyboardText text_; + Hand hands_[2]; + std::vector keys_; + std::uint64_t revision_ = 1; +}; + +// Draws the keyboard into a tightly packed RGBA8 buffer +// (VirtualKeyboard::kTextureWidth x kTextureHeight, sRGB-encoded, opaque). +// Glyphs come from SDL3_ttf with fontconfig's sans fallback chain and are +// cached per label, so redraws after the first are plain fills and blits. +class VirtualKeyboardRasterizer { + public: + VirtualKeyboardRasterizer(); + ~VirtualKeyboardRasterizer(); + VirtualKeyboardRasterizer(const VirtualKeyboardRasterizer&) = delete; + VirtualKeyboardRasterizer& operator=(const VirtualKeyboardRasterizer&) = + delete; + + // Returns false when no font could be opened; keys still draw, unlabelled. + bool Draw(const VirtualKeyboard& keyboard, std::vector* rgba); + + private: + struct Impl; + Impl* impl_ = nullptr; +}; + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_VIRTUAL_KEYBOARD_H_ diff --git a/include/runtime/roblox_text_editor.h b/include/runtime/roblox_text_editor.h index a620170..e288bcf 100644 --- a/include/runtime/roblox_text_editor.h +++ b/include/runtime/roblox_text_editor.h @@ -89,6 +89,12 @@ struct RobloxTextDisplaySink { UpdateFn update = nullptr; }; +// Process-wide second observer of the focused TextBox display, called +// synchronously after the surface overlay with the same sensitivity contract. +// The VR keyboard uses it to show the text being typed in the headset, where +// the desktop overlay is not visible. Passing an empty sink clears it. +void SetRobloxTextDisplayMirror(RobloxTextDisplaySink sink); + struct RobloxTextSink { using SyncFn = Status (*)(void* context, const char* utf8, std::size_t size, int32_t cursor_utf16); diff --git a/src/main.cc b/src/main.cc index 5f02d77..4d20ded 100644 --- a/src/main.cc +++ b/src/main.cc @@ -27,6 +27,7 @@ #include "mocktail/vr/openxr_backend.h" #include "mocktail/vr/openxr_probe.h" #include "mocktail/vr/roblox_vr_device_bridge.h" +#include "mocktail/vr/virtual_keyboard.h" #include "runtime/auth_runtime_composition.h" #include "runtime/command_line.h" #include "runtime/crash_report_policy.h" @@ -45,6 +46,7 @@ #include "runtime/roblox_desktop_app_policy.h" #include "runtime/roblox_experience_launch_bridge.h" #include "runtime/roblox_fullscreen_runtime_bridge.h" +#include "runtime/roblox_text_editor.h" #include "runtime/runtime_config_bootstrap.h" #include "runtime/runtime_config_file.h" #include "runtime/runtime_paths.h" @@ -1031,6 +1033,38 @@ int main(int argc, char* argv[]) { "native-stereo diagnostic mode without a runtime.\n"; return EXIT_FAILURE; } + // The in-headset keyboard follows Roblox's TextBox keyboard requests + // and types through the same events as the desktop keyboard. + mocktail::vr::VirtualKeyboardHost keyboard_host; + keyboard_host.input_requested = [](void*) { + return mocktail::window::TextInputRequested(); + }; + keyboard_host.commit_text = [](void*, const char* utf8) { + mocktail::window::QueueVirtualTextInput(utf8); + }; + keyboard_host.press_key = [](void*, std::uint32_t scancode) { + mocktail::window::QueueVirtualKeyPress(scancode); + }; + mocktail::vr::SetVirtualKeyboardHost(&keyboard_host); + // TextBox text is drawn as a desktop window overlay that never reaches + // the eye images; mirror it into the keyboard's text field. + mocktail::runtime::SetRobloxTextDisplayMirror( + {nullptr, [](void*, + const mocktail::runtime::RobloxTextDisplayUpdate& + update) { + if (update.event == + mocktail::runtime::RobloxTextDisplayEvent::kHide) { + mocktail::vr::ClearVirtualKeyboardText(); + return; + } + // RbxKeyboard password input types (see the SDL mapping). + const bool password = update.text_input_type == 5 || + update.text_input_type == 9 || + update.text_input_type == 10; + mocktail::vr::SetVirtualKeyboardText( + update.utf8, update.utf8_size, update.cursor_utf16, + password); + }}); } } } diff --git a/src/runtime/roblox_text_surface_overlay.cc b/src/runtime/roblox_text_surface_overlay.cc index 5982d70..c62530d 100644 --- a/src/runtime/roblox_text_surface_overlay.cc +++ b/src/runtime/roblox_text_surface_overlay.cc @@ -325,6 +325,16 @@ Status RobloxTextSurfaceOverlay::UpdateViewport( return Status::Ok(); } +namespace { +std::mutex g_display_mirror_mutex; +RobloxTextDisplaySink g_display_mirror; +} // namespace + +void SetRobloxTextDisplayMirror(RobloxTextDisplaySink sink) { + std::lock_guard lock(g_display_mirror_mutex); + g_display_mirror = sink; +} + RobloxTextDisplaySink RobloxTextSurfaceOverlay::sink() { return {this, &RobloxTextSurfaceOverlay::UpdateCallback}; } @@ -334,6 +344,10 @@ void RobloxTextSurfaceOverlay::UpdateCallback( if (context != nullptr) { static_cast(context)->ApplyUpdate(update); } + std::lock_guard lock(g_display_mirror_mutex); + if (g_display_mirror.update != nullptr) { + g_display_mirror.update(g_display_mirror.context, update); + } } void RobloxTextSurfaceOverlay::ApplyUpdate( diff --git a/src/vr/gles_transport.cc b/src/vr/gles_transport.cc index 4ed92e4..3731ccc 100644 --- a/src/vr/gles_transport.cc +++ b/src/vr/gles_transport.cc @@ -38,7 +38,13 @@ namespace { PFNGLDELETERENDERBUFFERSPROC) \ X(BlitFramebuffer, \ PFNGLBLITFRAMEBUFFERPROC) \ - X(Flush, PFNGLFLUSHPROC) + X(Flush, PFNGLFLUSHPROC) \ + X(BindTexture, \ + PFNGLBINDTEXTUREPROC) \ + X(TexSubImage2D, \ + PFNGLTEXSUBIMAGE2DPROC)\ + X(PixelStorei, \ + PFNGLPIXELSTOREIPROC) struct Procs { #define DECLARE(name, type) type name = nullptr; GL_PROCS(DECLARE) @@ -391,4 +397,28 @@ bool GlesTransport::Copy(const unsigned textures[2], unsigned width, gl.Flush(); return copied; } + +bool GlesTransport::Upload(unsigned texture, int width, int height, + const void *rgba) { + if (!IsCurrent() || texture == 0 || rgba == nullptr) + return false; + // Restore the guest's bindings and unpack state around the upload. + GLint bound = 0, alignment = 4, row_length = 0, unpack_buffer = 0; + gl.GetIntegerv(GL_TEXTURE_BINDING_2D, &bound); + gl.GetIntegerv(GL_UNPACK_ALIGNMENT, &alignment); + gl.GetIntegerv(GL_UNPACK_ROW_LENGTH, &row_length); + gl.GetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &unpack_buffer); + if (unpack_buffer != 0) + return false; // a bound PBO would reinterpret the pointer as an offset + gl.PixelStorei(GL_UNPACK_ALIGNMENT, 4); + gl.PixelStorei(GL_UNPACK_ROW_LENGTH, 0); + gl.BindTexture(GL_TEXTURE_2D, texture); + gl.TexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_RGBA, + GL_UNSIGNED_BYTE, rgba); + gl.BindTexture(GL_TEXTURE_2D, static_cast(bound)); + gl.PixelStorei(GL_UNPACK_ALIGNMENT, alignment); + gl.PixelStorei(GL_UNPACK_ROW_LENGTH, row_length); + gl.Flush(); + return true; +} } // namespace mocktail::vr diff --git a/src/vr/gles_transport.h b/src/vr/gles_transport.h index 30fc95d..9dfb400 100644 --- a/src/vr/gles_transport.h +++ b/src/vr/gles_transport.h @@ -29,6 +29,8 @@ public: static void *Wrap(const char *name, void *raw); void ResetEyes(); bool Copy(const unsigned textures[2], unsigned width, unsigned height); + // Uploads tightly packed RGBA8 pixels into a runtime swapchain texture. + bool Upload(unsigned texture, int width, int height, const void *rgba); const Eye &eye(int index) const { return eyes_[index]; } void *display() const { return display_; } void *config() const { return config_; } diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index e48d771..6b32f70 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -174,6 +174,7 @@ struct OpenXrBackend::VkProcs { PFN_vkMapMemory MapMemory = nullptr; PFN_vkUnmapMemory UnmapMemory = nullptr; PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer = nullptr; + PFN_vkCmdCopyBufferToImage CmdCopyBufferToImage = nullptr; PFN_vkCreateImageView CreateImageView = nullptr; PFN_vkDestroyImageView DestroyImageView = nullptr; PFN_vkCreateCommandPool CreateCommandPool = nullptr; @@ -698,6 +699,7 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( MOCKTAIL_BACKEND_VK_FN(MapMemory) MOCKTAIL_BACKEND_VK_FN(UnmapMemory) MOCKTAIL_BACKEND_VK_FN(CmdCopyImageToBuffer) + MOCKTAIL_BACKEND_VK_FN(CmdCopyBufferToImage) MOCKTAIL_BACKEND_VK_FN(CreateImageView) MOCKTAIL_BACKEND_VK_FN(DestroyImageView) MOCKTAIL_BACKEND_VK_FN(CreateCommandPool) @@ -1238,6 +1240,7 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi mirror_.DestroyStaging(MirrorProcs()); } mirror_.Close(); + DestroyVirtualKeyboardResources(); DestroyCopyResources(); DestroySwapchains(); // Action pose spaces belong to this session; detach before the session and @@ -2115,8 +2118,12 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { XrInfo( XR_TYPE_COMPOSITION_LAYER_PROJECTION); std::array views{}; - const XrCompositionLayerBaseHeader *layers[1] = { - reinterpret_cast(&projection)}; + XrCompositionLayerQuad keyboard_layer = + XrInfo(XR_TYPE_COMPOSITION_LAYER_QUAD); + const XrCompositionLayerBaseHeader *layers[2] = { + reinterpret_cast(&projection), + reinterpret_cast( + &keyboard_layer)}; const bool eyes_usable = eyes_bound_.load(std::memory_order_acquire) && eyes_[0].valid && eyes_[1].valid; const bool pose_ready = frame_views_valid_ && published_pose_.valid; @@ -2224,6 +2231,24 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { projection.views = views.data(); end.layerCount = 1; end.layers = layers; + if (PrepareVirtualKeyboardLayerLocked(queue)) { + // World-locked panel in LOCAL space, composited over the eyes. + const Pose &panel = keyboard_.panel_pose(); + keyboard_layer.space = static_cast(xr_local_space_); + keyboard_layer.eyeVisibility = XR_EYE_VISIBILITY_BOTH; + keyboard_layer.subImage.swapchain = + static_cast(keyboard_swapchain_.swapchain); + keyboard_layer.subImage.imageRect.extent = { + VirtualKeyboard::kTextureWidth, VirtualKeyboard::kTextureHeight}; + keyboard_layer.pose.position = {panel.position.x, panel.position.y, + panel.position.z}; + keyboard_layer.pose.orientation = { + panel.orientation.x, panel.orientation.y, panel.orientation.z, + panel.orientation.w}; + keyboard_layer.size = {VirtualKeyboard::kPanelWidthMetres, + VirtualKeyboard::kPanelHeightMetres}; + end.layerCount = 2; + } } } CheckXr(instance, xrEndFrame(session, &end), "xrEndFrame"); @@ -2327,6 +2352,12 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { const std::string pose_source = EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime"); Pose head_pose = head_valid ? XrToPose(head.pose) : Pose{}; + if (head_valid) { + // Tracked LOCAL head pose, independent of the scripted test source; + // the keyboard panel opens in front of it. + keyboard_head_ = head_pose; + keyboard_head_valid_ = true; + } if (pose_source == "script") { const ScriptedPoseSample scripted = ComputeScriptedPose(++xr_frame_counter_); @@ -2494,9 +2525,310 @@ void OpenXrBackend::SyncControllersLocked(std::uint64_t predicted_display_time) published_pose_.hands[0] = VrHandPose{}; published_pose_.hands[1] = VrHandPose{}; } + UpdateVirtualKeyboardLocked(); perf::ProcessCollector().Count(perf::Counter::kControllerSyncs); } +void OpenXrBackend::UpdateVirtualKeyboardLocked() { + static const bool enabled = EnvString("MOCKTAIL_VR_KEYBOARD", "1") != "0"; + if (!enabled) { + return; + } + const VirtualKeyboardHost host = CurrentVirtualKeyboardHost(); + const bool requested = + host.input_requested != nullptr && host.input_requested(host.context); + if (!requested) { + keyboard_.Hide(); + keyboard_armed_ = true; + } else if (keyboard_armed_ && keyboard_head_valid_) { + keyboard_.Show(keyboard_head_); + keyboard_armed_ = false; + Log(" [vr-keyboard] shown for focused TextBox\n"); + } + if (!keyboard_.visible()) { + return; + } + { + VirtualKeyboardText text = CurrentVirtualKeyboardText(); + keyboard_.SetText(text); + std::fill(text.utf8.begin(), text.utf8.end(), '\0'); + } + + VirtualKeyboardPointer pointers[2]; + const ControllerSnapshot& snapshot = actions_.snapshot(); + for (int hand = 0; hand < 2; ++hand) { + const VrHandPose& pose = actions_.hand(hand); + const ControllerHandState& state = snapshot.hands[hand]; + pointers[hand].valid = state.connected && pose.aim_valid; + pointers[hand].aim.position = + Vec3{pose.aim_position[0], pose.aim_position[1], pose.aim_position[2]}; + pointers[hand].aim.orientation = + Quat{pose.aim_orientation[0], pose.aim_orientation[1], + pose.aim_orientation[2], pose.aim_orientation[3]}; + pointers[hand].trigger = + state.axes[static_cast(ControllerAxis::kTrigger)]; + } + const VirtualKeyboardOutput output = + keyboard_.Update(perf::NowNs(), pointers); + for (const std::string& text : output.text) { + if (host.commit_text != nullptr) host.commit_text(host.context, text.c_str()); + } + for (const std::uint32_t scancode : output.scancodes) { + if (host.press_key != nullptr) host.press_key(host.context, scancode); + } + // A trigger aimed at the panel types; withhold it from Roblox so it is not + // also a click on whatever UI sits behind the keyboard. + for (int hand = 0; hand < 2; ++hand) { + if (keyboard_.CapturesTrigger(hand)) { + published_pose_.controller_channels[hand * 9 + 3] = 0.f; + } + } +} + +bool OpenXrBackend::PrepareVirtualKeyboardLayerLocked(VkQueue queue) { + if (!keyboard_.visible() || keyboard_swapchain_failed_) { + return false; + } + const auto instance = static_cast(xr_instance_); + try { + EyeSwapchain& slot = keyboard_swapchain_; + if (slot.swapchain == nullptr) { + auto create_info = + XrInfo(XR_TYPE_SWAPCHAIN_CREATE_INFO); + create_info.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | + XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; + // Same format as the eyes, so the panel's sRGB-encoded bytes are + // interpreted exactly like the guest's eye pixels. + create_info.format = + static_cast(eye_swapchains_[0].format); + create_info.sampleCount = 1; + create_info.width = VirtualKeyboard::kTextureWidth; + create_info.height = VirtualKeyboard::kTextureHeight; + create_info.faceCount = 1; + create_info.arraySize = 1; + create_info.mipCount = 1; + XrSwapchain swapchain = XR_NULL_HANDLE; + CheckXr(instance, + xrCreateSwapchain(static_cast(xr_session_), + &create_info, &swapchain), + "xrCreateSwapchain keyboard"); + slot.swapchain = swapchain; + std::uint32_t count = 0; + CheckXr(instance, + xrEnumerateSwapchainImages(swapchain, 0, &count, nullptr), + "xrEnumerateSwapchainImages keyboard count"); + Require(count > 0 && count <= 16, "empty keyboard swapchain"); + if (graphics_api_ == VrGraphicsApi::kVulkan) { + std::vector images(count); + for (auto& image : images) image.type = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR; + CheckXr(instance, + xrEnumerateSwapchainImages( + swapchain, count, &count, + reinterpret_cast( + images.data())), + "xrEnumerateSwapchainImages keyboard"); + for (const auto& image : images) slot.images.push_back(image.image); + } else { + std::vector images(count); + for (auto& image : images) + image.type = XR_TYPE_SWAPCHAIN_IMAGE_OPENGL_ES_KHR; + CheckXr(instance, + xrEnumerateSwapchainImages( + swapchain, count, &count, + reinterpret_cast( + images.data())), + "xrEnumerateSwapchainImages keyboard GLES"); + for (const auto& image : images) slot.gl_images.push_back(image.image); + } + slot.width = VirtualKeyboard::kTextureWidth; + slot.height = VirtualKeyboard::kTextureHeight; + slot.format = eye_swapchains_[0].format; + keyboard_image_ready_ = false; + Log(" [vr-keyboard] panel swapchain %ux%u images=%u\n", slot.width, + slot.height, count); + } + if (keyboard_image_ready_ && + keyboard_uploaded_revision_ == keyboard_.revision()) { + return true; + } + if (!keyboard_raster_) { + keyboard_raster_ = std::make_unique(); + } + (void)keyboard_raster_->Draw(keyboard_, &keyboard_pixels_); + + std::uint32_t index = 0; + auto acquire = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO); + CheckXr(instance, + xrAcquireSwapchainImage(static_cast(slot.swapchain), + &acquire, &index), + "xrAcquireSwapchainImage keyboard"); + auto wait = + XrInfo(XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO); + wait.timeout = 100'000'000; + const XrResult wait_result = + xrWaitSwapchainImage(static_cast(slot.swapchain), &wait); + bool uploaded = false; + if (XR_SUCCEEDED(wait_result) && wait_result != XR_TIMEOUT_EXPIRED) { + if (graphics_api_ == VrGraphicsApi::kVulkan) { + uploaded = index < slot.images.size() && + UploadVirtualKeyboardVulkan(queue, slot.images[index]); + } else { + uploaded = gles_ != nullptr && index < slot.gl_images.size() && + gles_->Upload(slot.gl_images[index], + VirtualKeyboard::kTextureWidth, + VirtualKeyboard::kTextureHeight, + keyboard_pixels_.data()); + } + } + auto release = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO); + (void)xrReleaseSwapchainImage(static_cast(slot.swapchain), + &release); + // A released image is what the quad shows, even if this upload failed + // after an earlier one succeeded. + if (uploaded) { + keyboard_image_ready_ = true; + keyboard_uploaded_revision_ = keyboard_.revision(); + } + return keyboard_image_ready_; + } catch (const std::exception& exception) { + Log(" [vr-keyboard] disabled: %s\n", exception.what()); + keyboard_swapchain_failed_ = true; + return false; + } +} + +bool OpenXrBackend::UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image) { + std::string error; + if (vk_ == nullptr || + !EnsureCopyResources(vk_device_, vk_queue_family_, &error)) { + return false; + } + const VkDeviceSize size = static_cast(keyboard_pixels_.size()); + if (keyboard_staging_ == VK_NULL_HANDLE) { + VkBufferCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + info.size = size; + info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + if (vk_->CreateBuffer(vk_device_, &info, nullptr, &keyboard_staging_) != + VK_SUCCESS) { + keyboard_staging_ = VK_NULL_HANDLE; + return false; + } + VkMemoryRequirements requirements{}; + vk_->GetBufferMemoryRequirements(vk_device_, keyboard_staging_, + &requirements); + VkPhysicalDeviceMemoryProperties properties{}; + vk_->GetPhysicalDeviceMemoryProperties(vk_physical_device_, &properties); + const auto flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + std::uint32_t type = UINT32_MAX; + for (std::uint32_t t = 0; t < properties.memoryTypeCount; ++t) { + if ((requirements.memoryTypeBits & (1u << t)) && + (properties.memoryTypes[t].propertyFlags & flags) == flags) { + type = t; + break; + } + } + VkMemoryAllocateInfo allocation{}; + allocation.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = type; + if (type == UINT32_MAX || + vk_->AllocateMemory(vk_device_, &allocation, nullptr, + &keyboard_staging_memory_) != VK_SUCCESS || + vk_->BindBufferMemory(vk_device_, keyboard_staging_, + keyboard_staging_memory_, 0) != VK_SUCCESS || + vk_->MapMemory(vk_device_, keyboard_staging_memory_, 0, size, 0, + &keyboard_staging_mapped_) != VK_SUCCESS) { + DestroyVirtualKeyboardResources(); + return false; + } + } + std::memcpy(keyboard_staging_mapped_, keyboard_pixels_.data(), + keyboard_pixels_.size()); + + if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) return false; + VkCommandBufferBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) { + return false; + } + VkImageMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.srcAccessMask = 0; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 1, &barrier); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {VirtualKeyboard::kTextureWidth, + VirtualKeyboard::kTextureHeight, 1}; + vk_->CmdCopyBufferToImage(copy_command_, keyboard_staging_, image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, + nullptr, 1, &barrier); + if (vk_->EndCommandBuffer(copy_command_) != VK_SUCCESS || + vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) { + return false; + } + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = ©_command_; + if (SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, &submit, + copy_fence_) != VK_SUCCESS) { + return false; + } + if (vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, + 2'000'000'000ULL) != VK_SUCCESS) { + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); + throw std::runtime_error("keyboard upload fence wait failed"); + } + return true; +} + +void OpenXrBackend::DestroyVirtualKeyboardResources() { + if (vk_ != nullptr && vk_device_ != VK_NULL_HANDLE) { + if (keyboard_staging_mapped_ != nullptr) { + vk_->UnmapMemory(vk_device_, keyboard_staging_memory_); + } + if (keyboard_staging_ != VK_NULL_HANDLE) { + vk_->DestroyBuffer(vk_device_, keyboard_staging_, nullptr); + } + if (keyboard_staging_memory_ != VK_NULL_HANDLE) { + vk_->FreeMemory(vk_device_, keyboard_staging_memory_, nullptr); + } + } + keyboard_staging_mapped_ = nullptr; + keyboard_staging_ = VK_NULL_HANDLE; + keyboard_staging_memory_ = VK_NULL_HANDLE; + EyeSwapchain& slot = keyboard_swapchain_; + slot.images.clear(); + slot.gl_images.clear(); + if (slot.swapchain != nullptr) { + (void)xrDestroySwapchain(static_cast(slot.swapchain)); + slot.swapchain = nullptr; + } + keyboard_image_ready_ = false; + keyboard_uploaded_revision_ = 0; + keyboard_swapchain_failed_ = false; +} + bool OpenXrBackend::EnsureCopyResources(VkDevice device, std::uint32_t queue_family, std::string* error) { diff --git a/src/vr/virtual_keyboard.cc b/src/vr/virtual_keyboard.cc new file mode 100644 index 0000000..83333c8 --- /dev/null +++ b/src/vr/virtual_keyboard.cc @@ -0,0 +1,371 @@ +#include "mocktail/vr/virtual_keyboard.h" + +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { + +using pose::kPi; +using pose::Pose; +using pose::Quat; +using pose::QuatConjugate; +using pose::QuatFromAxisAngle; +using pose::QuatMultiply; +using pose::QuatRotate; +using pose::Vec3; + +// SDL scancodes, kept numeric so this pure module needs no SDL headers. +constexpr std::uint32_t kScancodeReturn = 40; // SDL_SCANCODE_RETURN +constexpr std::uint32_t kScancodeEscape = 41; // SDL_SCANCODE_ESCAPE +constexpr std::uint32_t kScancodeBackspace = 42; // SDL_SCANCODE_BACKSPACE +constexpr std::uint32_t kScancodeTab = 43; // SDL_SCANCODE_TAB + +// Matches ControllerStateBuilder's analog click hysteresis. +constexpr float kTriggerPress = 0.75f; +constexpr float kTriggerRelease = 0.6f; +constexpr std::uint64_t kRepeatDelayNs = 450'000'000ULL; +constexpr std::uint64_t kRepeatIntervalNs = 60'000'000ULL; +// Cursor moves smaller than this (in UV) do not force a redraw. +constexpr float kCursorRedrawStep = 0.004f; + +constexpr float kPanelDistanceMetres = 0.45f; +constexpr float kPanelDropMetres = 0.28f; +constexpr float kPanelTiltRadians = 35.f * kPi / 180.f; + +// US QWERTY without the function row or Ctrl/Alt/Menu, in normal-key units. +// Widths follow the desktop shell OSK (Tab 1.6, Shift 2.5, wide keys fill). +constexpr float kColumns = 15.f; +constexpr int kRows = 5; + +std::mutex g_host_mutex; +VirtualKeyboardHost g_host; +std::mutex g_text_mutex; +VirtualKeyboardText g_text; + +void ScrubText(std::string* text) { + std::fill(text->begin(), text->end(), '\0'); + text->clear(); +} + +struct KeySpec { + const char* label; + const char* text; + const char* shift_text; + VirtualKeyAction action; + float width; +}; + +KeySpec Char(const char* text, const char* shift_text) { + return {text, text, shift_text, VirtualKeyAction::kText, 1.f}; +} +KeySpec Special(const char* label, VirtualKeyAction action, float width) { + return {label, "", "", action, width}; +} + +std::vector> Rows() { + using A = VirtualKeyAction; + return { + {Char("`", "~"), Char("1", "!"), Char("2", "@"), Char("3", "#"), + Char("4", "$"), Char("5", "%"), Char("6", "^"), Char("7", "&"), + Char("8", "*"), Char("9", "("), Char("0", ")"), Char("-", "_"), + Char("=", "+"), Special("Backspace", A::kBackspace, 2.f)}, + {Special("Tab", A::kTab, 1.6f), Char("q", "Q"), Char("w", "W"), + Char("e", "E"), Char("r", "R"), Char("t", "T"), Char("y", "Y"), + Char("u", "U"), Char("i", "I"), Char("o", "O"), Char("p", "P"), + Char("[", "{"), Char("]", "}"), {"\\", "\\", "|", A::kText, 1.4f}}, + {Special("Caps", A::kCapsLock, 2.f), Char("a", "A"), Char("s", "S"), + Char("d", "D"), Char("f", "F"), Char("g", "G"), Char("h", "H"), + Char("j", "J"), Char("k", "K"), Char("l", "L"), Char(";", ":"), + Char("'", "\""), Special("Enter", A::kEnter, 2.f)}, + {Special("Shift", A::kShift, 2.5f), Char("z", "Z"), Char("x", "X"), + Char("c", "C"), Char("v", "V"), Char("b", "B"), Char("n", "N"), + Char("m", "M"), Char(",", "<"), Char(".", ">"), Char("/", "?"), + Special("Shift", A::kShift, 2.5f)}, + {Special("Space", A::kSpace, 12.5f), Special("Close", A::kClose, 2.5f)}, + }; +} + +bool IsLetter(const std::string& text) { + return text.size() == 1 && + std::isalpha(static_cast(text[0])) != 0; +} + +float Dot(const Vec3& a, const Vec3& b) { + return a.x * b.x + a.y * b.y + a.z * b.z; +} +Vec3 Sub(const Vec3& a, const Vec3& b) { + return {a.x - b.x, a.y - b.y, a.z - b.z}; +} + +} // namespace + +void SetVirtualKeyboardHost(const VirtualKeyboardHost* host) { + std::lock_guard lock(g_host_mutex); + g_host = host != nullptr ? *host : VirtualKeyboardHost{}; +} + +VirtualKeyboardHost CurrentVirtualKeyboardHost() { + std::lock_guard lock(g_host_mutex); + return g_host; +} + +void SetVirtualKeyboardText(const char* utf8, std::size_t size, + int cursor_utf16, bool password) { + std::lock_guard lock(g_text_mutex); + ScrubText(&g_text.utf8); + if (utf8 != nullptr) g_text.utf8.assign(utf8, size); + g_text.cursor_utf16 = std::max(0, cursor_utf16); + g_text.password = password; + ++g_text.revision; +} + +void ClearVirtualKeyboardText() { + std::lock_guard lock(g_text_mutex); + ScrubText(&g_text.utf8); + g_text.cursor_utf16 = 0; + g_text.password = false; + ++g_text.revision; +} + +VirtualKeyboardText CurrentVirtualKeyboardText() { + std::lock_guard lock(g_text_mutex); + return g_text; +} + +bool IntersectVirtualKeyboardPanel(const Pose& quad, float width, float height, + const Pose& aim, float* u, float* v, + float* distance) { + const Vec3 normal = QuatRotate(quad.orientation, Vec3{0.f, 0.f, 1.f}); + const Vec3 direction = QuatRotate(aim.orientation, Vec3{0.f, 0.f, -1.f}); + const float denominator = Dot(direction, normal); + // Only the front face takes input; a ray from behind passes through. + if (denominator > -1e-4f) return false; + const float t = Dot(Sub(quad.position, aim.position), normal) / denominator; + if (!(t > 0.f)) return false; + const Vec3 hit{aim.position.x + direction.x * t, + aim.position.y + direction.y * t, + aim.position.z + direction.z * t}; + const Vec3 local = + QuatRotate(QuatConjugate(quad.orientation), Sub(hit, quad.position)); + const float hit_u = local.x / width + 0.5f; + const float hit_v = 0.5f - local.y / height; + if (hit_u < 0.f || hit_u > 1.f || hit_v < 0.f || hit_v > 1.f) return false; + if (u != nullptr) *u = hit_u; + if (v != nullptr) *v = hit_v; + if (distance != nullptr) *distance = t; + return true; +} + +VirtualKeyboard::VirtualKeyboard() { RebuildLayout(); } + +void VirtualKeyboard::RebuildLayout() { + const auto rows = Rows(); + keys_.clear(); + for (int row = 0; row < kRows; ++row) { + float column = 0.f; + for (const KeySpec& spec : rows[row]) { + VirtualKey key; + key.action = spec.action; + key.text = spec.text; + key.shift_text = spec.shift_text; + key.label = spec.action == VirtualKeyAction::kText ? Output(key) + : spec.label; + key.u0 = column / kColumns; + key.u1 = std::min(1.f, (column + spec.width) / kColumns); + constexpr float kField = VirtualKeyboard::kFieldFraction; + key.v0 = kField + (1.f - kField) * static_cast(row) / kRows; + key.v1 = kField + (1.f - kField) * static_cast(row + 1) / kRows; + keys_.push_back(std::move(key)); + column += spec.width; + } + } + for (Hand& hand : hands_) { + hand.hovered = -1; + hand.pressed = -1; + } + Touch(); +} + +std::string VirtualKeyboard::Output(const VirtualKey& key) const { + // Caps Lock only affects letters; Shift inverts it for letters and selects + // the upper symbol on every other key. + if (IsLetter(key.text)) { + return (shift_ != caps_lock_) ? key.shift_text : key.text; + } + return shift_ ? key.shift_text : key.text; +} + +const std::vector& VirtualKeyboard::keys() const { return keys_; } + +void VirtualKeyboard::Show(const Pose& head) { + Vec3 forward = QuatRotate(head.orientation, Vec3{0.f, 0.f, -1.f}); + forward.y = 0.f; + const float length = std::sqrt(forward.x * forward.x + forward.z * forward.z); + if (length < 1e-3f) { + forward = Vec3{0.f, 0.f, -1.f}; + } else { + forward.x /= length; + forward.z /= length; + } + const float yaw = std::atan2(-forward.x, -forward.z); + panel_.orientation = + QuatMultiply(QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, yaw), + QuatFromAxisAngle(Vec3{1.f, 0.f, 0.f}, -kPanelTiltRadians)); + panel_.position = + Vec3{head.position.x + forward.x * kPanelDistanceMetres, + head.position.y - kPanelDropMetres, + head.position.z + forward.z * kPanelDistanceMetres}; + visible_ = true; + shift_ = false; + for (Hand& hand : hands_) hand = Hand{}; + RebuildLayout(); +} + +void VirtualKeyboard::SetText(const VirtualKeyboardText& text) { + if (text.revision == text_.revision) return; + const bool changed = text.utf8 != text_.utf8 || + text.cursor_utf16 != text_.cursor_utf16 || + text.password != text_.password; + ScrubText(&text_.utf8); + text_ = text; + if (changed) Touch(); +} + +void VirtualKeyboard::Hide() { + ScrubText(&text_.utf8); + text_.revision = UINT64_MAX; // the next SetText always reloads + if (!visible_) return; + visible_ = false; + for (Hand& hand : hands_) hand = Hand{}; + Touch(); +} + +int VirtualKeyboard::KeyAt(float u, float v) const { + for (std::size_t index = 0; index < keys_.size(); ++index) { + const VirtualKey& key = keys_[index]; + if (u >= key.u0 && u < key.u1 && v >= key.v0 && v < key.v1) { + return static_cast(index); + } + } + return -1; +} + +bool VirtualKeyboard::CapturesTrigger(int hand) const { + const Hand& state = hands_[hand & 1]; + return visible_ && (state.over_panel || state.captured); +} + +bool VirtualKeyboard::cursor(int hand, float* u, float* v) const { + const Hand& state = hands_[hand & 1]; + if (!visible_ || !state.over_panel) return false; + *u = state.u; + *v = state.v; + return true; +} + +void VirtualKeyboard::Activate(const VirtualKey& key, + VirtualKeyboardOutput* output) { + switch (key.action) { + case VirtualKeyAction::kText: + output->text.push_back(Output(key)); + if (shift_) { + shift_ = false; // one-shot, like a phone keyboard + RebuildLayout(); + } + return; + case VirtualKeyAction::kSpace: + output->text.emplace_back(" "); + return; + case VirtualKeyAction::kBackspace: + output->scancodes.push_back(kScancodeBackspace); + return; + case VirtualKeyAction::kEnter: + output->scancodes.push_back(kScancodeReturn); + return; + case VirtualKeyAction::kTab: + output->scancodes.push_back(kScancodeTab); + return; + case VirtualKeyAction::kShift: + shift_ = !shift_; + RebuildLayout(); + return; + case VirtualKeyAction::kCapsLock: + caps_lock_ = !caps_lock_; + RebuildLayout(); + return; + case VirtualKeyAction::kClose: + output->scancodes.push_back(kScancodeEscape); + output->closed = true; + Hide(); + return; + } +} + +VirtualKeyboardOutput VirtualKeyboard::Update( + std::uint64_t now_ns, const VirtualKeyboardPointer (&pointers)[2]) { + VirtualKeyboardOutput output; + if (!visible_) return output; + for (int index = 0; index < 2 && visible_; ++index) { + Hand& hand = hands_[index]; + const VirtualKeyboardPointer& pointer = pointers[index]; + float u = 0.f, v = 0.f; + const bool over = + pointer.valid && + IntersectVirtualKeyboardPanel(panel_, kPanelWidthMetres, + kPanelHeightMetres, pointer.aim, &u, &v, + nullptr); + const int hovered = over ? KeyAt(u, v) : -1; + if (over != hand.over_panel || hovered != hand.hovered || + (over && (std::abs(u - hand.u) > kCursorRedrawStep || + std::abs(v - hand.v) > kCursorRedrawStep))) { + Touch(); + hand.u = u; + hand.v = v; + } + hand.over_panel = over; + hand.hovered = hovered; + + const bool was_down = hand.trigger_down; + if (!pointer.valid) { + hand.trigger_down = false; + } else if (was_down) { + hand.trigger_down = pointer.trigger > kTriggerRelease; + } else { + hand.trigger_down = pointer.trigger >= kTriggerPress; + } + + if (hand.trigger_down && !was_down) { + hand.captured = over; + if (hovered >= 0) { + hand.pressed = hovered; + hand.repeat_at_ns = now_ns + kRepeatDelayNs; + Touch(); + // Activate may rebuild the layout and clear pressed/hovered. + const VirtualKey key = keys_[hovered]; + Activate(key, &output); + } + } else if (hand.trigger_down) { + if (hand.pressed >= 0 && hand.pressed != hovered) { + hand.pressed = -1; + Touch(); + } else if (hand.pressed >= 0 && + keys_[hand.pressed].action == VirtualKeyAction::kBackspace && + now_ns >= hand.repeat_at_ns) { + output.scancodes.push_back(kScancodeBackspace); + hand.repeat_at_ns = now_ns + kRepeatIntervalNs; + } + } else if (was_down) { + hand.captured = false; + if (hand.pressed >= 0) { + hand.pressed = -1; + Touch(); + } + } + } + return output; +} + +} // namespace mocktail::vr diff --git a/src/vr/virtual_keyboard_raster.cc b/src/vr/virtual_keyboard_raster.cc new file mode 100644 index 0000000..530623a --- /dev/null +++ b/src/vr/virtual_keyboard_raster.cc @@ -0,0 +1,325 @@ +#include "mocktail/vr/virtual_keyboard.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { + +struct Rgb { + std::uint8_t r, g, b; +}; + +constexpr Rgb kBackground{24, 25, 30}; +constexpr Rgb kKey{58, 60, 70}; +constexpr Rgb kSpecialKey{42, 44, 52}; +constexpr Rgb kHover{92, 96, 112}; +constexpr Rgb kAccent{82, 132, 255}; +constexpr Rgb kAccentHover{120, 160, 255}; +constexpr Rgb kLabel{240, 242, 248}; +constexpr Rgb kField{12, 13, 17}; +constexpr int kFieldPadding = 24; +constexpr int kGap = 8; +constexpr int kRadius = 12; +constexpr float kLetterPoints = 40.f; +constexpr float kSpecialPoints = 28.f; + +// fontconfig's best sans face; MOCKTAIL_TEXT_FONT wins like the TextBox +// overlay. +std::string ResolveSansFont() { + const char* configured = std::getenv("MOCKTAIL_TEXT_FONT"); + if (configured != nullptr && configured[0] != '\0') return configured; + std::string file; + if (!FcInit()) return file; + FcPattern* pattern = + FcNameParse(reinterpret_cast("sans:weight=medium")); + if (pattern == nullptr) return file; + FcConfigSubstitute(nullptr, pattern, FcMatchPattern); + FcDefaultSubstitute(pattern); + FcResult result = FcResultNoMatch; + FcPattern* match = FcFontMatch(nullptr, pattern, &result); + FcPatternDestroy(pattern); + if (match == nullptr) return file; + FcChar8* path = nullptr; + if (FcPatternGetString(match, FC_FILE, 0, &path) == FcResultMatch && + path != nullptr) { + file = reinterpret_cast(path); + } + FcPatternDestroy(match); + return file; +} + +struct AlphaMask { + int width = 0; + int height = 0; + std::vector alpha; +}; + +// Field text as drawn: password input shows one dot per code point. +std::string DisplayText(const VirtualKeyboard& keyboard) { + if (!keyboard.text_password()) return keyboard.text(); + std::string dots; + for (const char c : keyboard.text()) + if ((static_cast(c) & 0xC0) != 0x80) dots += "\u2022"; + return dots; +} + +// Byte offset in `text` of the UTF-16 cursor. Password dots are one UTF-16 +// unit each, like the BMP characters they stand for. +std::size_t CursorByte(const std::string& text, int cursor_utf16) { + std::size_t byte = 0; + int units = 0; + while (byte < text.size() && units < cursor_utf16) { + const auto lead = static_cast(text[byte]); + const std::size_t length = lead < 0x80 ? 1 : lead < 0xE0 ? 2 + : lead < 0xF0 ? 3 : 4; + units += length == 4 ? 2 : 1; + byte = std::min(text.size(), byte + length); + } + return byte; +} + +void Scrub(std::vector* bytes) { + std::fill(bytes->begin(), bytes->end(), 0); + bytes->clear(); +} + +bool InsideRoundedRect(int x, int y, int left, int top, int right, int bottom, + int radius) { + const int cx = std::clamp(x, left + radius, right - 1 - radius); + const int cy = std::clamp(y, top + radius, bottom - 1 - radius); + const int dx = x - cx; + const int dy = y - cy; + return dx * dx + dy * dy <= radius * radius; +} + +} // namespace + +struct VirtualKeyboardRasterizer::Impl { + bool font_attempted = false; + bool ttf_initialized = false; + TTF_Font* letter_font = nullptr; + TTF_Font* special_font = nullptr; + std::map, AlphaMask> masks; + // Last rendered field text; not cached by content, since it is sensitive. + AlphaMask field; + + ~Impl() { + Scrub(&field.alpha); + if (letter_font != nullptr) TTF_CloseFont(letter_font); + if (special_font != nullptr) TTF_CloseFont(special_font); + if (ttf_initialized) TTF_Quit(); + } + + bool EnsureFonts() { + if (font_attempted) return letter_font != nullptr; + font_attempted = true; + if (!TTF_Init()) { + std::fprintf(stderr, " [vr-keyboard] SDL3_ttf unavailable: %s\n", + SDL_GetError()); + return false; + } + ttf_initialized = true; + const std::string file = ResolveSansFont(); + if (!file.empty()) { + letter_font = TTF_OpenFont(file.c_str(), kLetterPoints); + special_font = TTF_OpenFont(file.c_str(), kSpecialPoints); + } + if (letter_font == nullptr || special_font == nullptr) { + std::fprintf(stderr, + " [vr-keyboard] no usable font (%s); keys are unlabelled\n", + file.empty() ? "fontconfig found none" : file.c_str()); + return false; + } + std::fprintf(stderr, " [vr-keyboard] labels use %s\n", file.c_str()); + return true; + } + + static AlphaMask Render(TTF_Font* font, const std::string& text) { + AlphaMask mask; + SDL_Surface* rendered = TTF_RenderText_Blended( + font, text.c_str(), text.size(), SDL_Color{255, 255, 255, 255}); + if (rendered != nullptr) { + SDL_Surface* rgba = SDL_ConvertSurface(rendered, SDL_PIXELFORMAT_RGBA32); + if (rgba != nullptr && SDL_LockSurface(rgba)) { + mask.width = rgba->w; + mask.height = rgba->h; + mask.alpha.resize(static_cast(mask.width) * mask.height); + for (int y = 0; y < mask.height; ++y) { + const auto* row = static_cast(rgba->pixels) + + static_cast(y) * rgba->pitch; + for (int x = 0; x < mask.width; ++x) { + mask.alpha[static_cast(y) * mask.width + x] = + row[x * 4 + 3]; + } + } + SDL_UnlockSurface(rgba); + } + if (rgba != nullptr) SDL_DestroySurface(rgba); + SDL_DestroySurface(rendered); + } + return mask; + } + + const AlphaMask* Mask(const std::string& label, bool special) { + if (label.empty() || !EnsureFonts()) return nullptr; + const auto key = std::make_pair(label, special); + auto found = masks.find(key); + if (found != masks.end()) return &found->second; + return &masks + .emplace(key, Render(special ? special_font : letter_font, + label)) + .first->second; + } +}; + +VirtualKeyboardRasterizer::VirtualKeyboardRasterizer() : impl_(new Impl) {} +VirtualKeyboardRasterizer::~VirtualKeyboardRasterizer() { delete impl_; } + +bool VirtualKeyboardRasterizer::Draw(const VirtualKeyboard& keyboard, + std::vector* rgba) { + constexpr int width = VirtualKeyboard::kTextureWidth; + constexpr int height = VirtualKeyboard::kTextureHeight; + rgba->resize(static_cast(width) * height * 4); + std::uint8_t* pixels = rgba->data(); + auto put = [&](int x, int y, Rgb color) { + std::uint8_t* p = pixels + (static_cast(y) * width + x) * 4; + p[0] = color.r; + p[1] = color.g; + p[2] = color.b; + p[3] = 255; + }; + auto blend = [&](int x, int y, Rgb color, std::uint8_t alpha) { + if (x < 0 || y < 0 || x >= width || y >= height || alpha == 0) return; + std::uint8_t* p = pixels + (static_cast(y) * width + x) * 4; + const int a = alpha; + p[0] = static_cast((color.r * a + p[0] * (255 - a)) / 255); + p[1] = static_cast((color.g * a + p[1] * (255 - a)) / 255); + p[2] = static_cast((color.b * a + p[2] * (255 - a)) / 255); + }; + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; ++x) put(x, y, kBackground); + + // Text field: the focused TextBox's contents and caret, scrolled so the + // caret stays visible. + { + const int left = kGap / 2; + const int right = width - kGap / 2; + const int top = kGap / 2; + const int bottom = static_cast(std::lround( + VirtualKeyboard::kFieldFraction * height)) - + kGap / 2; + for (int y = top; y < bottom; ++y) + for (int x = left; x < right; ++x) + if (InsideRoundedRect(x, y, left, top, right, bottom, kRadius)) + put(x, y, kField); + const int clip_left = left + kFieldPadding; + const int clip_right = right - kFieldPadding; + const int inner = clip_right - clip_left; + const std::string shown = DisplayText(keyboard); + Scrub(&impl_->field.alpha); + impl_->field = AlphaMask{}; + int caret_x = 0; + int line_height = static_cast(kLetterPoints * 1.2f); + if (impl_->EnsureFonts()) { + line_height = TTF_GetFontHeight(impl_->letter_font); + if (!shown.empty()) { + impl_->field = Impl::Render(impl_->letter_font, shown); + const std::size_t cursor = + CursorByte(shown, keyboard.text_cursor_utf16()); + int unused = 0; + if (cursor > 0) + (void)TTF_GetStringSize(impl_->letter_font, shown.c_str(), cursor, + &caret_x, &unused); + } + } + const int scroll = std::max(0, caret_x - inner + 4); + const int origin_y = (top + bottom - line_height) / 2; + const AlphaMask& text = impl_->field; + for (int y = 0; y < text.height; ++y) + for (int x = 0; x < text.width; ++x) { + const int px = clip_left + x - scroll; + if (px < clip_left || px >= clip_right) continue; + blend(px, origin_y + y, kLabel, + text.alpha[static_cast(y) * text.width + x]); + } + const int caret = clip_left + caret_x - scroll; + const int caret_top = origin_y + line_height / 8; + const int caret_bottom = origin_y + line_height - line_height / 8; + for (int y = caret_top; y < caret_bottom; ++y) + for (int x = caret; x < caret + 3; ++x) + if (x >= clip_left - 2 && x < clip_right + 2) put(x, y, kAccent); + } + + bool labelled = true; + const auto& keys = keyboard.keys(); + for (std::size_t index = 0; index < keys.size(); ++index) { + const VirtualKey& key = keys[index]; + const int left = static_cast(std::lround(key.u0 * width)) + kGap / 2; + const int right = static_cast(std::lround(key.u1 * width)) - kGap / 2; + const int top = static_cast(std::lround(key.v0 * height)) + kGap / 2; + const int bottom = + static_cast(std::lround(key.v1 * height)) - kGap / 2; + const int i = static_cast(index); + const bool pressed = + keyboard.pressed_key(0) == i || keyboard.pressed_key(1) == i; + const bool hovered = + keyboard.hovered_key(0) == i || keyboard.hovered_key(1) == i; + const bool special = key.action != VirtualKeyAction::kText; + // Latched modifiers stay lit; hover still shows on top of them. + const bool latched = + (key.action == VirtualKeyAction::kShift && keyboard.shifted()) || + (key.action == VirtualKeyAction::kCapsLock && keyboard.caps_lock()); + Rgb fill = special ? kSpecialKey : kKey; + if (latched) fill = kAccent; + if (hovered) fill = latched ? kAccentHover : kHover; + if (pressed) fill = kAccent; + for (int y = top; y < bottom; ++y) + for (int x = left; x < right; ++x) + if (InsideRoundedRect(x, y, left, top, right, bottom, kRadius)) + put(x, y, fill); + + const AlphaMask* mask = impl_->Mask(key.label, special); + if (!key.label.empty() && (mask == nullptr || mask->alpha.empty())) { + labelled = false; + continue; + } + if (mask == nullptr) continue; + const int origin_x = (left + right - mask->width) / 2; + const int origin_y = (top + bottom - mask->height) / 2; + for (int y = 0; y < mask->height; ++y) + for (int x = 0; x < mask->width; ++x) + blend(origin_x + x, origin_y + y, kLabel, + mask->alpha[static_cast(y) * mask->width + x]); + } + + // Pointer dots, one per hand over the panel. + for (int hand = 0; hand < 2; ++hand) { + float u = 0.f, v = 0.f; + if (!keyboard.cursor(hand, &u, &v)) continue; + const int cx = static_cast(std::lround(u * width)); + const int cy = static_cast(std::lround(v * height)); + constexpr int kOuter = 11, kInner = 8; + for (int y = -kOuter; y <= kOuter; ++y) + for (int x = -kOuter; x <= kOuter; ++x) { + const int d2 = x * x + y * y; + if (d2 <= kInner * kInner) + blend(cx + x, cy + y, Rgb{255, 255, 255}, 255); + else if (d2 <= kOuter * kOuter) + blend(cx + x, cy + y, Rgb{0, 0, 0}, 200); + } + } + return labelled; +} + +} // namespace mocktail::vr diff --git a/src/window/window.cc b/src/window/window.cc index a3a54c3..c2b4e52 100644 --- a/src/window/window.cc +++ b/src/window/window.cc @@ -11,8 +11,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -168,6 +170,15 @@ static WindowFullscreenStateSync g_fullscreen_state_sync; static std::unique_ptr g_resize_readiness_gate; static std::unique_ptr g_text_input_backend; static std::unique_ptr g_text_input_owner; +// Generation of the showKeyboard request still awaiting its hideKeyboard; +// 0 when Roblox wants no keyboard. +static std::atomic g_text_input_requested_generation{0}; +struct VirtualInput { + std::string text; // committed text when non-empty + uint32_t scancode = 0; // otherwise one key press +}; +static std::mutex g_virtual_input_mutex; +static std::deque g_virtual_input; static std::unique_ptr g_pointer_capture_backend; // The owner is created and destroyed on the SDL thread, but the experience // composition reaches it from its own thread. The mutex guards the pointer @@ -1650,18 +1661,51 @@ bool SetPreTextInputPumpCallback(PreTextInputPumpCallback callback, } void ClearPreTextInputPumpCallback() { g_pre_text_input_pump_gate.Clear(); } void SetWindowTextInputOwnerEnabled(bool enabled) { + if (!enabled) { + g_text_input_requested_generation.store(0, std::memory_order_release); + } if (g_text_input_owner != nullptr) { g_text_input_owner->SetEnabled(enabled); } } bool RequestShowTextInput(uint64_t generation, const TextInputArea& area, const TextInputOptions& options) { - return g_text_input_owner != nullptr && - g_text_input_owner->RequestShowTextInput(generation, area, options); + const bool accepted = + g_text_input_owner != nullptr && + g_text_input_owner->RequestShowTextInput(generation, area, options); + if (accepted) { + g_text_input_requested_generation.store(generation, + std::memory_order_release); + } + return accepted; } bool RequestHideTextInput(uint64_t generation) { - return g_text_input_owner != nullptr && - g_text_input_owner->RequestHideTextInput(generation); + const bool accepted = g_text_input_owner != nullptr && + g_text_input_owner->RequestHideTextInput(generation); + if (accepted) { + uint64_t expected = generation; + g_text_input_requested_generation.compare_exchange_strong( + expected, 0, std::memory_order_acq_rel); + } + return accepted; +} +bool TextInputRequested() { + return g_text_input_requested_generation.load(std::memory_order_acquire) != + 0; +} +void QueueVirtualTextInput(const char* utf8) { + if (utf8 == nullptr || utf8[0] == '\0') { + return; + } + std::lock_guard lock(g_virtual_input_mutex); + g_virtual_input.push_back(VirtualInput{utf8, 0}); +} +void QueueVirtualKeyPress(uint32_t scancode) { + if (scancode == 0) { + return; + } + std::lock_guard lock(g_virtual_input_mutex); + g_virtual_input.push_back(VirtualInput{{}, scancode}); } void* GetGLProcAddress(const char* name) { if (!g_state.initialised || name == nullptr || name[0] == '\0') { @@ -2276,6 +2320,31 @@ bool PumpEvents() { g_text_input_owner->active())) { fprintf(stderr, " [input] SDL pointer capture state update failed\n"); } + { + std::deque virtual_input; + { + std::lock_guard lock(g_virtual_input_mutex); + virtual_input.swap(g_virtual_input); + } + for (const VirtualInput& input : virtual_input) { + platform::PlatformEvent event{}; + event.timestamp_ns = SDL_GetTicksNS(); + if (!input.text.empty()) { + event.payload = platform::TextInputEvent{input.text}; + g_platform_event_observer.Notify(event); + continue; + } + const auto scancode = static_cast(input.scancode); + const auto keycode = static_cast( + SDL_GetKeyFromScancode(scancode, SDL_KMOD_NONE, false)); + event.payload = platform::KeyEvent{true, false, input.scancode, keycode, + 0}; + g_platform_event_observer.Notify(event); + event.payload = platform::KeyEvent{false, false, input.scancode, keycode, + 0}; + g_platform_event_observer.Notify(event); + } + } platform::PlatformEvent pending_motion{}; bool has_pending_motion = false; diff --git a/src/window/window.h b/src/window/window.h index 5472f1b..07dc85b 100644 --- a/src/window/window.h +++ b/src/window/window.h @@ -98,6 +98,15 @@ bool RequestShowTextInput(uint64_t generation, const TextInputArea& area, const TextInputOptions& options); bool RequestHideTextInput(uint64_t generation); +// In-headset keyboard support. TextInputRequested reports whether Roblox +// currently wants an on-screen keyboard (between showKeyboard and the matching +// hideKeyboard). The Queue calls are safe from any thread; PumpEvents delivers +// them on the main thread as the same platform events a physical keyboard +// produces, so the focused TextBox edits them identically. +bool TextInputRequested(); +void QueueVirtualTextInput(const char* utf8); +void QueueVirtualKeyPress(uint32_t scancode); + void* GetEGLDisplay(); void* GetEGLSurface(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0166d2a..7557cc0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,6 +40,9 @@ mocktail_apply_compile_options(vr_pose_math_test) add_executable(xr_controller_test vr/xr_controller_test.cc) target_link_libraries(xr_controller_test PRIVATE Mocktail::VR GTest::gtest_main) mocktail_apply_compile_options(xr_controller_test) +add_executable(virtual_keyboard_test vr/virtual_keyboard_test.cc) +target_link_libraries(virtual_keyboard_test PRIVATE Mocktail::VR GTest::gtest_main) +mocktail_apply_compile_options(virtual_keyboard_test) add_executable(vr_evidence_layer_test vr/vr_evidence_layer_test.cc) target_link_libraries(vr_evidence_layer_test PRIVATE Vulkan::Headers ${CMAKE_DL_LIBS} GTest::gtest_main) @@ -1127,6 +1130,7 @@ gtest_discover_tests(vr_evidence_layer_test) gtest_discover_tests(vr_mirror_test) gtest_discover_tests(vr_pose_math_test) gtest_discover_tests(xr_controller_test) +gtest_discover_tests(virtual_keyboard_test) gtest_discover_tests(jnivm_test) gtest_discover_tests(jnivm_fields_test) gtest_discover_tests(runtime_config_test) diff --git a/tests/vr/virtual_keyboard_test.cc b/tests/vr/virtual_keyboard_test.cc new file mode 100644 index 0000000..1dfb6da --- /dev/null +++ b/tests/vr/virtual_keyboard_test.cc @@ -0,0 +1,469 @@ +#include + +#include +#include +#include +#include +#include + +#include "mocktail/vr/virtual_keyboard.h" + +namespace mocktail::vr { +namespace { + +using pose::Pose; +using pose::Quat; +using pose::QuatFromAxisAngle; +using pose::QuatRotate; +using pose::Vec3; + +constexpr std::uint64_t kMs = 1'000'000ULL; +constexpr std::uint32_t kBackspace = 42; +constexpr std::uint32_t kReturn = 40; +constexpr std::uint32_t kEscape = 41; +constexpr std::uint32_t kTab = 43; + +Pose HeadAt(float x, float y, float z, float yaw = 0.f) { + Pose head; + head.position = Vec3{x, y, z}; + head.orientation = QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, yaw); + return head; +} + +// An aim pose 30 cm in front of the panel, pointing straight at (u, v). +Pose AimAt(const VirtualKeyboard& keyboard, float u, float v) { + const Pose& panel = keyboard.panel_pose(); + const Vec3 local{(u - 0.5f) * VirtualKeyboard::kPanelWidthMetres, + (0.5f - v) * VirtualKeyboard::kPanelHeightMetres, 0.3f}; + const Vec3 offset = QuatRotate(panel.orientation, local); + Pose aim; + aim.position = Vec3{panel.position.x + offset.x, panel.position.y + offset.y, + panel.position.z + offset.z}; + aim.orientation = panel.orientation; // -Z of the aim is -normal + return aim; +} + +int FindKey(const VirtualKeyboard& keyboard, const std::string& label) { + const auto& keys = keyboard.keys(); + for (std::size_t i = 0; i < keys.size(); ++i) + if (keys[i].label == label) return static_cast(i); + return -1; +} + +Pose AimAtKey(const VirtualKeyboard& keyboard, const std::string& label) { + const int index = FindKey(keyboard, label); + EXPECT_GE(index, 0) << label; + const VirtualKey& key = keyboard.keys()[index]; + return AimAt(keyboard, (key.u0 + key.u1) / 2, (key.v0 + key.v1) / 2); +} + +struct Driver { + VirtualKeyboard keyboard; + std::uint64_t now = 1'000 * kMs; + VirtualKeyboardOutput Frame(const Pose& aim, float trigger, int hand = 1, + std::uint64_t step = 11 * kMs) { + now += step; + VirtualKeyboardPointer pointers[2]; + pointers[hand].valid = true; + pointers[hand].aim = aim; + pointers[hand].trigger = trigger; + return keyboard.Update(now, pointers); + } + // Press and release a key, returning everything it produced. + VirtualKeyboardOutput Tap(const std::string& label) { + const Pose aim = AimAtKey(keyboard, label); + VirtualKeyboardOutput out = Frame(aim, 1.f); + const VirtualKeyboardOutput up = Frame(aim, 0.f); + out.text.insert(out.text.end(), up.text.begin(), up.text.end()); + out.scancodes.insert(out.scancodes.end(), up.scancodes.begin(), + up.scancodes.end()); + return out; + } +}; + +TEST(VirtualKeyboardTest, RayHitsQuadCentreAndRejectsBackAndAway) { + Pose quad; + quad.position = Vec3{0.f, 0.f, -1.f}; + Pose aim; // origin, looking down -Z + float u = 0.f, v = 0.f, distance = 0.f; + ASSERT_TRUE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + &distance)); + EXPECT_NEAR(u, 0.5f, 1e-5f); + EXPECT_NEAR(v, 0.5f, 1e-5f); + EXPECT_NEAR(distance, 1.f, 1e-5f); + + // Up and to the right maps to the top-right of the panel (v grows down). + aim.position = Vec3{0.2f, 0.08f, 0.f}; + ASSERT_TRUE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); + EXPECT_NEAR(u, 0.9f, 1e-5f); + EXPECT_NEAR(v, 0.1f, 1e-5f); + + aim.position = Vec3{0.3f, 0.f, 0.f}; // misses the 0.5 m width + EXPECT_FALSE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); + aim.position = Vec3{}; + aim.orientation = QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, pose::kPi); + EXPECT_FALSE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); + // Behind the panel looking back at it: the back face takes no input. + aim.position = Vec3{0.f, 0.f, -2.f}; + EXPECT_FALSE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); +} + +TEST(VirtualKeyboardTest, ShowPlacesPanelBelowAndInFrontFacingTheHead) { + VirtualKeyboard keyboard; + EXPECT_FALSE(keyboard.visible()); + // Facing +X (yaw -90 degrees) at standing height. + keyboard.Show(HeadAt(1.f, 1.6f, 2.f, -pose::kPi / 2)); + ASSERT_TRUE(keyboard.visible()); + const Pose& panel = keyboard.panel_pose(); + EXPECT_NEAR(panel.position.x, 1.45f, 1e-4f); + EXPECT_NEAR(panel.position.y, 1.32f, 1e-4f); + EXPECT_NEAR(panel.position.z, 2.f, 1e-4f); + const Vec3 normal = QuatRotate(panel.orientation, Vec3{0.f, 0.f, 1.f}); + EXPECT_LT(normal.x, -0.5f); // faces back toward the head + EXPECT_GT(normal.y, 0.3f); // tilted up toward the eyes + + // Looking from the eyes at the panel centre lands mid-panel. + Pose eyes = HeadAt(1.f, 1.6f, 2.f); + const Vec3 to_panel{panel.position.x - 1.f, panel.position.y - 1.6f, 0.f}; + const float pitch = std::atan2(to_panel.y, to_panel.x); + eyes.orientation = pose::QuatMultiply( + QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, -pose::kPi / 2), + QuatFromAxisAngle(Vec3{1.f, 0.f, 0.f}, pitch)); + float u = 0.f, v = 0.f; + ASSERT_TRUE(IntersectVirtualKeyboardPanel( + panel, VirtualKeyboard::kPanelWidthMetres, + VirtualKeyboard::kPanelHeightMetres, eyes, &u, &v, nullptr)); + EXPECT_NEAR(u, 0.5f, 1e-3f); + EXPECT_NEAR(v, 0.5f, 1e-3f); +} + +TEST(VirtualKeyboardTest, TriggerPressTypesHoveredKeyOnce) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose aim = AimAtKey(d.keyboard, "q"); + EXPECT_TRUE(d.Frame(aim, 0.f).text.empty()); + EXPECT_EQ(d.keyboard.hovered_key(1), FindKey(d.keyboard, "q")); + const auto pressed = d.Frame(aim, 0.9f); + ASSERT_EQ(pressed.text.size(), 1u); + EXPECT_EQ(pressed.text[0], "q"); + EXPECT_EQ(d.keyboard.pressed_key(1), FindKey(d.keyboard, "q")); + // Held, and hovering between the hysteresis thresholds: no repeat for text. + EXPECT_TRUE(d.Frame(aim, 0.7f, 1, 600 * kMs).text.empty()); + EXPECT_TRUE(d.Frame(aim, 0.f).text.empty()); + EXPECT_EQ(d.keyboard.pressed_key(1), -1); + EXPECT_EQ(d.Tap("Enter").scancodes, std::vector{kReturn}); + EXPECT_EQ(d.Tap("Space").text, std::vector{" "}); + EXPECT_EQ(d.Tap("Tab").scancodes, std::vector{kTab}); +} + +TEST(VirtualKeyboardTest, ShiftIsOneShotAndSelectsUpperSymbols) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Tap("Shift"); + EXPECT_TRUE(d.keyboard.shifted()); + EXPECT_GE(FindKey(d.keyboard, "!"), 0); // labels follow Shift + EXPECT_EQ(d.Tap("A").text, std::vector{"A"}); + EXPECT_FALSE(d.keyboard.shifted()); + EXPECT_EQ(d.Tap("b").text, std::vector{"b"}); + d.Tap("Shift"); + EXPECT_EQ(d.Tap("!").text, std::vector{"!"}); + EXPECT_EQ(d.Tap("1").text, std::vector{"1"}); + d.Tap("Shift"); + EXPECT_EQ(d.Tap("|").text, std::vector{"|"}); + EXPECT_EQ(d.Tap("\\").text, std::vector{"\\"}); + // Tapping Shift again cancels it. + d.Tap("Shift"); + d.Tap("Shift"); + EXPECT_FALSE(d.keyboard.shifted()); +} + +// US QWERTY pairs from the desktop shell OSK (KiiteShell layouts.json, +// "English (US)"), in layout order. +const std::vector> kUsShiftPairs = { + {"`", "~"}, {"1", "!"}, {"2", "@"}, {"3", "#"}, {"4", "$"}, + {"5", "%"}, {"6", "^"}, {"7", "&"}, {"8", "*"}, {"9", "("}, + {"0", ")"}, {"-", "_"}, {"=", "+"}, {"q", "Q"}, {"w", "W"}, + {"e", "E"}, {"r", "R"}, {"t", "T"}, {"y", "Y"}, {"u", "U"}, + {"i", "I"}, {"o", "O"}, {"p", "P"}, {"[", "{"}, {"]", "}"}, + {"\\", "|"}, {"a", "A"}, {"s", "S"}, {"d", "D"}, {"f", "F"}, + {"g", "G"}, {"h", "H"}, {"j", "J"}, {"k", "K"}, {"l", "L"}, + {";", ":"}, {"'", "\""}, {"z", "Z"}, {"x", "X"}, {"c", "C"}, + {"v", "V"}, {"b", "B"}, {"n", "N"}, {"m", "M"}, {",", "<"}, + {".", ">"}, {"/", "?"}}; + +std::vector TextKeyIndices(const VirtualKeyboard& keyboard) { + std::vector indices; + for (std::size_t i = 0; i < keyboard.keys().size(); ++i) + if (keyboard.keys()[i].action == VirtualKeyAction::kText) + indices.push_back(static_cast(i)); + return indices; +} + +TEST(VirtualKeyboardTest, EveryKeyShiftsToItsUsQwertyCharacter) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const std::vector text_keys = TextKeyIndices(d.keyboard); + ASSERT_EQ(text_keys.size(), kUsShiftPairs.size()); + for (std::size_t n = 0; n < kUsShiftPairs.size(); ++n) { + const auto& [plain, shifted] = kUsShiftPairs[n]; + const int index = text_keys[n]; + SCOPED_TRACE("key " + plain); + ASSERT_EQ(d.keyboard.keys()[index].label, plain); + const VirtualKey key = d.keyboard.keys()[index]; + const Pose aim = + AimAt(d.keyboard, (key.u0 + key.u1) / 2, (key.v0 + key.v1) / 2); + + // Unshifted press types the plain character. + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{plain}); + d.Frame(aim, 0.f); + + // Shift relabels this key, the press types the shifted character, and + // the one-shot Shift then puts the plain label back. + d.Tap("Shift"); + ASSERT_TRUE(d.keyboard.shifted()); + EXPECT_EQ(d.keyboard.keys()[index].label, shifted); + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{shifted}); + d.Frame(aim, 0.f); + EXPECT_FALSE(d.keyboard.shifted()); + EXPECT_EQ(d.keyboard.keys()[index].label, plain); + } +} + +TEST(VirtualKeyboardTest, CapsLockAndShiftCombineLikeAPhysicalKeyboard) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const std::vector text_keys = TextKeyIndices(d.keyboard); + ASSERT_EQ(text_keys.size(), kUsShiftPairs.size()); + d.Tap("Caps"); + ASSERT_TRUE(d.keyboard.caps_lock()); + for (std::size_t n = 0; n < kUsShiftPairs.size(); ++n) { + const auto& [plain, shifted] = kUsShiftPairs[n]; + const bool letter = plain.size() == 1 && std::isalpha( + static_cast(plain[0])); + const int index = text_keys[n]; + SCOPED_TRACE("key " + plain); + // Caps Lock: letters upper, everything else unchanged. + const std::string caps = letter ? shifted : plain; + EXPECT_EQ(d.keyboard.keys()[index].label, caps); + const VirtualKey key = d.keyboard.keys()[index]; + const Pose aim = + AimAt(d.keyboard, (key.u0 + key.u1) / 2, (key.v0 + key.v1) / 2); + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{caps}); + d.Frame(aim, 0.f); + // Caps Lock + Shift: letters back to lower, symbols shifted. + d.Tap("Shift"); + const std::string both = letter ? plain : shifted; + EXPECT_EQ(d.keyboard.keys()[index].label, both); + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{both}); + d.Frame(aim, 0.f); + EXPECT_TRUE(d.keyboard.caps_lock()); + } +} + +TEST(VirtualKeyboardTest, CapsLockAffectsLettersOnlyAndShiftInvertsIt) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Tap("Caps"); + EXPECT_TRUE(d.keyboard.caps_lock()); + EXPECT_EQ(d.Tap("C").text, std::vector{"C"}); + EXPECT_EQ(d.Tap("D").text, std::vector{"D"}); + EXPECT_EQ(d.Tap("1").text, std::vector{"1"}); + d.Tap("Shift"); + EXPECT_EQ(d.Tap("e").text, std::vector{"e"}); + EXPECT_TRUE(d.keyboard.caps_lock()); + d.Tap("Caps"); + EXPECT_EQ(d.Tap("f").text, std::vector{"f"}); +} + +TEST(VirtualKeyboardTest, BackspaceRepeatsWhileHeld) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose aim = AimAtKey(d.keyboard, "Backspace"); + EXPECT_EQ(d.Frame(aim, 1.f).scancodes, + std::vector{kBackspace}); + EXPECT_TRUE(d.Frame(aim, 1.f, 1, 300 * kMs).scancodes.empty()); + EXPECT_EQ(d.Frame(aim, 1.f, 1, 200 * kMs).scancodes.size(), 1u); + EXPECT_TRUE(d.Frame(aim, 1.f, 1, 30 * kMs).scancodes.empty()); + EXPECT_EQ(d.Frame(aim, 1.f, 1, 40 * kMs).scancodes.size(), 1u); + // Sliding off the key stops the repeat. + const Pose off = AimAtKey(d.keyboard, "p"); + EXPECT_TRUE(d.Frame(off, 1.f, 1, 500 * kMs).scancodes.empty()); +} + +TEST(VirtualKeyboardTest, CapturesTriggerOnPanelAndUntilARelease) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + Pose away = AimAt(d.keyboard, 0.5f, 0.5f); + away.orientation = QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, pose::kPi); + d.Frame(away, 0.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + const Pose on = AimAtKey(d.keyboard, "g"); + d.Frame(on, 0.f); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + EXPECT_FALSE(d.keyboard.CapturesTrigger(0)); + d.Frame(on, 1.f); + d.Frame(away, 1.f); // dragged off while held: still the keyboard's + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + d.Frame(away, 0.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + // A press that starts off the panel is never captured. + d.Frame(away, 1.f); + d.Frame(on, 1.f); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); // aimed at it now + d.Frame(away, 1.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); +} + +TEST(VirtualKeyboardTest, LayoutRowsFillThePanelAndCloseKeyHides) { + VirtualKeyboard keyboard; + // Five rows; every row starts at the left edge and ends at the right edge. + int rows = 0; + for (const VirtualKey& key : keyboard.keys()) { + if (key.u0 == 0.f) ++rows; + } + EXPECT_EQ(rows, 5); + for (const VirtualKey& key : keyboard.keys()) { + EXPECT_LT(key.u0, key.u1); + EXPECT_LE(key.u1, 1.f); + } + for (const char* label : {"`", "=", "Backspace", "Tab", "[", "]", "\\", + "Caps", ";", "'", "Enter", "Shift", "/", "Space", + "Close"}) { + EXPECT_GE(FindKey(keyboard, label), 0) << label; + } + for (const char* label : {"Esc", "F1", "Ctrl", "Alt", "Menu"}) { + EXPECT_EQ(FindKey(keyboard, label), -1) << label; + } + + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const auto closed = d.Frame(AimAtKey(d.keyboard, "Close"), 1.f); + EXPECT_TRUE(closed.closed); + EXPECT_EQ(closed.scancodes, std::vector{kEscape}); + EXPECT_FALSE(d.keyboard.visible()); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + EXPECT_TRUE(d.Frame(AimAtKey(d.keyboard, "q"), 0.f).text.empty()); +} + +TEST(VirtualKeyboardTest, RevisionMovesOnlyWhenTheImageChanges) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose aim = AimAtKey(d.keyboard, "h"); + d.Frame(aim, 0.f); + const std::uint64_t hovered = d.keyboard.revision(); + d.Frame(aim, 0.f); + EXPECT_EQ(d.keyboard.revision(), hovered); + d.Frame(AimAtKey(d.keyboard, "j"), 0.f); + EXPECT_GT(d.keyboard.revision(), hovered); +} + +TEST(VirtualKeyboardTest, TextFieldSitsAboveTheKeysAndTypesNothing) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + for (const VirtualKey& key : d.keyboard.keys()) + EXPECT_GE(key.v0, VirtualKeyboard::kFieldFraction - 1e-6f) << key.label; + const Pose field = AimAt(d.keyboard, 0.5f, VirtualKeyboard::kFieldFraction / 2); + d.Frame(field, 0.f); + EXPECT_EQ(d.keyboard.hovered_key(1), -1); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); // still the keyboard's + const auto out = d.Frame(field, 1.f); + EXPECT_TRUE(out.text.empty()); + EXPECT_TRUE(out.scancodes.empty()); +} + +TEST(VirtualKeyboardTest, TextMirrorFollowsTheFocusedTextBox) { + ClearVirtualKeyboardText(); + SetVirtualKeyboardText("hi", 2, 2, false); + VirtualKeyboardText text = CurrentVirtualKeyboardText(); + EXPECT_EQ(text.utf8, "hi"); + EXPECT_EQ(text.cursor_utf16, 2); + + VirtualKeyboard keyboard; + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const std::uint64_t before = keyboard.revision(); + keyboard.SetText(text); + EXPECT_EQ(keyboard.text(), "hi"); + EXPECT_GT(keyboard.revision(), before); + // Same mirror revision: nothing to redraw. + const std::uint64_t same = keyboard.revision(); + keyboard.SetText(text); + EXPECT_EQ(keyboard.revision(), same); + // Cursor moves alone still redraw. + SetVirtualKeyboardText("hi", 2, 1, false); + keyboard.SetText(CurrentVirtualKeyboardText()); + EXPECT_EQ(keyboard.text_cursor_utf16(), 1); + EXPECT_GT(keyboard.revision(), same); + + // Hiding forgets the text; reopening reloads it even without a new update. + keyboard.Hide(); + EXPECT_TRUE(keyboard.text().empty()); + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + keyboard.SetText(CurrentVirtualKeyboardText()); + EXPECT_EQ(keyboard.text(), "hi"); + + ClearVirtualKeyboardText(); + EXPECT_TRUE(CurrentVirtualKeyboardText().utf8.empty()); +} + +TEST(VirtualKeyboardTest, RasterizerDrawsFieldTextAndCaret) { + VirtualKeyboard keyboard; + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + VirtualKeyboardRasterizer rasterizer; + auto field_light_pixels = [&](const std::vector& rgba) { + const int bottom = static_cast(VirtualKeyboard::kFieldFraction * + VirtualKeyboard::kTextureHeight); + int count = 0; + for (int y = 8; y < bottom - 8; ++y) + for (int x = 30; x < VirtualKeyboard::kTextureWidth - 30; ++x) + if (rgba[(static_cast(y) * + VirtualKeyboard::kTextureWidth + x) * 4 + 1] > 120) + ++count; + return count; + }; + std::vector empty, typed, masked; + if (!rasterizer.Draw(keyboard, &empty)) GTEST_SKIP() << "no host font"; + VirtualKeyboardText text; + text.utf8 = "Hello world"; + text.cursor_utf16 = 11; + text.revision = 1; + keyboard.SetText(text); + ASSERT_TRUE(rasterizer.Draw(keyboard, &typed)); + // Text adds light glyph pixels; the empty field only has its caret. + EXPECT_GT(field_light_pixels(typed), field_light_pixels(empty) + 200); + text.password = true; + text.revision = 2; + keyboard.SetText(text); + ASSERT_TRUE(rasterizer.Draw(keyboard, &masked)); + EXPECT_NE(masked, typed); // dots, not the letters + EXPECT_GT(field_light_pixels(masked), field_light_pixels(empty)); +} + +TEST(VirtualKeyboardTest, RasterizerFillsAnOpaqueTexture) { + VirtualKeyboard keyboard; + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + VirtualKeyboardRasterizer rasterizer; + std::vector rgba; + (void)rasterizer.Draw(keyboard, &rgba); // labels depend on host fonts + ASSERT_EQ(rgba.size(), static_cast( + VirtualKeyboard::kTextureWidth) * + VirtualKeyboard::kTextureHeight * 4); + for (std::size_t i = 3; i < rgba.size(); i += 4) ASSERT_EQ(rgba[i], 255); + // A key is lighter than the background behind it. + const auto& q = keyboard.keys()[FindKey(keyboard, "q")]; + const int x = static_cast(q.u0 * VirtualKeyboard::kTextureWidth) + 20; + const int y = static_cast(q.v0 * VirtualKeyboard::kTextureHeight) + 20; + const std::size_t key_px = + (static_cast(y) * VirtualKeyboard::kTextureWidth + x) * 4; + EXPECT_GT(rgba[key_px], rgba[0]); +} + +} // namespace +} // namespace mocktail::vr