From 53ddf07f1889846531a10b09ca181841c5d80ef3 Mon Sep 17 00:00:00 2001 From: toorux Date: Fri, 16 Oct 2026 00:40:10 +0800 Subject: [PATCH] fix(cast): compensate stereo framing with live eye and UI depth --- docs/casting.md | 4 ++ native/host.cpp | 71 ++++++++++++++++++++++++- native/recording_region.h | 25 +++++---- native/recording_region_depth.h | 90 ++++++++++++++++++++++++++++++++ src/session.rs | 17 ++++++ tests/cast_spatial_lifecycle.mjs | 18 ++++++- tests/recording_region.cpp | 6 +-- tests/recording_region_depth.cpp | 86 ++++++++++++++++++++++++++++++ tools/test-native.sh | 2 +- ui/src/cast-spatial-framing.tsx | 13 +++++ 10 files changed, 313 insertions(+), 19 deletions(-) create mode 100644 native/recording_region_depth.h create mode 100644 tests/recording_region_depth.cpp diff --git a/docs/casting.md b/docs/casting.md index ad3d25f..793941d 100644 --- a/docs/casting.md +++ b/docs/casting.md @@ -95,6 +95,10 @@ DISPLAY=:0 FRAMELY_MEDIA_BIN=/绝对路径/media/bin \ 可见范围同时检查工厂畸变 UV 中的面板覆盖和 OpenVR render projection 中的 HiddenAreaMesh；两套坐标不可混用，也不能跟随面板扫描方向翻转隐藏网格。左右眼分别检查，边界标记测试覆盖最外侧像素。正式框和校准标记使用相同的前景层级，避免被 Steam 空间窗口遮挡。 +上述检查证明的是软件渲染/采样有效范围，不是当前佩戴者的完整光学可见范围。OpenVR 的 HiddenAreaMesh 保证被遮区域不可见，但补集并不保证在所有瞳位、眼距下可见；不得把标记录制对齐测试当作鼻侧可见问题已经通过验收。鼻侧边界需要闭另一眼进行真实佩戴验证，不能通过任意翻转坐标或猜测裁剪半径来宣称修复。 + +双眼辅助框采用实时 `GetEyeToHeadTransform` 的实际选眼原点（含当前瞳距），而不是固定或上次启动时的眼距。保持取景角度不变，按中心射线与当前可见 Steam/Framely 窗口的真实交点计算参考深度，框宽度同步为 `2 × depth × tanHalfFov`。因此录制眼里的边界不变，另一眼的视差由实测窗口距离决定，减少原固定 2 米平面对近处窗口的错位。拖动时冻结深度以保持激光交互稳定；暂时移出窗口时保留最近实测深度。首次没有可测窗口时仍使用 2 米近似，不能宣称读取了任意游戏的场景深度；倾斜/曲面窗口在中心处匹配，整个边框仍是平面近似。改变参考深度不会扩大录制眼的光学可见范围。 + 仅在用户同意短暂显示校准标记后，运行 `cast::region_device_test::recording_region_device_alignment` 忽略测试。指定 `FRAMELY_REGION_TEST_MEDIA` 和 `FRAMELY_REGION_TEST_MARKER`，后者由 `tests/recording_region_device.cpp` 编译。测试覆盖左右眼、4:3、16:9、竖屏、偏移及高分辨率，以实际录制帧内四角和中心标记的位置检查空间框与输出是否一致；使用隔离临时目录，不替换正式安装文件。 `tests/cast_frame.cpp` 验证原生视频与透明控件的预乘 Alpha 合成、控件纹理保留和收起后的完整画面，需要 GLX 环境。 diff --git a/native/host.cpp b/native/host.cpp index 9bde251..a979344 100644 --- a/native/host.cpp +++ b/native/host.cpp @@ -33,6 +33,7 @@ #include "ui_feedback.h" #include "recording_indicator.h" #include "recording_region.h" +#include "recording_region_depth.h" #include "framing_lifecycle.h" #include "plugin_navigation.h" #include @@ -130,6 +131,7 @@ template struct Handler { #include "cast_frame.h" struct View { CastFrame cast_frame; + json region_geometry=json::object(); std::string key,plugin,title,region_editor_id;int width=600,height=840;float render_scale=1.f;bool dashboard=false,shown=false,dirty=false,closing=false,closed=false,framing_left_down=false; OverlayPlacement placement;NotificationClick notification_click;LauncherPointer launcher_pointer;std::chrono::steady_clock::time_point haptic_at{},activation_at{}; cef_mouse_event_t pointer{};bool pointer_valid=false,stick_scrolling=false;vr::TrackedDeviceIndex_t pointer_device=vr::k_unTrackedDeviceIndexInvalid;std::chrono::steady_clock::time_point pointer_at{}; @@ -226,6 +228,71 @@ static vr::VROverlayHandle_t active_main_window(){ overlays->FindOverlay("valve.steam.gamepadui.main",&selected); return selected; } +struct RecordingRegionPresentation {json view;double distance=2.;}; +static RecordingRegionPresentation recording_region_presentation(const json& original,bool freeze_depth=false){ + RecordingRegionPresentation result{original,2.}; + // This changes stereo presentation only, never the recorded angular crop. + // A center-ray hit measures real UI depth, including tilted/curved windows. + static auto sampled=std::chrono::steady_clock::time_point{}; + static vr::HmdMatrix34_t left{},right{};static bool eyes_valid=false,measured=false; + static int previous_eye=-1;static double distance=2.; + try{ + int eye=original.at("eye");if(eye<0||eye>1)return result; + vr::HmdMatrix34_t capture{}; + for(int i=0;i<3;i++)for(int j=0;j<4;j++)capture.m[i][j]=original.at("captureToHead").at(i).at(j).get(); + auto now=std::chrono::steady_clock::now();bool probe=now-sampled>=std::chrono::milliseconds(60); + if(probe){left=vr_system->GetEyeToHeadTransform(vr::Eye_Left);right=vr_system->GetEyeToHeadTransform(vr::Eye_Right);eyes_valid=true;sampled=now;} + if(eyes_valid&&recording_region_refresh_eye_origin(static_cast(eye),left,right,capture)){ + for(int i=0;i<3;i++)result.view["captureToHead"][i][3]=capture.m[i][3]; + } + if(eye!=previous_eye){previous_eye=eye;distance=2.;measured=false;probe=true;} + if(probe&&!freeze_depth){ + vr::TrackedDevicePose_t poses[1]{}; + vr_system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding,0,poses,1); + if(poses[0].bPoseIsValid&&poses[0].bDeviceIsConnected){ + using namespace recording_region_depth_detail; + const auto& head=poses[0].mDeviceToAbsoluteTracking; + const double cx=original.at("centerTanX"),cy=original.at("centerTanY"); + auto origin=add(translation(head),rotate(head,translation(capture))); + auto direction=rotate(head,rotate(capture,{cx,-cy,-1.})); + double length=std::sqrt(dot(direction,direction)); + if(std::isfinite(length)&&length>1e-6){ + vr::VROverlayIntersectionParams_t ray{};ray.eOrigin=vr::TrackingUniverseStanding; + for(int i=0;i<3;i++){ray.vSource.v[i]=origin[i];ray.vDirection.v[i]=direction[i]/length;} + std::set candidates; + for(const char* key:{"valve.steam.gamepadui.main","valve.steam.gamepadui.fallback"}){vr::VROverlayHandle_t handle=0;if(!overlays->FindOverlay(key,&handle))candidates.insert(handle);} + for(const auto& key:dashboard_keys)if(key.rfind("valve.steam.",0)==0&&key!="valve.steam.gamepadui.bar"){vr::VROverlayHandle_t handle=0;if(!overlays->FindOverlay(key.c_str(),&handle))candidates.insert(handle);} + for(const auto& [key,v]:views)if(!v->closing&&v->shown&&(v->dashboard||key.rfind("framely.window.",0)==0||key.rfind("framely.cast.",0)==0)&&key!="launcher")candidates.insert(v->overlay); + std::optional nearest; + for(auto handle:candidates){ + if(!overlays->IsOverlayVisible(handle))continue; + vr::VROverlayIntersectionResults_t hit{}; + if(!overlays->ComputeOverlayIntersection(handle,&ray,&hit)||!std::isfinite(hit.vUVs.v[0])||!std::isfinite(hit.vUVs.v[1])||hit.vUVs.v[0]<0||hit.vUVs.v[0]>1||hit.vUVs.v[1]<0||hit.vUVs.v[1]>1)continue; + auto depth=recording_region_hit_depth(head,capture,cx,cy,hit.vPoint); + if(depth&&(!nearest||*depth<*nearest))nearest=depth; + } + if(nearest){ + if(!measured){std::cout<<"Recording frame measured UI depth "<<*nearest<<" m, IPD "<SetOverlayWidthInMeters(v.overlay,plane.width); + overlays->SetOverlayTransformTrackedDeviceRelative(v.overlay,vr::k_unTrackedDeviceIndex_Hmd,&plane.transform); +} static void hover_haptic(vr::VROverlayHandle_t overlay,std::chrono::steady_clock::time_point& last,bool activation=false){ auto now=std::chrono::steady_clock::now();if(now-lastIsOverlayVisible(overlay)||!overlays->IsHoverTargetOverlay(overlay))return; last=now;if(ui_sounds)ui_audio.play(activation); @@ -622,7 +689,7 @@ static void entry_actions(vr::VROverlayHandle_t button,bool available){ if(gaze_allowed&&gaze_active)launcher_outside_at={}; if(launcher_open&&launcher_outside_at.time_since_epoch().count()&&now>=launcher_outside_at){launcher_outside_at={};if(!keyboard_open&&!launcher_dock_interaction())launcher_back();} } -static void commands(){std::deque commands;{std::lock_guard lock(queue_mutex);commands.swap(queue);}for(const auto& c:commands){try{auto kind=c.value("kind","");if(kind=="cast.framing.open"){auto key="framely.framing";if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto geometry=c.at("geometry");RecordingRegionPlane plane;auto view=geometry.at("view");if(!recording_region_plane(view,geometry.at("referenceTan"),0,0,1,plane))continue;auto& v=create_view(key,"Recording frame",origin+"/cast/framing",false,1600,1600);v.region_editor_id=geometry.at("id");overlays->SetOverlayWidthInMeters(v.overlay,plane.width);overlays->SetOverlayTransformTrackedDeviceRelative(v.overlay,vr::k_unTrackedDeviceIndex_Hmd,&plane.transform);menu_open=false;hide_launcher(false);}else if(kind=="cast.framing.close"){if(views.count("framely.framing")&&views.at("framely.framing")->region_editor_id==c.value("id",std::string{}))close_view(*views.at("framely.framing"));}else if(kind=="feedback.settings"){ui_haptics=c.value("haptics",true);ui_sounds=c.value("sounds",false);if(!ui_sounds)ui_audio.silence();}else if(kind=="file.dialog.complete"){for(auto& [key,view]:views){auto& v=*view;if(!v.file_dialog||v.file_dialog_id!=c.value("id",std::string{}))continue;auto paths=c.value("paths",json::array());if(paths.empty()){cancel_file_dialog(v);break;}if((!v.file_multiple&&paths.size()!=1)||paths.size()>64)break;auto* callback=v.file_dialog;v.file_dialog=nullptr;v.file_dialog_id.clear();auto list=cef_string_list_alloc();for(const auto& path:paths){cef_string_t value{};str(value,path.get());cef_string_list_append(list,&value);cef_string_utf16_clear(&value);}callback->cont(callback,list);cef_string_list_free(list);callback->base.release(&callback->base);break;}}else if(kind=="gaze.correction"){gaze_correction=c.value("calibration",json(nullptr));}else if(kind=="gaze.target"){gaze_target_id=c.value("id",std::string{});gaze_target_u=c.value("x",.5f);gaze_target_v=1-c.value("y",.5f);gaze_preview=c.value("preview",json(nullptr));gaze_target_cached=false;}else if(kind=="launcher.calibrate"){gaze_target_id.clear();gaze_target_cached=false;gaze_preview=nullptr;launcher_calibrating=c.value("active",true);if(launcher_calibrating&&!launcher_open){if(!views.count("launcher"))create_view("launcher","Framely launcher",origin+"/launcher#calibrate",true,1440,800);toggle_entry(true);}if(views.count("launcher")){auto& v=*views.at("launcher");auto* frame=v.browser->get_main_frame(v.browser);execute(frame,std::string("location.hash=")+(launcher_calibrating?"'#calibrate'":"''")+";window.dispatchEvent(new Event('hashchange'))");frame->base.release(&frame->base);}}else if(kind=="launcher.close"){if(c.contains("launcherSession")&&!c["launcherSession"].is_null()&&c["launcherSession"]!=launcher_session)continue;bool handoff=c.value("handoff",false);hide_launcher(!handoff,handoff);}else if(kind=="menu.open"){hide_launcher(false);auto& menu=create_view("menu","Framely",origin+"/",false,600,840);navigate(menu,"#plugin:"+c.at("plugin").get());menu_open=true;launcher_open=false;menu_button.opened(std::chrono::steady_clock::now());}else if(kind=="haptic"){auto key=c.value("view","");if(views.count(key)&&views.at(key)->shown&&!views.at(key)->closing){auto& view=*views.at(key);bool activation=c.value("feedback","hover")=="activate";if(activation&&key=="launcher"&&gaze_active&&launcher_gaze_allowed()){auto now=std::chrono::steady_clock::now();if(now-view.activation_at>=std::chrono::milliseconds(80)){view.activation_at=now;if(ui_sounds)ui_audio.play(true);}}else hover_haptic(view.overlay,activation?view.activation_at:view.haptic_at,activation);}}else if(kind=="manager.open"){hide_launcher(false);if(views.count("framely.manager")&&views.at("framely.manager")->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto page=c.value("page",std::string{});auto& manager=create_view("framely.manager","Framely",origin+"/manager"+((manager_navigation_page(page)||page.rfind("plugin:",0)==0)?"#"+page:""),true,1440,810);if(page.rfind("plugin:",0)==0)navigate(manager,"#"+page);if(manager_navigation_page(page)){auto* frame=manager.browser->get_main_frame(manager.browser);execute(frame,"location.hash="+json(page).dump()+";window.dispatchEvent(new Event(\"hashchange\"))");frame->base.release(&frame->base);}menu_open=false;launcher_open=false;}else if(kind=="agreement.decline"){auto key=c.value("view","");if(key=="menu"){menu_open=false;if(views.count("menu")){show(*views.at("menu"),false);views.at("menu")->browser->reload(views.at("menu")->browser);}}else if(views.count(key))close_view(*views.at(key));}else if(kind=="manager.close"){if(views.count("framely.manager"))close_view(*views.at("framely.manager"));}else if(kind=="menu.close"){menu_open=false;}else if(kind=="apk.permission.open"){auto key="framely.apk.permission";if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto& v=create_view(key,"Android permission",origin+"/apk/permission",false,1000,560,"",.85f);anchor_permission(v);}else if(kind=="apk.permission.close"){if(views.count("framely.apk.permission"))close_view(*views.at("framely.apk.permission"));}else if(kind=="cast.window"){auto id=c.at("id").get();auto key="framely.cast."+id;if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}int width=c.value("width",640),height=c.value("height",240);if(width<160||width>1920||height<112||height>8192)continue;auto& v=create_view(key,"Framely casting",origin+"/cast/window/"+id,false,width,height,"",2.f);v.cast_frame.path=c.at("frame").get();if(v.width!=width||v.height!=height){v.width=width;v.height=height;v.pixels.clear();v.texture_ready[0]=v.texture_ready[1]=false;v.host->was_resized(v.host);vr::HmdVector2_t scale{{float(width),float(height)}};overlays->SetOverlayMouseScale(v.overlay,&scale);}menu_open=false;launcher_open=false;}else if(kind=="cast.close"){auto key="framely.cast."+c.at("id").get();if(views.count(key))close_view(*views.at(key));}else if(kind=="window.open"){hide_launcher(false);std::string plugin=c.at("plugin"),win=c.at("window");auto key="framely.window."+plugin+"."+win;if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto spec=c.at("spec");auto& opened=create_view("framely.window."+plugin+"."+win,spec.at("title"),origin+"/window/"+plugin+"/"+win,spec.value("dockIcon",false),spec.value("width",1600),spec.value("height",900),plugin,spec.value("widthMeters",3.f));plugin_thumbnail(opened,spec.value("iconData",std::string{}));menu_open=false;launcher_open=false;}else if(kind=="window.close"){auto key="framely.window."+c.at("plugin").get()+"."+c.at("window").get();if(views.count(key))close_view(*views.at(key));}else if(kind=="plugin.disabled"){for(auto&[key,v]:views)if(v->plugin==c.at("plugin"))close_view(*v);}else if(kind=="notification.changed"){bool requested=c.value("visible",false);if(!requested){if(views.count("notifications")){auto& v=*views.at("notifications");v.notification_requested=false;v.notification_content=false;show(v,false);}continue;}auto& v=create_view("notifications","Framely notifications",origin+"/notifications",false,600,360);v.notification_requested=true;anchor_notification(v);v.host->was_hidden(v.host,0);v.host->invalidate(v.host,PET_VIEW);}else if(kind=="keyboard"){auto key=c.value("view","menu");if(!views.count(key))continue;auto& v=*views.at(key);if(v.closing)continue;close_keyboard();v.host->set_focus(v.host,1);keyboard_frame=v.browser->get_focused_frame(v.browser);keyboard_view=key;keyboard_target=v.overlay;++keyboard_token;auto existing=c.value("existing","");if(key=="menu")overlays->SetOverlayFlag(v.overlay,vr::VROverlayFlags_WantsModalBehavior,false);auto e=overlays->ShowKeyboardForOverlay(v.overlay,c.value("password",false)?vr::k_EGamepadTextInputModePassword:vr::k_EGamepadTextInputModeNormal,c.value("multiline",false)?vr::k_EGamepadTextInputLineModeMultipleLines:vr::k_EGamepadTextInputLineModeSingleLine,vr::KeyboardFlag_Modal|vr::KeyboardFlag_Minimal|vr::KeyboardFlag_ShowArrowKeys,"Framely",4096,existing.c_str(),keyboard_token);keyboard_open=e==vr::VROverlayError_None;keyboard_menu.begin(keyboard_open&&key=="menu");keyboard_state(keyboard_open);if(!keyboard_open)close_keyboard();}else if(kind=="notifications.empty"){if(views.count("notifications")){views.at("notifications")->notification_requested=false;views.at("notifications")->notification_content=false;show(*views.at("notifications"),false);}}}catch(const std::exception& e){std::cerr<<"Native command error: "< commands;{std::lock_guard lock(queue_mutex);commands.swap(queue);}for(const auto& c:commands){try{auto kind=c.value("kind","");if(kind=="cast.framing.open"){auto key="framely.framing";if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto geometry=c.at("geometry");RecordingRegionPlane plane;auto view=geometry.at("view");if(!recording_region_plane(view,geometry.at("referenceTan"),0,0,1,plane))continue;auto& v=create_view(key,"Recording frame",origin+"/cast/framing",false,1600,1600);v.region_editor_id=geometry.at("id");v.region_geometry=geometry;update_framing_placement(v);menu_open=false;hide_launcher(false);}else if(kind=="cast.framing.reference"){auto found=views.find("framely.framing");if(found!=views.end()&&!found->second->closing&&found->second->region_editor_id==c.value("id",std::string{})){double cx=c.at("centerTanX"),cy=c.at("centerTanY");if(std::isfinite(cx)&&std::isfinite(cy)&&std::abs(cx)<=1.733&&std::abs(cy)<=1.733){found->second->region_geometry["view"]["centerTanX"]=cx;found->second->region_geometry["view"]["centerTanY"]=cy;}}}else if(kind=="cast.framing.close"){if(views.count("framely.framing")&&views.at("framely.framing")->region_editor_id==c.value("id",std::string{}))close_view(*views.at("framely.framing"));}else if(kind=="feedback.settings"){ui_haptics=c.value("haptics",true);ui_sounds=c.value("sounds",false);if(!ui_sounds)ui_audio.silence();}else if(kind=="file.dialog.complete"){for(auto& [key,view]:views){auto& v=*view;if(!v.file_dialog||v.file_dialog_id!=c.value("id",std::string{}))continue;auto paths=c.value("paths",json::array());if(paths.empty()){cancel_file_dialog(v);break;}if((!v.file_multiple&&paths.size()!=1)||paths.size()>64)break;auto* callback=v.file_dialog;v.file_dialog=nullptr;v.file_dialog_id.clear();auto list=cef_string_list_alloc();for(const auto& path:paths){cef_string_t value{};str(value,path.get());cef_string_list_append(list,&value);cef_string_utf16_clear(&value);}callback->cont(callback,list);cef_string_list_free(list);callback->base.release(&callback->base);break;}}else if(kind=="gaze.correction"){gaze_correction=c.value("calibration",json(nullptr));}else if(kind=="gaze.target"){gaze_target_id=c.value("id",std::string{});gaze_target_u=c.value("x",.5f);gaze_target_v=1-c.value("y",.5f);gaze_preview=c.value("preview",json(nullptr));gaze_target_cached=false;}else if(kind=="launcher.calibrate"){gaze_target_id.clear();gaze_target_cached=false;gaze_preview=nullptr;launcher_calibrating=c.value("active",true);if(launcher_calibrating&&!launcher_open){if(!views.count("launcher"))create_view("launcher","Framely launcher",origin+"/launcher#calibrate",true,1440,800);toggle_entry(true);}if(views.count("launcher")){auto& v=*views.at("launcher");auto* frame=v.browser->get_main_frame(v.browser);execute(frame,std::string("location.hash=")+(launcher_calibrating?"'#calibrate'":"''")+";window.dispatchEvent(new Event('hashchange'))");frame->base.release(&frame->base);}}else if(kind=="launcher.close"){if(c.contains("launcherSession")&&!c["launcherSession"].is_null()&&c["launcherSession"]!=launcher_session)continue;bool handoff=c.value("handoff",false);hide_launcher(!handoff,handoff);}else if(kind=="menu.open"){hide_launcher(false);auto& menu=create_view("menu","Framely",origin+"/",false,600,840);navigate(menu,"#plugin:"+c.at("plugin").get());menu_open=true;launcher_open=false;menu_button.opened(std::chrono::steady_clock::now());}else if(kind=="haptic"){auto key=c.value("view","");if(views.count(key)&&views.at(key)->shown&&!views.at(key)->closing){auto& view=*views.at(key);bool activation=c.value("feedback","hover")=="activate";if(activation&&key=="launcher"&&gaze_active&&launcher_gaze_allowed()){auto now=std::chrono::steady_clock::now();if(now-view.activation_at>=std::chrono::milliseconds(80)){view.activation_at=now;if(ui_sounds)ui_audio.play(true);}}else hover_haptic(view.overlay,activation?view.activation_at:view.haptic_at,activation);}}else if(kind=="manager.open"){hide_launcher(false);if(views.count("framely.manager")&&views.at("framely.manager")->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto page=c.value("page",std::string{});auto& manager=create_view("framely.manager","Framely",origin+"/manager"+((manager_navigation_page(page)||page.rfind("plugin:",0)==0)?"#"+page:""),true,1440,810);if(page.rfind("plugin:",0)==0)navigate(manager,"#"+page);if(manager_navigation_page(page)){auto* frame=manager.browser->get_main_frame(manager.browser);execute(frame,"location.hash="+json(page).dump()+";window.dispatchEvent(new Event(\"hashchange\"))");frame->base.release(&frame->base);}menu_open=false;launcher_open=false;}else if(kind=="agreement.decline"){auto key=c.value("view","");if(key=="menu"){menu_open=false;if(views.count("menu")){show(*views.at("menu"),false);views.at("menu")->browser->reload(views.at("menu")->browser);}}else if(views.count(key))close_view(*views.at(key));}else if(kind=="manager.close"){if(views.count("framely.manager"))close_view(*views.at("framely.manager"));}else if(kind=="menu.close"){menu_open=false;}else if(kind=="apk.permission.open"){auto key="framely.apk.permission";if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto& v=create_view(key,"Android permission",origin+"/apk/permission",false,1000,560,"",.85f);anchor_permission(v);}else if(kind=="apk.permission.close"){if(views.count("framely.apk.permission"))close_view(*views.at("framely.apk.permission"));}else if(kind=="cast.window"){auto id=c.at("id").get();auto key="framely.cast."+id;if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}int width=c.value("width",640),height=c.value("height",240);if(width<160||width>1920||height<112||height>8192)continue;auto& v=create_view(key,"Framely casting",origin+"/cast/window/"+id,false,width,height,"",2.f);v.cast_frame.path=c.at("frame").get();if(v.width!=width||v.height!=height){v.width=width;v.height=height;v.pixels.clear();v.texture_ready[0]=v.texture_ready[1]=false;v.host->was_resized(v.host);vr::HmdVector2_t scale{{float(width),float(height)}};overlays->SetOverlayMouseScale(v.overlay,&scale);}menu_open=false;launcher_open=false;}else if(kind=="cast.close"){auto key="framely.cast."+c.at("id").get();if(views.count(key))close_view(*views.at(key));}else if(kind=="window.open"){hide_launcher(false);std::string plugin=c.at("plugin"),win=c.at("window");auto key="framely.window."+plugin+"."+win;if(views.count(key)&&views.at(key)->closing){std::lock_guard lock(queue_mutex);queue.push_back(c);continue;}auto spec=c.at("spec");auto& opened=create_view("framely.window."+plugin+"."+win,spec.at("title"),origin+"/window/"+plugin+"/"+win,spec.value("dockIcon",false),spec.value("width",1600),spec.value("height",900),plugin,spec.value("widthMeters",3.f));plugin_thumbnail(opened,spec.value("iconData",std::string{}));menu_open=false;launcher_open=false;}else if(kind=="window.close"){auto key="framely.window."+c.at("plugin").get()+"."+c.at("window").get();if(views.count(key))close_view(*views.at(key));}else if(kind=="plugin.disabled"){for(auto&[key,v]:views)if(v->plugin==c.at("plugin"))close_view(*v);}else if(kind=="notification.changed"){bool requested=c.value("visible",false);if(!requested){if(views.count("notifications")){auto& v=*views.at("notifications");v.notification_requested=false;v.notification_content=false;show(v,false);}continue;}auto& v=create_view("notifications","Framely notifications",origin+"/notifications",false,600,360);v.notification_requested=true;anchor_notification(v);v.host->was_hidden(v.host,0);v.host->invalidate(v.host,PET_VIEW);}else if(kind=="keyboard"){auto key=c.value("view","menu");if(!views.count(key))continue;auto& v=*views.at(key);if(v.closing)continue;close_keyboard();v.host->set_focus(v.host,1);keyboard_frame=v.browser->get_focused_frame(v.browser);keyboard_view=key;keyboard_target=v.overlay;++keyboard_token;auto existing=c.value("existing","");if(key=="menu")overlays->SetOverlayFlag(v.overlay,vr::VROverlayFlags_WantsModalBehavior,false);auto e=overlays->ShowKeyboardForOverlay(v.overlay,c.value("password",false)?vr::k_EGamepadTextInputModePassword:vr::k_EGamepadTextInputModeNormal,c.value("multiline",false)?vr::k_EGamepadTextInputLineModeMultipleLines:vr::k_EGamepadTextInputLineModeSingleLine,vr::KeyboardFlag_Modal|vr::KeyboardFlag_Minimal|vr::KeyboardFlag_ShowArrowKeys,"Framely",4096,existing.c_str(),keyboard_token);keyboard_open=e==vr::VROverlayError_None;keyboard_menu.begin(keyboard_open&&key=="menu");keyboard_state(keyboard_open);if(!keyboard_open)close_keyboard();}else if(kind=="notifications.empty"){if(views.count("notifications")){views.at("notifications")->notification_requested=false;views.at("notifications")->notification_content=false;show(*views.at("notifications"),false);}}}catch(const std::exception& e){std::cerr<<"Native command error: "< lock(queue_mutex);recording_state=data.value("recording",json::object());recording_region_state=data.value("recordingRegion",json::object());framing_editor_state=data.value("framingEditor",json(nullptr));if(!recording_state.is_object())recording_state=json::object();auto feedback=data.value("feedback",json::object());if(!feedback.is_object())feedback=json::object();if(feedback!=last_feedback){queue.push_back({{"kind","feedback.settings"},{"haptics",feedback.value("haptics",true)},{"sounds",feedback.value("sounds",false)}});last_feedback=feedback;}auto calibration=data.value("gazeCalibration",json(nullptr));if(!gaze_loaded||calibration!=last_gaze){queue.push_back({{"kind","gaze.correction"},{"calibration",calibration}});last_gaze=calibration;gaze_loaded=true;}for(const auto& c:data.at("commands")){if(queue.size()<256)queue.push_back(c);}failures=0;}catch(const std::exception& e){if(++failures>=10){std::cerr<<"Agent connection lost: "< lock(queue_mutex);recording=recording_state;region=recording_region_state;framing=framing_editor_state;}if(auto editor=views.find("framely.framing");editor!=views.end()&&framing_editor_should_close(framing,editor->second->region_editor_id))close_view(*editor->second);recording_indicator.tick(overlays,recording.value("active",false)?recording.value("id",std::string{}):std::string{},recording.value("elapsedSeconds",uint64_t(0)));recording_region_overlay.tick(overlays,region);vr::VREvent_t global{};while(vr_system->PollNextEvent(&global,sizeof(global))){if(global.eventType==vr::VREvent_Quit){vr_system->AcknowledgeQuit_Exiting();running=0;}observe_dashboard(global);keyboard_event(global);}launcher_priority_active.store(launcher_open&&launcher_prioritize_ui.load());vr::VROverlayHandle_t dock=0;vr::HmdVector2_t mouse_scale{};bool visible=!overlays->FindOverlay("valve.steam.gamepadui.bar",&dock)&&overlays->IsDashboardVisible()&&overlays->IsOverlayVisible(dock)&&!overlays->GetOverlayMouseScale(dock,&mouse_scale); + while(running){auto frame_start=std::chrono::steady_clock::now();cef_do_message_loop_work();commands();json recording,region,framing;{std::lock_guard lock(queue_mutex);recording=recording_state;region=recording_region_state;framing=framing_editor_state;}if(auto editor=views.find("framely.framing");editor!=views.end()&&framing_editor_should_close(framing,editor->second->region_editor_id))close_view(*editor->second);recording_indicator.tick(overlays,recording.value("active",false)?recording.value("id",std::string{}):std::string{},recording.value("elapsedSeconds",uint64_t(0)));if(auto editor=views.find("framely.framing");editor!=views.end())update_framing_placement(*editor->second);if(region.value("visible",false)&®ion.contains("view")){auto presentation=recording_region_presentation(region["view"]);recording_region_overlay.tick(overlays,region,presentation.distance,&presentation.view);}else recording_region_overlay.tick(overlays,region);vr::VREvent_t global{};while(vr_system->PollNextEvent(&global,sizeof(global))){if(global.eventType==vr::VREvent_Quit){vr_system->AcknowledgeQuit_Exiting();running=0;}observe_dashboard(global);keyboard_event(global);}launcher_priority_active.store(launcher_open&&launcher_prioritize_ui.load());vr::VROverlayHandle_t dock=0;vr::HmdVector2_t mouse_scale{};bool visible=!overlays->FindOverlay("valve.steam.gamepadui.bar",&dock)&&overlays->IsDashboardVisible()&&overlays->IsOverlayVisible(dock)&&!overlays->GetOverlayMouseScale(dock,&mouse_scale); if(visible&&(!was_visible||dock!=last_dock)){uv={};anchored_at={};sampled_at={};}last_dock=dock; // Width and aspect are only ray-solver seeds; physical size comes from intersections. float width=0,height=0;bool fresh=visible&&!overlays->GetOverlayWidthInMeters(dock,&width)&&std::isfinite(width)&&width>0&&width<10&&std::isfinite(mouse_scale.v[0])&&std::isfinite(mouse_scale.v[1])&&mouse_scale.v[0]>0&&mouse_scale.v[1]>0; diff --git a/native/recording_region.h b/native/recording_region.h index c2b4a24..c0cd68f 100644 --- a/native/recording_region.h +++ b/native/recording_region.h @@ -11,11 +11,10 @@ inline void recording_region_foreground(vr::IVROverlay* overlays,vr::VROverlayHa overlays->SetOverlayFlag(handle,vr::VROverlayFlags_SortWithNonSceneOverlays,true); overlays->SetOverlaySortOrder(handle,1000); } -inline bool recording_region_plane(const RegionJson& view,double tan,double cx,double cy,double aspect,RecordingRegionPlane& plane){ +inline bool recording_region_plane(const RegionJson& view,double tan,double cx,double cy,double aspect,RecordingRegionPlane& plane,double distance=2.){ try{ - if(!std::isfinite(tan)||tan<=0||!std::isfinite(cx)||!std::isfinite(cy)||!std::isfinite(aspect)||aspect<=0)return false; + if(!std::isfinite(tan)||tan<=0||!std::isfinite(cx)||!std::isfinite(cy)||!std::isfinite(aspect)||aspect<=0||!std::isfinite(distance)||distance<=0)return false; const auto& basis=view.at("captureToHead"); - constexpr double distance=2.; for(int i=0;i<3;i++){ double translation=basis.at(i).at(3).get(); for(int j=0;j<3;j++){ @@ -46,16 +45,16 @@ inline std::vector recording_region_pixels(int w,int h,const std: return pixels; } class RecordingRegionOverlay { - vr::IVROverlay* overlays=nullptr;vr::VROverlayHandle_t handle=0;std::string last; + vr::IVROverlay* overlays=nullptr;vr::VROverlayHandle_t handle=0;std::string last,last_pixels; public: - void destroy(){if(handle&&overlays)overlays->DestroyOverlay(handle);handle=0;last.clear();} - void tick(vr::IVROverlay* api,const RegionJson& state){ + void destroy(){if(handle&&overlays)overlays->DestroyOverlay(handle);handle=0;last.clear();last_pixels.clear();} + void tick(vr::IVROverlay* api,const RegionJson& state,double distance=2.,const RegionJson* live_view=nullptr){ overlays=api; if(!state.value("visible",false)){destroy();return;} - auto color=recording_region_color();auto serialized=state.dump()+std::string(color.begin(),color.end());if(serialized==last)return; + auto color=recording_region_color();auto serialized=state.dump()+std::string(color.begin(),color.end())+std::to_string(distance)+(live_view?live_view->dump():"");if(serialized==last)return; try{ - const auto& view=state.at("view");double aspect=view.at("aspect"); - RecordingRegionPlane plane;if(!recording_region_plane(view,view.at("tanHalfHorizontal"),view.at("centerTanX"),view.at("centerTanY"),aspect,plane)){destroy();return;} + const auto& view=live_view?*live_view:state.at("view");double aspect=view.at("aspect"); + RecordingRegionPlane plane;if(!recording_region_plane(view,view.at("tanHalfHorizontal"),view.at("centerTanX"),view.at("centerTanY"),aspect,plane,distance)){destroy();return;} if(!handle){ if(overlays->CreateOverlay("framely.recording.region","Framely recording region",&handle))return; overlays->SetOverlayInputMethod(handle,vr::VROverlayInputMethod_None); @@ -71,11 +70,15 @@ public: // Bound raw-overlay traffic independently of recording resolution. // Square texels are explicitly stretched to the exact capture aspect. width=height=512;plane.width*=512.f/508.f; - auto pixels=recording_region_pixels(width,height,color); auto error=overlays->SetOverlayWidthInMeters(handle,plane.width); if(!error)error=overlays->SetOverlayTexelAspect(handle,aspect); if(!error)error=overlays->SetOverlayTransformTrackedDeviceRelative(handle,vr::k_unTrackedDeviceIndex_Hmd,&plane.transform); - if(!error)error=overlays->SetOverlayRaw(handle,pixels.data(),width,height,4); + const auto pixel_key=std::string(color.begin(),color.end()); + if(!error&&(last_pixels.empty()||pixel_key!=last_pixels)){ + auto pixels=recording_region_pixels(width,height,color); + error=overlays->SetOverlayRaw(handle,pixels.data(),width,height,4); + if(!error)last_pixels=pixel_key; + } if(error){destroy();return;}overlays->ShowOverlay(handle);last=serialized; }catch(...){destroy();} } diff --git a/native/recording_region_depth.h b/native/recording_region_depth.h new file mode 100644 index 0000000..bda8c47 --- /dev/null +++ b/native/recording_region_depth.h @@ -0,0 +1,90 @@ +#pragma once +#include +#include +#include +#include "openvr.h" + +// Pure geometry only. Pose validity/freshness is owned by the caller: a +// numerically rigid matrix cannot reveal whether its tracking sample is old. +namespace recording_region_depth_detail { +using Point=std::array; +inline double dot(const Point& a,const Point& b){return a[0]*b[0]+a[1]*b[1]+a[2]*b[2];} +inline Point translation(const vr::HmdMatrix34_t& m){return {m.m[0][3],m.m[1][3],m.m[2][3]};} +inline Point rotate(const vr::HmdMatrix34_t& m,const Point& p){ + Point result{};for(int i=0;i<3;i++)for(int j=0;j<3;j++)result[i]+=m.m[i][j]*p[j];return result; +} +inline Point add(const Point& a,const Point& b){return {a[0]+b[0],a[1]+b[1],a[2]+b[2]};} +inline Point subtract(const Point& a,const Point& b){return {a[0]-b[0],a[1]-b[1],a[2]-b[2]};} +inline bool rigid(const vr::HmdMatrix34_t& m){ + for(const auto& row:m.m)for(float v:row)if(!std::isfinite(v))return false; + for(int i=0;i<3;i++)for(int j=0;j<3;j++){ + double product=0;for(int k=0;k<3;k++)product+=double(m.m[k][i])*m.m[k][j]; + if(std::abs(product-(i==j?1.:0.))>0.01)return false; + } + const double determinant=m.m[0][0]*(double(m.m[1][1])*m.m[2][2]-double(m.m[1][2])*m.m[2][1]) + -m.m[0][1]*(double(m.m[1][0])*m.m[2][2]-double(m.m[1][2])*m.m[2][0]) + +m.m[0][2]*(double(m.m[1][0])*m.m[2][1]-double(m.m[1][1])*m.m[2][0]); + return std::abs(determinant-1.)<=0.01; +} +} + +// Refresh just the eye origin. Capture's basis already encodes the LUT's ray +// convention and must not receive the physical-panel rotation a second time. +inline bool recording_region_refresh_eye_origin(vr::EVREye eye, + const vr::HmdMatrix34_t& left_to_head,const vr::HmdMatrix34_t& right_to_head, + vr::HmdMatrix34_t& capture_to_head){ + using namespace recording_region_depth_detail; + if((eye!=vr::Eye_Left&&eye!=vr::Eye_Right)||!rigid(left_to_head)||!rigid(right_to_head)||!rigid(capture_to_head))return false; + const auto left=translation(left_to_head),right=translation(right_to_head),baseline=subtract(right,left); + const double ipd=std::sqrt(dot(baseline,baseline)); + // Broad plausibility guards, not a replacement for live tracking validity. + if(ipd<0.04||ipd>0.09||baseline[0]<0.8*ipd||std::abs(baseline[1])>0.02||std::abs(baseline[2])>0.02 + ||dot(left,left)>0.25*0.25||dot(right,right)>0.25*0.25)return false; + const auto& selected=eye==vr::Eye_Left?left_to_head:right_to_head; + for(int i=0;i<3;i++)capture_to_head.m[i][3]=selected.m[i][3]; + return true; +} + +// Intersect the selected capture-eye center ray with a known UI window plane. +// The returned parameter D is measured along capture -Z, not Euclidean ray +// length: X = eye + D * basis * (centerTanX,-centerTanY,-1). +// It therefore plugs directly into a guide whose width is 2*tanHalfFov*D. +inline std::optional recording_region_window_depth( + const vr::HmdMatrix34_t& hmd_to_world,const vr::HmdMatrix34_t& window_to_world, + const vr::HmdMatrix34_t& capture_to_head,double center_tan_x,double center_tan_y){ + using namespace recording_region_depth_detail; + if(!rigid(hmd_to_world)||!rigid(window_to_world)||!rigid(capture_to_head) + ||!std::isfinite(center_tan_x)||!std::isfinite(center_tan_y) + ||std::abs(center_tan_x)>10.||std::abs(center_tan_y)>10. + ||dot(translation(capture_to_head),translation(capture_to_head))>0.25*0.25)return std::nullopt; + const Point origin=add(translation(hmd_to_world),rotate(hmd_to_world,translation(capture_to_head))); + const Point ray=rotate(hmd_to_world,rotate(capture_to_head,{center_tan_x,-center_tan_y,-1.})); + const Point normal={window_to_world.m[0][2],window_to_world.m[1][2],window_to_world.m[2][2]}; + const double denominator=dot(normal,ray); + if(std::abs(denominator)<1e-5*std::sqrt(dot(ray,ray)))return std::nullopt; + const double distance=dot(normal,subtract(translation(window_to_world),origin))/denominator; + if(!std::isfinite(distance)||distance<0.25||distance>6.)return std::nullopt; + return distance; +} + +// ComputeOverlayIntersection can supply an exact curved/tilted UI hit. Recover +// the same capture -Z parameter, rejecting unrelated/off-ray intersection data. +inline std::optional recording_region_hit_depth( + const vr::HmdMatrix34_t& hmd_to_world,const vr::HmdMatrix34_t& capture_to_head, + double center_tan_x,double center_tan_y,const vr::HmdVector3_t& world_hit){ + using namespace recording_region_depth_detail; + if(!rigid(hmd_to_world)||!rigid(capture_to_head) + ||!std::isfinite(center_tan_x)||!std::isfinite(center_tan_y) + ||std::abs(center_tan_x)>10.||std::abs(center_tan_y)>10. + ||dot(translation(capture_to_head),translation(capture_to_head))>0.25*0.25)return std::nullopt; + Point hit{};for(int i=0;i<3;i++){hit[i]=world_hit.v[i];if(!std::isfinite(hit[i]))return std::nullopt;} + const Point origin=add(translation(hmd_to_world),rotate(hmd_to_world,translation(capture_to_head))); + const Point ray=rotate(hmd_to_world,rotate(capture_to_head,{center_tan_x,-center_tan_y,-1.})); + const Point delta=subtract(hit,origin); + const double squared_length=dot(ray,ray),distance=dot(delta,ray)/squared_length; + if(!std::isfinite(distance)||distance<0.25||distance>6.)return std::nullopt; + const Point residual={delta[0]-distance*ray[0],delta[1]-distance*ray[1],delta[2]-distance*ray[2]}; + const double tolerance=1e-4+1e-4*distance*std::sqrt(squared_length); + if(dot(residual,residual)>tolerance*tolerance)return std::nullopt; + return distance; +} diff --git a/src/session.rs b/src/session.rs index ce2d355..5f76c8e 100644 --- a/src/session.rs +++ b/src/session.rs @@ -1091,6 +1091,16 @@ impl Agent { "cast.framing.get" => { let cast=self.casting.lock().unwrap();ensure!(cast.region.editing,"画幅调整窗口已关闭");cast.region.editor() }, + "cast.framing.reference" => { + let cast=self.casting.lock().unwrap(); + ensure!(cast.region.editing && p["id"]==cast.region.editor_id,"画幅调整窗口已关闭"); + let cx=p["centerX"].as_f64().context("Invalid reference center")?; + let cy=p["centerY"].as_f64().context("Invalid reference center")?; + ensure!([cx,cy].into_iter().all(|v|v.is_finite()&&(-60.0..=60.0).contains(&v)),"Invalid reference center"); + // Presentation-only depth probe; never persist a draft crop. + self.commands.lock().unwrap().push_back(json!({"kind":"cast.framing.reference","id":cast.region.editor_id,"centerTanX":cx.to_radians().tan(),"centerTanY":cy.to_radians().tan()})); + Ok(json!(true)) + }, "cast.framing.cancel" => { let mut cast=self.casting.lock().unwrap(); ensure!(p["id"]==cast.region.editor_id,"画幅调整窗口已关闭"); @@ -2703,6 +2713,13 @@ mod tests { } Ok::<_,anyhow::Error>(()) }); + assert!(agent.api(json!({"method":"cast.framing.reference","params":{"id":"old-test","centerX":0.,"centerY":0.}})).is_err()); + assert!(agent.api(json!({"method":"cast.framing.reference","params":{"id":"apply-test","centerX":61.,"centerY":0.}})).is_err()); + agent.api(json!({"method":"cast.framing.reference","params":{"id":"apply-test","centerX":4.5,"centerY":-3.}})).unwrap(); + let reference=agent.commands.lock().unwrap().back().unwrap().clone(); + assert_eq!(reference["kind"],"cast.framing.reference"); + assert!((reference["centerTanX"].as_f64().unwrap()-4.5_f64.to_radians().tan()).abs()<1e-12); + assert_eq!(agent.core("status",json!({})).unwrap()["database"]["casting"]["centerX"],0.); let result=agent.api(json!({"method":"cast.framing.apply","params":{"id":"apply-test","expectedEye":0,"width":1280,"height":960,"horizontalFov":60.,"centerX":0.,"centerY":0.}})); assert_eq!(result.is_err(),running); if let Err(error)=result {assert!(format!("{error:#}").contains("取景已保存，但重新启动串流失败"));} diff --git a/tests/cast_spatial_lifecycle.mjs b/tests/cast_spatial_lifecycle.mjs index ba46918..4473c64 100644 --- a/tests/cast_spatial_lifecycle.mjs +++ b/tests/cast_spatial_lifecycle.mjs @@ -33,7 +33,7 @@ const tick=()=>new Promise(resolve=>setImmediate(resolve)); function walk(node,visit){if(!node||typeof node!=='object')return;visit(node);for(const child of [node.props?.children].flat(Infinity))walk(child,visit);} function draw(){const tree=render(SpatialFraming);walk(tree,node=>{if(node.props?.ref){const panel=node.type==='section';node.props.ref.current={getBoundingClientRect:()=>({width:panel?340:1600,height:panel?300:1600})};}});flush();return tree;} function button(tree,label){let found;walk(tree,node=>{if(node.type==='button'&&node.props.children===label)found=node;});assert.ok(found,label);return found;} -async function mount(finish){reset();setHandler((method,params)=>method==='status'?{database:{language:'zh-CN'}}:method==='language.list'?{packs:[]}:method==='cast.framing.get'?geometry:finish(method,params));for(let i=0;i<4;i++){draw();await tick();}return draw();} +async function mount(finish){reset();setHandler((method,params)=>method==='status'?{database:{language:'zh-CN'}}:method==='language.list'?{packs:[]}:method==='cast.framing.get'?geometry:method==='cast.framing.reference'?true:finish(method,params));for(let i=0;i<4;i++){draw();await tick();}return draw();} let resolveApply; let tree=await mount(method=>{assert.equal(method,'cast.framing.apply');return new Promise(resolve=>{resolveApply=resolve;});}); @@ -56,4 +56,18 @@ tree=await mount(()=>{throw Error('Saved, but restart failed');}); setHandler(method=>{if(method==='cast.framing.apply')throw Error('Saved, but restart failed');if(method==='cast.framing.get')throw Error('画幅调整窗口已关闭');throw Error(method);}); button(tree,'应用取景').props.onClick();await tick();assert.equal(draw(),null,'post-save restart failure cannot revive a closed editor'); -console.log('PASS: apply/cancel close immediately, pending controls are inert, save failures preserve editing, post-save errors cannot revive stale ids'); +const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); +function frame(tree){let found;walk(tree,node=>{if(node.props?.spatial&&node.props?.settings&&node.props?.onChange)found=node;});assert.ok(found,'framing editor');return found;} +tree=await mount(()=>true);await delay(120); +let references=calls.filter(call=>call.method==='cast.framing.reference'); +assert.equal(references.length,1);assert.deepEqual(references[0].params,{id:geometry.id,centerX:0,centerY:0}); +frame(tree).props.onChange({centerX:5,centerY:-3});tree=draw(); +frame(tree).props.onChange({centerX:6,centerY:-4});tree=draw();await delay(120); +references=calls.filter(call=>call.method==='cast.framing.reference'); +assert.equal(references.length,2,'rapid committed centers are debounced'); +assert.ok(Math.abs(references[1].params.centerX-6)<1e-8&&Math.abs(references[1].params.centerY+4)<1e-8,'live draft is sent in degrees'); +assert.ok(references[0].signal.aborted,'previous reference request is canceled after a new center'); +frame(tree).props.onChange({centerX:7});tree=draw(); +button(tree,'取消').props.onClick();await tick();assert.equal(draw(),null);await delay(120); +assert.equal(calls.filter(call=>call.method==='cast.framing.reference').length,2,'closing cancels the pending depth-probe update'); +console.log('PASS: terminal editor lifecycle, error recovery, debounced live-center depth updates and closing cancellation'); diff --git a/tests/recording_region.cpp b/tests/recording_region.cpp index 6398050..78edf0c 100644 --- a/tests/recording_region.cpp +++ b/tests/recording_region.cpp @@ -3,16 +3,16 @@ #include #include int main(){ - for(double roll:{-.2,0.,.2})for(double yaw:{-.3,0.,.3})for(double aspect:{4./3,16./9,9./16,1.}){ + for(double distance:{.5,1.,2.,4.})for(double roll:{-.2,0.,.2})for(double yaw:{-.3,0.,.3})for(double aspect:{4./3,16./9,9./16,1.}){ double c=cos(roll),s=sin(roll),cy=cos(yaw),sy=sin(yaw); RegionJson view={{"captureToHead",{{c*cy,-s, c*sy,-.032},{s*cy,c,s*sy,.003},{-sy,0,cy,.01}}}}; const double tan=.8,cx=.15,center_y=-.08; - RecordingRegionPlane plane;assert(recording_region_plane(view,tan,cx,center_y,aspect,plane)); + RecordingRegionPlane plane;assert(recording_region_plane(view,tan,cx,center_y,aspect,plane,distance)); // Undo the exported head basis: each overlay corner must be exactly the // tangent ray consumed at the corresponding output-LUT corner. for(double u:{0.,.25,1.})for(double v:{0.,.75,1.}){ double world[3];for(int i=0;i<3;i++)world[i]=plane.transform.m[i][3]+plane.transform.m[i][0]*(u-.5)*plane.width+plane.transform.m[i][1]*(.5-v)*plane.width/aspect-view["captureToHead"][i][3].get(); - double ray[3]{};for(int j=0;j<3;j++)for(int i=0;i<3;i++)ray[j]+=view["captureToHead"][i][j].get()*world[i]/2.; + double ray[3]{};for(int j=0;j<3;j++)for(int i=0;i<3;i++)ray[j]+=view["captureToHead"][i][j].get()*world[i]/distance; assert(std::abs(ray[0]-(cx+(2*u-1)*tan))<1e-6); assert(std::abs(ray[1]-(-center_y+(1-2*v)*tan/aspect))<1e-6); assert(std::abs(ray[2]+1.)<1e-6); diff --git a/tests/recording_region_depth.cpp b/tests/recording_region_depth.cpp new file mode 100644 index 0000000..d124970 --- /dev/null +++ b/tests/recording_region_depth.cpp @@ -0,0 +1,86 @@ +#include "recording_region_depth.h" +#include +#include +#include +using namespace recording_region_depth_detail; +static vr::HmdMatrix34_t pose(double yaw=0,double pitch=0,double roll=0,Point t={0,0,0}){ + const double cy=cos(yaw),sy=sin(yaw),cp=cos(pitch),sp=sin(pitch),cr=cos(roll),sr=sin(roll); + vr::HmdMatrix34_t m{{ + {float(cy*cr+sy*sp*sr),float(-cy*sr+sy*sp*cr),float(sy*cp),float(t[0])}, + {float(cp*sr),float(cp*cr),float(-sp),float(t[1])}, + {float(-sy*cr+cy*sp*sr),float(sy*sr+cy*sp*cr),float(cy*cp),float(t[2])}}};return m; +} +static vr::HmdMatrix34_t multiply(const vr::HmdMatrix34_t& a,const vr::HmdMatrix34_t& b){ + vr::HmdMatrix34_t m{};for(int i=0;i<3;i++){ + for(int j=0;j<3;j++)for(int k=0;k<3;k++)m.m[i][j]+=a.m[i][k]*b.m[k][j]; + m.m[i][3]=a.m[i][3];for(int k=0;k<3;k++)m.m[i][3]+=a.m[i][k]*b.m[k][3]; + }return m; +} +static void near(double a,double b,double tolerance=1e-5){assert(std::abs(a-b)0.001); + } + } + // Changing IPD refreshes a stale exported origin without altering ray basis. + auto stale=pose(0,0,0,{0.032,0,0}); + assert(recording_region_refresh_eye_origin(vr::Eye_Right,pose(0,0,0,{-0.0375,0,0}),pose(0,0,0,{0.0375,0,0}),stale)); + near(stale.m[0][3],0.0375); + const auto identity=pose(),left=pose(0,0,0,{-0.032,0,0}),right=pose(0,0,0,{0.032,0,0}); + auto unchanged=identity; + assert(!recording_region_refresh_eye_origin(vr::Eye_Right,pose(0,0,0,{-0.1,0,0}),pose(0,0,0,{0.1,0,0}),unchanged)); + assert(!recording_region_refresh_eye_origin(vr::Eye_Right,right,left,unchanged)); + assert(unchanged.m[0][3]==0); + auto invalid=identity;invalid.m[0][0]=std::numeric_limits::quiet_NaN(); + assert(!recording_region_refresh_eye_origin(vr::Eye_Right,invalid,right,unchanged)); + assert(!recording_region_window_depth(invalid,pose(0,0,0,{0,0,-2}),identity,0,0)); + assert(!recording_region_window_depth(vr::HmdMatrix34_t{},pose(0,0,0,{0,0,-2}),identity,0,0)); + assert(!recording_region_window_depth(identity,pose(acos(-1.)/2,0,0,{0,0,-2}),identity,0,0)); + for(double z:{-0.1,-8.,1.})assert(!recording_region_window_depth(identity,pose(0,0,0,{0,0,z}),identity,0,0)); + assert(!recording_region_window_depth(identity,pose(0,0,0,{0,0,-2}),identity,std::numeric_limits::infinity(),0)); + // Eye Z matters: head-relative depth alone would return 2, but D=2.012. + const auto zdepth=recording_region_window_depth(identity,pose(0,0,0,{0,0,-2}),right,0,0); + assert(zdepth);near(*zdepth,2.0); // This fixture's right eye has Z=0. + const auto elevated=pose(0,0,0,{0.032,0,0.012}); + const auto exact=recording_region_window_depth(identity,pose(0,0,0,{0,0,-2}),elevated,0,0); + assert(exact);near(*exact,2.012); + for(double depth:{0.25,0.5,1.,2.,4.,6.}){ + const vr::HmdVector3_t hit{{0,0,float(-depth)}}; + const auto result=recording_region_hit_depth(identity,identity,0,0,hit);assert(result);near(*result,depth); + } + for(const auto& hit:std::array{{{{0.02f,0,-2}},{{0,0,2}},{{0,0,-8}},{{std::numeric_limits::quiet_NaN(),0,-2}}}}) + assert(!recording_region_hit_depth(identity,identity,0,0,hit)); + assert(!recording_region_hit_depth(invalid,identity,0,0,vr::HmdVector3_t{{0,0,-2}})); + std::cout<<"PASS: current eye origin, exact center-ray UI depth, both-eye parallax and invalid-pose guards\n"; +} diff --git a/tools/test-native.sh b/tools/test-native.sh index 0e47356..c3f7cef 100755 --- a/tools/test-native.sh +++ b/tools/test-native.sh @@ -3,7 +3,7 @@ set -euo pipefail base=$(cd -- "$(dirname -- "$0")/.." && pwd) cd "$base" mkdir -p target/tests -for test in gamepad_mapping gamepad_steam gamepad_direct entry_hold menu_geometry keyboard_state notification_geometry notification_badge ui_visibility plugin_navigation launcher_input launcher_gaze gaze_geometry launcher_fade launcher_pointer page_scroll paint_buffer render_timing launcher_priority recording_indicator recording_region framing_lifecycle; do +for test in gamepad_mapping gamepad_steam gamepad_direct entry_hold menu_geometry keyboard_state notification_geometry notification_badge ui_visibility plugin_navigation launcher_input launcher_gaze gaze_geometry launcher_fade launcher_pointer page_scroll paint_buffer render_timing launcher_priority recording_indicator recording_region framing_lifecycle recording_region_depth; do "${CXX:-g++}" -std=c++17 -O2 -pthread -Inative -Inative/vendor/openvr "tests/$test.cpp" -ldl -o "target/tests/$test" "target/tests/$test" done diff --git a/ui/src/cast-spatial-framing.tsx b/ui/src/cast-spatial-framing.tsx index b235832..4220cdc 100644 --- a/ui/src/cast-spatial-framing.tsx +++ b/ui/src/cast-spatial-framing.tsx @@ -45,6 +45,19 @@ export function SpatialFraming(){ useEffect(()=>{const close=()=>void finish(false);window.addEventListener('framely.back',close);return()=>window.removeEventListener('framely.back',close);},[geometry,busy]); const aspect=draft?draft.width/draft.height:4/3; const ready=!!geometry&&minimum.referenceTan===geometry.referenceTan; + useEffect(()=>{ + if(!ready||!geometry||!draft||busy||closed||finishing.current||finished.current)return; + const controller=new AbortController(),params={id:geometry.id,centerX:draft.centerX,centerY:draft.centerY}; + // Draft centers commit after a drag ends. Update only the native depth probe, + // not capture settings, and coalesce rapid keyboard/ratio/reset adjustments. + const timer=setTimeout(()=>{ + if(finishing.current||finished.current)return; + void api('cast.framing.reference',params,controller.signal).catch(()=>{}); + },100); + // This is presentation-only, so aborted/stale updates must not overwrite a + // real save error or revive an editor whose native window is closing. + return()=>{clearTimeout(timer);controller.abort();}; + },[ready,geometry?.id,draft?.centerX,draft?.centerY,busy,closed]); if(closed)return null; return
{draft&&geometry&&ready&&setError(t(message))}/>}