diff --git a/CMakeLists.txt b/CMakeLists.txt index 968a800..0df5b53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -266,6 +266,7 @@ add_library(mocktail_runtime STATIC src/runtime/crash_report_policy.cc src/runtime/discord_rpc.cc src/runtime/device_profile.cc + src/runtime/engine_pump_rest.cc src/runtime/environment.cc src/runtime/external_launch_broker.cc src/runtime/frame_rate_policy.cc diff --git a/include/runtime/engine_pump_rest.h b/include/runtime/engine_pump_rest.h new file mode 100644 index 0000000..eafae5c --- /dev/null +++ b/include/runtime/engine_pump_rest.h @@ -0,0 +1,65 @@ +#ifndef MOCKTAIL_RUNTIME_ENGINE_PUMP_REST_H_ +#define MOCKTAIL_RUNTIME_ENGINE_PUMP_REST_H_ + +#include + +namespace mocktail::runtime { + +// The engine posts its per-frame main-thread step at a time of its own +// choosing, with no signal the host can block on, so the host main thread +// polls nativeCallMessagesFromMainThread. An empty poll costs a few +// microseconds; polled back to back it burns a core. +// +// Between polls the thread rests for kEnginePumpRestNs. A post is then picked +// up at most kEnginePumpRestNs + timer slack + wake latency late, far under +// the engine's sub-millisecond slack. +// +// How it rests decides the clock. Under a power-saving governor the CPU clocks +// a bursty core down unless some core stays awake, and the engine's render +// thread then runs slower and drops frames. So: +// kSleep nanosleep. The core idles. Correct only when the governor holds +// the clock up on its own (performance, which GameMode sets). +// kTpause TPAUSE in C0.2. The core stays awake at reduced power, the clock +// holds, and the OS books the time as busy. Needs WAITPKG. +// kSpin No rest. The core stays awake at full power. +inline constexpr uint64_t kEnginePumpRestNs = 100'000; +// Timer slack the calling thread is set to; the kernel default of 50 us would +// otherwise stretch every sleep. +inline constexpr uint64_t kEnginePumpTimerSlackNs = 10'000; +// Shallow C-state exit plus scheduler latency on the host, for the bound. +inline constexpr uint64_t kShallowWakeNs = 50'000; + +enum class EnginePumpRestMode { kSleep, kTpause, kSpin }; + +const char* EnginePumpRestModeName(EnginePumpRestMode mode); + +struct EnginePumpRestInputs { + bool governor_performance = false; + bool cpu_has_waitpkg = false; + // MOCKTAIL_ENGINE_PUMP_REST: "sleep", "tpause" or "spin". Null, empty or + // unknown defers to the policy; "tpause" without WAITPKG does too. + const char* override = nullptr; +}; + +EnginePumpRestMode ChooseEnginePumpRestMode(const EnginePumpRestInputs& inputs); + +// True when the file holds "performance". `path` is a cpufreq +// scaling_governor file; cpu0's stands for the machine. +bool CpuGovernorIsPerformance(const char* path); + +bool CpuHasWaitPkg(); + +// The mode chosen from the host on first use. +EnginePumpRestMode ActiveEnginePumpRestMode(); + +// Rests the calling thread in the given mode and returns the rest length, 0 +// for kSpin. The first call lowers the thread's timer slack to +// kEnginePumpTimerSlackNs. +uint64_t RestAfterEnginePump(EnginePumpRestMode mode); +inline uint64_t RestAfterEnginePump() { + return RestAfterEnginePump(ActiveEnginePumpRestMode()); +} + +} // namespace mocktail::runtime + +#endif // MOCKTAIL_RUNTIME_ENGINE_PUMP_REST_H_ diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index 3f91280..f7ce9ef 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -58,6 +58,8 @@ #include "libc_shim/libc_shim.h" #include "linker/linker.h" #include "mocktail/graphics/bionic_egl_bridge.h" +#include "mocktail/graphics/chrome_trace_writer.h" +#include "runtime/engine_pump_rest.h" #include "runtime/device_memory_profile.h" #include "runtime/display_size.h" #include "runtime/environment.h" @@ -942,6 +944,43 @@ bool RunTaskSchedulerForegroundOnMainThread( } } +// The profile trace writer is compiled into the Vulkan adapter, which the host +// dlopens RTLD_GLOBAL, so the pump slice goes through symbols resolved from it. +// Until the adapter is loaded there is no writer and nothing is recorded. +struct PumpTraceHooks { + using ActiveFn = mocktail::graphics::ChromeTraceWriter* (*)(); + using ClockFn = std::uint64_t (*)(); + using SliceFn = void (*)(mocktail::graphics::ChromeTraceWriter*, const char*, + const char*, std::uint64_t, std::uint64_t, + const mocktail::graphics::TraceArg*, std::size_t); + ActiveFn active = nullptr; + ClockFn clock = nullptr; + SliceFn slice = nullptr; +}; + +const PumpTraceHooks& PumpTrace() { + static PumpTraceHooks hooks; + // The adapter loads during startup, so a few hundred ticks cover it. After + // that the lookups stop and the pump stays untraced. + static int attempts = 0; + if (hooks.active == nullptr && attempts++ < 1000) { + PumpTraceHooks resolved; + resolved.active = reinterpret_cast( + ::dlsym(RTLD_DEFAULT, "_ZN8mocktail8graphics18ActiveProfileTraceEv")); + resolved.clock = reinterpret_cast( + ::dlsym(RTLD_DEFAULT, "_ZN8mocktail8graphics15TraceClockNanosEv")); + resolved.slice = reinterpret_cast(::dlsym( + RTLD_DEFAULT, + "_ZN8mocktail8graphics17ChromeTraceWriter5SliceEPKcS3_mmPKNS0_" + "8TraceArgEm")); + if (resolved.active != nullptr && resolved.clock != nullptr && + resolved.slice != nullptr) { + hooks = resolved; + } + } + return hooks; +} + void PumpRobloxMainThreadMessagesOnce() { static const bool force_early = IsEnabled("MOCKTAIL_FORCE_EARLY_MAIN_THREAD_MESSAGE_PUMP"); @@ -1003,8 +1042,21 @@ void PumpRobloxMainThreadMessagesOnce() { << std::flush; } } + const PumpTraceHooks& hooks = PumpTrace(); + mocktail::graphics::ChromeTraceWriter* trace = + hooks.active != nullptr ? hooks.active() : nullptr; + const std::uint64_t pump_start_ns = trace != nullptr ? hooks.clock() : 0; g_native_call_messages_from_main_thread( env, g_native_gl_class_for_main_thread); + if (trace != nullptr) { + // Empty polls run at kilohertz rates and would swamp the trace, so only + // calls that ran a message are recorded. + const std::uint64_t pump_end_ns = hooks.clock(); + if (pump_end_ns - pump_start_ns >= 20'000) { + hooks.slice(trace, "nativeCallMessagesFromMainThread", "pump", + pump_start_ns, pump_end_ns, nullptr, 0); + } + } if (__builtin_expect(trace_pump, 0)) { if (pump_count <= 10 || pump_count % 100 == 0) { std::cerr << " [main] nativeCallMessagesFromMainThread returned #" @@ -6871,6 +6923,10 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, bool input_shutdown_completed = window_input_runtime == nullptr; bool resize_readiness_completed = true; if (mocktail::window::IsInitialised()) { + std::cout << " [main] engine pump rest: " + << mocktail::runtime::EnginePumpRestModeName( + mocktail::runtime::ActiveEnginePumpRestMode()) + << " (MOCKTAIL_ENGINE_PUMP_REST=sleep|tpause|spin)\n"; std::cout << " [main] entering SDL event loop (close the window to quit)\n" << std::flush; std::unique_ptr surface_bridge; @@ -6986,7 +7042,14 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, } } const uint64_t fps_after_launch_ns = fps_trace ? MonotonicNanos() : 0; - const uint64_t fps_pace_ns = mocktail::window::PaceInputPump(); + // The pacer sleeps only under throttled presentation. Unthrottled, the + // engine pump above is a poll, and the rest between polls is what keeps + // it from spinning. + uint64_t fps_pace_ns = mocktail::window::PaceInputPump(); + if (fps_pace_ns == 0 && + mocktail::window::UnthrottledPresentationRequested()) { + fps_pace_ns = mocktail::runtime::RestAfterEnginePump(); + } if (fps_trace) { const uint64_t fps_tick_end_ns = MonotonicNanos(); ++fps_samples; diff --git a/src/runtime/engine_pump_rest.cc b/src/runtime/engine_pump_rest.cc new file mode 100644 index 0000000..23b0e92 --- /dev/null +++ b/src/runtime/engine_pump_rest.cc @@ -0,0 +1,130 @@ +#include "runtime/engine_pump_rest.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace mocktail::runtime { +namespace { + +uint64_t NowNs() { + timespec now{}; + clock_gettime(CLOCK_MONOTONIC, &now); + return static_cast(now.tv_sec) * 1'000'000'000ULL + + static_cast(now.tv_nsec); +} + +// The kernel caps one TPAUSE at umwait_control/max_time TSC cycles (100000 by +// default), so the wait is a loop of short pauses up to the deadline. Control +// bit 0 clear selects C0.2, the deeper of the two light states. +__attribute__((target("waitpkg"))) void TpauseUntil(uint64_t deadline_ns) { + while (NowNs() < deadline_ns) { + _tpause(0, __rdtsc() + 100000ULL); + } +} + +void LowerTimerSlackOnce() { + // Only the main thread rests, so the slack is set once per process. + static const bool slack_set = [] { + return prctl(PR_SET_TIMERSLACK, kEnginePumpTimerSlackNs) == 0; + }(); + (void)slack_set; +} + +} // namespace + +const char* EnginePumpRestModeName(EnginePumpRestMode mode) { + switch (mode) { + case EnginePumpRestMode::kSleep: + return "sleep"; + case EnginePumpRestMode::kTpause: + return "tpause"; + case EnginePumpRestMode::kSpin: + return "spin"; + } + return "spin"; +} + +EnginePumpRestMode ChooseEnginePumpRestMode( + const EnginePumpRestInputs& inputs) { + if (inputs.override != nullptr) { + if (std::strcmp(inputs.override, "sleep") == 0) { + return EnginePumpRestMode::kSleep; + } + if (std::strcmp(inputs.override, "spin") == 0) { + return EnginePumpRestMode::kSpin; + } + if (std::strcmp(inputs.override, "tpause") == 0 && + inputs.cpu_has_waitpkg) { + return EnginePumpRestMode::kTpause; + } + } + if (inputs.governor_performance) { + return EnginePumpRestMode::kSleep; + } + return inputs.cpu_has_waitpkg ? EnginePumpRestMode::kTpause + : EnginePumpRestMode::kSpin; +} + +bool CpuGovernorIsPerformance(const char* path) { + std::FILE* file = std::fopen(path, "r"); + if (file == nullptr) { + return false; + } + char governor[32] = {}; + const bool read = std::fgets(governor, sizeof(governor), file) != nullptr; + std::fclose(file); + if (!read) { + return false; + } + governor[std::strcspn(governor, "\r\n")] = '\0'; + return std::strcmp(governor, "performance") == 0; +} + +bool CpuHasWaitPkg() { + unsigned eax = 0, ebx = 0, ecx = 0, edx = 0; + if (__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) == 0) { + return false; + } + return (ecx & (1u << 5)) != 0; +} + +EnginePumpRestMode ActiveEnginePumpRestMode() { + static const EnginePumpRestMode mode = [] { + EnginePumpRestInputs inputs; + inputs.governor_performance = CpuGovernorIsPerformance( + "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"); + inputs.cpu_has_waitpkg = CpuHasWaitPkg(); + inputs.override = std::getenv("MOCKTAIL_ENGINE_PUMP_REST"); + return ChooseEnginePumpRestMode(inputs); + }(); + return mode; +} + +uint64_t RestAfterEnginePump(EnginePumpRestMode mode) { + LowerTimerSlackOnce(); + switch (mode) { + case EnginePumpRestMode::kSpin: + return 0; + case EnginePumpRestMode::kTpause: + TpauseUntil(NowNs() + kEnginePumpRestNs); + return kEnginePumpRestNs; + case EnginePumpRestMode::kSleep: + break; + } + timespec rest{}; + rest.tv_sec = 0; + rest.tv_nsec = static_cast(kEnginePumpRestNs); + // A signal may cut the rest short. The remainder is not slept again: an + // early poll only costs a few microseconds. + (void)clock_nanosleep(CLOCK_MONOTONIC, 0, &rest, nullptr); + return kEnginePumpRestNs; +} + +} // namespace mocktail::runtime diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ce48303..a10983a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -208,6 +208,13 @@ target_link_libraries(external_launch_broker_test PRIVATE ) mocktail_apply_compile_options(external_launch_broker_test) +add_executable(engine_pump_rest_test engine_pump_rest_test.cc) +target_link_libraries(engine_pump_rest_test PRIVATE + Mocktail::Runtime + GTest::gtest_main +) +mocktail_apply_compile_options(engine_pump_rest_test) + add_executable(supported_launch_policy_test supported_launch_policy_test.cc) target_link_libraries(supported_launch_policy_test PRIVATE Mocktail::Runtime @@ -986,6 +993,7 @@ gtest_discover_tests(browser_tracker_service_test) gtest_discover_tests(command_line_test) gtest_discover_tests(roblox_launch_uri_test) gtest_discover_tests(external_launch_broker_test) +gtest_discover_tests(engine_pump_rest_test) gtest_discover_tests(webview_helper_launcher_test) gtest_discover_tests(webview_helper_policy_test) gtest_discover_tests(webview_roblox_cookie_test) diff --git a/tests/engine_pump_rest_test.cc b/tests/engine_pump_rest_test.cc new file mode 100644 index 0000000..32d85e7 --- /dev/null +++ b/tests/engine_pump_rest_test.cc @@ -0,0 +1,131 @@ +#include "runtime/engine_pump_rest.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace mocktail::runtime { +namespace { + +uint64_t NowNs() { + timespec now{}; + clock_gettime(CLOCK_MONOTONIC, &now); + return static_cast(now.tv_sec) * 1'000'000'000ULL + + static_cast(now.tv_nsec); +} + +class GovernorFile final { + public: + explicit GovernorFile(const char* content) { + char pattern[] = "/tmp/mocktail_governor_XXXXXX"; + const int fd = mkstemp(pattern); + if (fd >= 0) { + close(fd); + path_ = pattern; + std::ofstream(path_) << content; + } + } + ~GovernorFile() { + std::error_code error; + std::filesystem::remove(path_, error); + } + const char* path() const { return path_.c_str(); } + + private: + std::string path_; +}; + +// A performance governor holds the clock up on its own, so the thread can +// sleep. Without it, only a core that stays awake keeps the clock up: tpause +// where the CPU has it, a spin otherwise. +TEST(EnginePumpRestTest, PolicyFollowsGovernorThenCpuFeature) { + EXPECT_EQ(ChooseEnginePumpRestMode({true, true, nullptr}), + EnginePumpRestMode::kSleep); + EXPECT_EQ(ChooseEnginePumpRestMode({true, false, nullptr}), + EnginePumpRestMode::kSleep); + EXPECT_EQ(ChooseEnginePumpRestMode({false, true, nullptr}), + EnginePumpRestMode::kTpause); + EXPECT_EQ(ChooseEnginePumpRestMode({false, false, nullptr}), + EnginePumpRestMode::kSpin); +} + +TEST(EnginePumpRestTest, OverrideWinsWhenTheCpuCanHonorIt) { + EXPECT_EQ(ChooseEnginePumpRestMode({false, true, "sleep"}), + EnginePumpRestMode::kSleep); + EXPECT_EQ(ChooseEnginePumpRestMode({true, true, "spin"}), + EnginePumpRestMode::kSpin); + EXPECT_EQ(ChooseEnginePumpRestMode({true, true, "tpause"}), + EnginePumpRestMode::kTpause); + // tpause needs the instruction; without it the policy decides. + EXPECT_EQ(ChooseEnginePumpRestMode({false, false, "tpause"}), + EnginePumpRestMode::kSpin); + EXPECT_EQ(ChooseEnginePumpRestMode({false, true, "bogus"}), + EnginePumpRestMode::kTpause); + EXPECT_EQ(ChooseEnginePumpRestMode({false, true, ""}), + EnginePumpRestMode::kTpause); +} + +TEST(EnginePumpRestTest, ReadsThePerformanceGovernorFromSysfs) { + GovernorFile performance("performance\n"); + GovernorFile powersave("powersave\n"); + EXPECT_TRUE(CpuGovernorIsPerformance(performance.path())); + EXPECT_FALSE(CpuGovernorIsPerformance(powersave.path())); + EXPECT_FALSE(CpuGovernorIsPerformance("/nonexistent/scaling_governor")); +} + +TEST(EnginePumpRestTest, ModeNamesAreStable) { + EXPECT_STREQ(EnginePumpRestModeName(EnginePumpRestMode::kSleep), "sleep"); + EXPECT_STREQ(EnginePumpRestModeName(EnginePumpRestMode::kTpause), "tpause"); + EXPECT_STREQ(EnginePumpRestModeName(EnginePumpRestMode::kSpin), "spin"); +} + +// The rest bounds how late an engine post is picked up. With the thread's +// timer slack lowered and a shallow C-state exit, the bound sits far under +// the engine's sub-millisecond slack. +TEST(EnginePumpRestTest, BoundStaysUnderTheEngineSlack) { + EXPECT_LE(kEnginePumpRestNs + kEnginePumpTimerSlackNs + kShallowWakeNs, + 300'000ULL); +} + +TEST(EnginePumpRestTest, LowersTheCallingThreadTimerSlackOnce) { + (void)RestAfterEnginePump(EnginePumpRestMode::kSleep); + EXPECT_EQ(prctl(PR_GET_TIMERSLACK), static_cast(kEnginePumpTimerSlackNs)); +} + +// A real sleep, not a spin: the call takes at least the rest, and on an idle +// core not much more. The upper bound is loose so a loaded box does not flake. +TEST(EnginePumpRestTest, SleepModeSleepsForTheRestInterval) { + const uint64_t start = NowNs(); + const uint64_t reported = RestAfterEnginePump(EnginePumpRestMode::kSleep); + const uint64_t elapsed = NowNs() - start; + EXPECT_EQ(reported, kEnginePumpRestNs); + EXPECT_GE(elapsed, kEnginePumpRestNs); + EXPECT_LT(elapsed, 2'000'000ULL) << "rest overshot by " << elapsed << " ns"; +} + +TEST(EnginePumpRestTest, TpauseModeWaitsForTheRestInterval) { + if (!CpuHasWaitPkg()) { + GTEST_SKIP() << "CPU has no WAITPKG"; + } + const uint64_t start = NowNs(); + const uint64_t reported = RestAfterEnginePump(EnginePumpRestMode::kTpause); + const uint64_t elapsed = NowNs() - start; + EXPECT_EQ(reported, kEnginePumpRestNs); + EXPECT_GE(elapsed, kEnginePumpRestNs); + EXPECT_LT(elapsed, 2'000'000ULL) << "rest overshot by " << elapsed << " ns"; +} + +TEST(EnginePumpRestTest, SpinModeDoesNotRest) { + const uint64_t start = NowNs(); + EXPECT_EQ(RestAfterEnginePump(EnginePumpRestMode::kSpin), 0U); + EXPECT_LT(NowNs() - start, kEnginePumpRestNs); +} + +} // namespace +} // namespace mocktail::runtime diff --git a/tests/input_pump_pacing_runtime_test.cc b/tests/input_pump_pacing_runtime_test.cc index fc718d8..4cf1e52 100644 --- a/tests/input_pump_pacing_runtime_test.cc +++ b/tests/input_pump_pacing_runtime_test.cc @@ -20,13 +20,13 @@ bool RealSdlWindowTestsEnabled() { return enabled != nullptr && std::string(enabled) == "1"; } -// Unthrottled presentation must not park the main-thread pump. +// Unthrottled presentation must not pace the main-thread pump. // -// Roblox's render thread posts work to the main thread and waits on the -// reply. That queue is inside libroblox.so, so polling is the only way to -// observe it and a timed sleep cannot be woken by a post. A frame that misses -// its slot slips a whole period, and the engine's slack before that happens -// is under a millisecond, shorter than any sleep long enough to save CPU. +// Roblox posts its per-frame main-thread step at a time of its own choosing +// and nothing signals the host, so the pump is a poll. A frame that misses +// its slot slips a whole period, and the engine's slack is under a +// millisecond, so the pacer's 240 Hz cadence is far too coarse here. The rest +// between polls is runtime/engine_pump_rest, bounded at 100 us. // // A throttled present blocks on vsync, so the pacer still runs there. TEST(InputPumpPacingRuntimeTest, UnthrottledPresentationDoesNotParkThePump) {