perf(runtime): rest the engine pump between polls instead of spinning

The main thread polls nativeCallMessagesFromMainThread 146k times a second
with nothing to do. Between polls it now rests 100 us: nanosleep when the
governor is performance, TPAUSE (C0.2) when the CPU has WAITPKG, a spin
otherwise. Sleeping under a power-saving governor lets the render thread's
core clock down and drops frames, which is why TPAUSE is the default there.
MOCKTAIL_ENGINE_PUMP_REST=sleep|tpause|spin overrides. The profile trace
gains a slice for pumps that ran a message.

CMakeLists.txt

@@ -266,6 +266,7 @@ add_library(mocktail_runtime STATIC
   src/runtime/crash_report_policy.cc
   src/runtime/discord_rpc.cc
   src/runtime/device_profile.cc
+  src/runtime/engine_pump_rest.cc
   src/runtime/environment.cc
   src/runtime/external_launch_broker.cc
   src/runtime/frame_rate_policy.cc

include/runtime/engine_pump_rest.h

@@ -0,0 +1,65 @@
+#ifndef MOCKTAIL_RUNTIME_ENGINE_PUMP_REST_H_
+#define MOCKTAIL_RUNTIME_ENGINE_PUMP_REST_H_
+
+#include <cstdint>
+
+namespace mocktail::runtime {
+
+// The engine posts its per-frame main-thread step at a time of its own
+// choosing, with no signal the host can block on, so the host main thread
+// polls nativeCallMessagesFromMainThread. An empty poll costs a few
+// microseconds; polled back to back it burns a core.
+//
+// Between polls the thread rests for kEnginePumpRestNs. A post is then picked
+// up at most kEnginePumpRestNs + timer slack + wake latency late, far under
+// the engine's sub-millisecond slack.
+//
+// How it rests decides the clock. Under a power-saving governor the CPU clocks
+// a bursty core down unless some core stays awake, and the engine's render
+// thread then runs slower and drops frames. So:
+//   kSleep   nanosleep. The core idles. Correct only when the governor holds
+//            the clock up on its own (performance, which GameMode sets).
+//   kTpause  TPAUSE in C0.2. The core stays awake at reduced power, the clock
+//            holds, and the OS books the time as busy. Needs WAITPKG.
+//   kSpin    No rest. The core stays awake at full power.
+inline constexpr uint64_t kEnginePumpRestNs = 100'000;
+// Timer slack the calling thread is set to; the kernel default of 50 us would
+// otherwise stretch every sleep.
+inline constexpr uint64_t kEnginePumpTimerSlackNs = 10'000;
+// Shallow C-state exit plus scheduler latency on the host, for the bound.
+inline constexpr uint64_t kShallowWakeNs = 50'000;
+
+enum class EnginePumpRestMode { kSleep, kTpause, kSpin };
+
+const char* EnginePumpRestModeName(EnginePumpRestMode mode);
+
+struct EnginePumpRestInputs {
+  bool governor_performance = false;
+  bool cpu_has_waitpkg = false;
+  // MOCKTAIL_ENGINE_PUMP_REST: "sleep", "tpause" or "spin". Null, empty or
+  // unknown defers to the policy; "tpause" without WAITPKG does too.
+  const char* override = nullptr;
+};
+
+EnginePumpRestMode ChooseEnginePumpRestMode(const EnginePumpRestInputs& inputs);
+
+// True when the file holds "performance". `path` is a cpufreq
+// scaling_governor file; cpu0's stands for the machine.
+bool CpuGovernorIsPerformance(const char* path);
+
+bool CpuHasWaitPkg();
+
+// The mode chosen from the host on first use.
+EnginePumpRestMode ActiveEnginePumpRestMode();
+
+// Rests the calling thread in the given mode and returns the rest length, 0
+// for kSpin. The first call lowers the thread's timer slack to
+// kEnginePumpTimerSlackNs.
+uint64_t RestAfterEnginePump(EnginePumpRestMode mode);
+inline uint64_t RestAfterEnginePump() {
+  return RestAfterEnginePump(ActiveEnginePumpRestMode());
+}
+
+}  // namespace mocktail::runtime
+
+#endif  // MOCKTAIL_RUNTIME_ENGINE_PUMP_REST_H_

src/legacy/legacy_runtime.cc

@@ -58,6 +58,8 @@
 #include "libc_shim/libc_shim.h"
 #include "linker/linker.h"
 #include "mocktail/graphics/bionic_egl_bridge.h"
+#include "mocktail/graphics/chrome_trace_writer.h"
+#include "runtime/engine_pump_rest.h"
 #include "runtime/device_memory_profile.h"
 #include "runtime/display_size.h"
 #include "runtime/environment.h"
@@ -942,6 +944,43 @@ bool RunTaskSchedulerForegroundOnMainThread(
   }
 }
 
+// The profile trace writer is compiled into the Vulkan adapter, which the host
+// dlopens RTLD_GLOBAL, so the pump slice goes through symbols resolved from it.
+// Until the adapter is loaded there is no writer and nothing is recorded.
+struct PumpTraceHooks {
+  using ActiveFn = mocktail::graphics::ChromeTraceWriter* (*)();
+  using ClockFn = std::uint64_t (*)();
+  using SliceFn = void (*)(mocktail::graphics::ChromeTraceWriter*, const char*,
+                           const char*, std::uint64_t, std::uint64_t,
+                           const mocktail::graphics::TraceArg*, std::size_t);
+  ActiveFn active = nullptr;
+  ClockFn clock = nullptr;
+  SliceFn slice = nullptr;
+};
+
+const PumpTraceHooks& PumpTrace() {
+  static PumpTraceHooks hooks;
+  // The adapter loads during startup, so a few hundred ticks cover it. After
+  // that the lookups stop and the pump stays untraced.
+  static int attempts = 0;
+  if (hooks.active == nullptr && attempts++ < 1000) {
+    PumpTraceHooks resolved;
+    resolved.active = reinterpret_cast<PumpTraceHooks::ActiveFn>(
+        ::dlsym(RTLD_DEFAULT, "_ZN8mocktail8graphics18ActiveProfileTraceEv"));
+    resolved.clock = reinterpret_cast<PumpTraceHooks::ClockFn>(
+        ::dlsym(RTLD_DEFAULT, "_ZN8mocktail8graphics15TraceClockNanosEv"));
+    resolved.slice = reinterpret_cast<PumpTraceHooks::SliceFn>(::dlsym(
+        RTLD_DEFAULT,
+        "_ZN8mocktail8graphics17ChromeTraceWriter5SliceEPKcS3_mmPKNS0_"
+        "8TraceArgEm"));
+    if (resolved.active != nullptr && resolved.clock != nullptr &&
+        resolved.slice != nullptr) {
+      hooks = resolved;
+    }
+  }
+  return hooks;
+}
+
 void PumpRobloxMainThreadMessagesOnce() {
   static const bool force_early =
       IsEnabled("MOCKTAIL_FORCE_EARLY_MAIN_THREAD_MESSAGE_PUMP");
@@ -1003,8 +1042,21 @@ void PumpRobloxMainThreadMessagesOnce() {
                 << std::flush;
     }
   }
+  const PumpTraceHooks& hooks = PumpTrace();
+  mocktail::graphics::ChromeTraceWriter* trace =
+      hooks.active != nullptr ? hooks.active() : nullptr;
+  const std::uint64_t pump_start_ns = trace != nullptr ? hooks.clock() : 0;
   g_native_call_messages_from_main_thread(
       env, g_native_gl_class_for_main_thread);
+  if (trace != nullptr) {
+    // Empty polls run at kilohertz rates and would swamp the trace, so only
+    // calls that ran a message are recorded.
+    const std::uint64_t pump_end_ns = hooks.clock();
+    if (pump_end_ns - pump_start_ns >= 20'000) {
+      hooks.slice(trace, "nativeCallMessagesFromMainThread", "pump",
+                  pump_start_ns, pump_end_ns, nullptr, 0);
+    }
+  }
   if (__builtin_expect(trace_pump, 0)) {
     if (pump_count <= 10 || pump_count % 100 == 0) {
       std::cerr << "  [main] nativeCallMessagesFromMainThread returned #"
@@ -6871,6 +6923,10 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options,
   bool input_shutdown_completed = window_input_runtime == nullptr;
   bool resize_readiness_completed = true;
   if (mocktail::window::IsInitialised()) {
+    std::cout << "  [main] engine pump rest: "
+              << mocktail::runtime::EnginePumpRestModeName(
+                     mocktail::runtime::ActiveEnginePumpRestMode())
+              << " (MOCKTAIL_ENGINE_PUMP_REST=sleep|tpause|spin)\n";
     std::cout << "  [main] entering SDL event loop (close the window to quit)\n"
               << std::flush;
     std::unique_ptr<mocktail::window::WindowGameSurfaceBridge> surface_bridge;
@@ -6986,7 +7042,14 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options,
         }
       }
       const uint64_t fps_after_launch_ns = fps_trace ? MonotonicNanos() : 0;
-      const uint64_t fps_pace_ns = mocktail::window::PaceInputPump();
+      // The pacer sleeps only under throttled presentation. Unthrottled, the
+      // engine pump above is a poll, and the rest between polls is what keeps
+      // it from spinning.
+      uint64_t fps_pace_ns = mocktail::window::PaceInputPump();
+      if (fps_pace_ns == 0 &&
+          mocktail::window::UnthrottledPresentationRequested()) {
+        fps_pace_ns = mocktail::runtime::RestAfterEnginePump();
+      }
       if (fps_trace) {
         const uint64_t fps_tick_end_ns = MonotonicNanos();
         ++fps_samples;

src/runtime/engine_pump_rest.cc

@@ -0,0 +1,130 @@
+#include "runtime/engine_pump_rest.h"
+
+#include <cpuid.h>
+#include <immintrin.h>
+#include <sys/prctl.h>
+#include <time.h>
+#include <x86intrin.h>
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+namespace mocktail::runtime {
+namespace {
+
+uint64_t NowNs() {
+  timespec now{};
+  clock_gettime(CLOCK_MONOTONIC, &now);
+  return static_cast<uint64_t>(now.tv_sec) * 1'000'000'000ULL +
+         static_cast<uint64_t>(now.tv_nsec);
+}
+
+// The kernel caps one TPAUSE at umwait_control/max_time TSC cycles (100000 by
+// default), so the wait is a loop of short pauses up to the deadline. Control
+// bit 0 clear selects C0.2, the deeper of the two light states.
+__attribute__((target("waitpkg"))) void TpauseUntil(uint64_t deadline_ns) {
+  while (NowNs() < deadline_ns) {
+    _tpause(0, __rdtsc() + 100000ULL);
+  }
+}
+
+void LowerTimerSlackOnce() {
+  // Only the main thread rests, so the slack is set once per process.
+  static const bool slack_set = [] {
+    return prctl(PR_SET_TIMERSLACK, kEnginePumpTimerSlackNs) == 0;
+  }();
+  (void)slack_set;
+}
+
+}  // namespace
+
+const char* EnginePumpRestModeName(EnginePumpRestMode mode) {
+  switch (mode) {
+    case EnginePumpRestMode::kSleep:
+      return "sleep";
+    case EnginePumpRestMode::kTpause:
+      return "tpause";
+    case EnginePumpRestMode::kSpin:
+      return "spin";
+  }
+  return "spin";
+}
+
+EnginePumpRestMode ChooseEnginePumpRestMode(
+    const EnginePumpRestInputs& inputs) {
+  if (inputs.override != nullptr) {
+    if (std::strcmp(inputs.override, "sleep") == 0) {
+      return EnginePumpRestMode::kSleep;
+    }
+    if (std::strcmp(inputs.override, "spin") == 0) {
+      return EnginePumpRestMode::kSpin;
+    }
+    if (std::strcmp(inputs.override, "tpause") == 0 &&
+        inputs.cpu_has_waitpkg) {
+      return EnginePumpRestMode::kTpause;
+    }
+  }
+  if (inputs.governor_performance) {
+    return EnginePumpRestMode::kSleep;
+  }
+  return inputs.cpu_has_waitpkg ? EnginePumpRestMode::kTpause
+                                : EnginePumpRestMode::kSpin;
+}
+
+bool CpuGovernorIsPerformance(const char* path) {
+  std::FILE* file = std::fopen(path, "r");
+  if (file == nullptr) {
+    return false;
+  }
+  char governor[32] = {};
+  const bool read = std::fgets(governor, sizeof(governor), file) != nullptr;
+  std::fclose(file);
+  if (!read) {
+    return false;
+  }
+  governor[std::strcspn(governor, "\r\n")] = '\0';
+  return std::strcmp(governor, "performance") == 0;
+}
+
+bool CpuHasWaitPkg() {
+  unsigned eax = 0, ebx = 0, ecx = 0, edx = 0;
+  if (__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) == 0) {
+    return false;
+  }
+  return (ecx & (1u << 5)) != 0;
+}
+
+EnginePumpRestMode ActiveEnginePumpRestMode() {
+  static const EnginePumpRestMode mode = [] {
+    EnginePumpRestInputs inputs;
+    inputs.governor_performance = CpuGovernorIsPerformance(
+        "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor");
+    inputs.cpu_has_waitpkg = CpuHasWaitPkg();
+    inputs.override = std::getenv("MOCKTAIL_ENGINE_PUMP_REST");
+    return ChooseEnginePumpRestMode(inputs);
+  }();
+  return mode;
+}
+
+uint64_t RestAfterEnginePump(EnginePumpRestMode mode) {
+  LowerTimerSlackOnce();
+  switch (mode) {
+    case EnginePumpRestMode::kSpin:
+      return 0;
+    case EnginePumpRestMode::kTpause:
+      TpauseUntil(NowNs() + kEnginePumpRestNs);
+      return kEnginePumpRestNs;
+    case EnginePumpRestMode::kSleep:
+      break;
+  }
+  timespec rest{};
+  rest.tv_sec = 0;
+  rest.tv_nsec = static_cast<long>(kEnginePumpRestNs);
+  // A signal may cut the rest short. The remainder is not slept again: an
+  // early poll only costs a few microseconds.
+  (void)clock_nanosleep(CLOCK_MONOTONIC, 0, &rest, nullptr);
+  return kEnginePumpRestNs;
+}
+
+}  // namespace mocktail::runtime

tests/CMakeLists.txt

@@ -208,6 +208,13 @@ target_link_libraries(external_launch_broker_test PRIVATE
 )
 mocktail_apply_compile_options(external_launch_broker_test)
 
+add_executable(engine_pump_rest_test engine_pump_rest_test.cc)
+target_link_libraries(engine_pump_rest_test PRIVATE
+  Mocktail::Runtime
+  GTest::gtest_main
+)
+mocktail_apply_compile_options(engine_pump_rest_test)
+
 add_executable(supported_launch_policy_test supported_launch_policy_test.cc)
 target_link_libraries(supported_launch_policy_test PRIVATE
   Mocktail::Runtime
@@ -986,6 +993,7 @@ gtest_discover_tests(browser_tracker_service_test)
 gtest_discover_tests(command_line_test)
 gtest_discover_tests(roblox_launch_uri_test)
 gtest_discover_tests(external_launch_broker_test)
+gtest_discover_tests(engine_pump_rest_test)
 gtest_discover_tests(webview_helper_launcher_test)
 gtest_discover_tests(webview_helper_policy_test)
 gtest_discover_tests(webview_roblox_cookie_test)

tests/engine_pump_rest_test.cc

@@ -0,0 +1,131 @@
+#include "runtime/engine_pump_rest.h"
+
+#include <gtest/gtest.h>
+#include <sys/prctl.h>
+#include <unistd.h>
+
+#include <cstdio>
+#include <ctime>
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+namespace mocktail::runtime {
+namespace {
+
+uint64_t NowNs() {
+  timespec now{};
+  clock_gettime(CLOCK_MONOTONIC, &now);
+  return static_cast<uint64_t>(now.tv_sec) * 1'000'000'000ULL +
+         static_cast<uint64_t>(now.tv_nsec);
+}
+
+class GovernorFile final {
+ public:
+  explicit GovernorFile(const char* content) {
+    char pattern[] = "/tmp/mocktail_governor_XXXXXX";
+    const int fd = mkstemp(pattern);
+    if (fd >= 0) {
+      close(fd);
+      path_ = pattern;
+      std::ofstream(path_) << content;
+    }
+  }
+  ~GovernorFile() {
+    std::error_code error;
+    std::filesystem::remove(path_, error);
+  }
+  const char* path() const { return path_.c_str(); }
+
+ private:
+  std::string path_;
+};
+
+// A performance governor holds the clock up on its own, so the thread can
+// sleep. Without it, only a core that stays awake keeps the clock up: tpause
+// where the CPU has it, a spin otherwise.
+TEST(EnginePumpRestTest, PolicyFollowsGovernorThenCpuFeature) {
+  EXPECT_EQ(ChooseEnginePumpRestMode({true, true, nullptr}),
+            EnginePumpRestMode::kSleep);
+  EXPECT_EQ(ChooseEnginePumpRestMode({true, false, nullptr}),
+            EnginePumpRestMode::kSleep);
+  EXPECT_EQ(ChooseEnginePumpRestMode({false, true, nullptr}),
+            EnginePumpRestMode::kTpause);
+  EXPECT_EQ(ChooseEnginePumpRestMode({false, false, nullptr}),
+            EnginePumpRestMode::kSpin);
+}
+
+TEST(EnginePumpRestTest, OverrideWinsWhenTheCpuCanHonorIt) {
+  EXPECT_EQ(ChooseEnginePumpRestMode({false, true, "sleep"}),
+            EnginePumpRestMode::kSleep);
+  EXPECT_EQ(ChooseEnginePumpRestMode({true, true, "spin"}),
+            EnginePumpRestMode::kSpin);
+  EXPECT_EQ(ChooseEnginePumpRestMode({true, true, "tpause"}),
+            EnginePumpRestMode::kTpause);
+  // tpause needs the instruction; without it the policy decides.
+  EXPECT_EQ(ChooseEnginePumpRestMode({false, false, "tpause"}),
+            EnginePumpRestMode::kSpin);
+  EXPECT_EQ(ChooseEnginePumpRestMode({false, true, "bogus"}),
+            EnginePumpRestMode::kTpause);
+  EXPECT_EQ(ChooseEnginePumpRestMode({false, true, ""}),
+            EnginePumpRestMode::kTpause);
+}
+
+TEST(EnginePumpRestTest, ReadsThePerformanceGovernorFromSysfs) {
+  GovernorFile performance("performance\n");
+  GovernorFile powersave("powersave\n");
+  EXPECT_TRUE(CpuGovernorIsPerformance(performance.path()));
+  EXPECT_FALSE(CpuGovernorIsPerformance(powersave.path()));
+  EXPECT_FALSE(CpuGovernorIsPerformance("/nonexistent/scaling_governor"));
+}
+
+TEST(EnginePumpRestTest, ModeNamesAreStable) {
+  EXPECT_STREQ(EnginePumpRestModeName(EnginePumpRestMode::kSleep), "sleep");
+  EXPECT_STREQ(EnginePumpRestModeName(EnginePumpRestMode::kTpause), "tpause");
+  EXPECT_STREQ(EnginePumpRestModeName(EnginePumpRestMode::kSpin), "spin");
+}
+
+// The rest bounds how late an engine post is picked up. With the thread's
+// timer slack lowered and a shallow C-state exit, the bound sits far under
+// the engine's sub-millisecond slack.
+TEST(EnginePumpRestTest, BoundStaysUnderTheEngineSlack) {
+  EXPECT_LE(kEnginePumpRestNs + kEnginePumpTimerSlackNs + kShallowWakeNs,
+            300'000ULL);
+}
+
+TEST(EnginePumpRestTest, LowersTheCallingThreadTimerSlackOnce) {
+  (void)RestAfterEnginePump(EnginePumpRestMode::kSleep);
+  EXPECT_EQ(prctl(PR_GET_TIMERSLACK), static_cast<int>(kEnginePumpTimerSlackNs));
+}
+
+// A real sleep, not a spin: the call takes at least the rest, and on an idle
+// core not much more. The upper bound is loose so a loaded box does not flake.
+TEST(EnginePumpRestTest, SleepModeSleepsForTheRestInterval) {
+  const uint64_t start = NowNs();
+  const uint64_t reported = RestAfterEnginePump(EnginePumpRestMode::kSleep);
+  const uint64_t elapsed = NowNs() - start;
+  EXPECT_EQ(reported, kEnginePumpRestNs);
+  EXPECT_GE(elapsed, kEnginePumpRestNs);
+  EXPECT_LT(elapsed, 2'000'000ULL) << "rest overshot by " << elapsed << " ns";
+}
+
+TEST(EnginePumpRestTest, TpauseModeWaitsForTheRestInterval) {
+  if (!CpuHasWaitPkg()) {
+    GTEST_SKIP() << "CPU has no WAITPKG";
+  }
+  const uint64_t start = NowNs();
+  const uint64_t reported = RestAfterEnginePump(EnginePumpRestMode::kTpause);
+  const uint64_t elapsed = NowNs() - start;
+  EXPECT_EQ(reported, kEnginePumpRestNs);
+  EXPECT_GE(elapsed, kEnginePumpRestNs);
+  EXPECT_LT(elapsed, 2'000'000ULL) << "rest overshot by " << elapsed << " ns";
+}
+
+TEST(EnginePumpRestTest, SpinModeDoesNotRest) {
+  const uint64_t start = NowNs();
+  EXPECT_EQ(RestAfterEnginePump(EnginePumpRestMode::kSpin), 0U);
+  EXPECT_LT(NowNs() - start, kEnginePumpRestNs);
+}
+
+}  // namespace
+}  // namespace mocktail::runtime

tests/input_pump_pacing_runtime_test.cc

@@ -20,13 +20,13 @@ bool RealSdlWindowTestsEnabled() {
   return enabled != nullptr && std::string(enabled) == "1";
 }
 
-// Unthrottled presentation must not park the main-thread pump.
+// Unthrottled presentation must not pace the main-thread pump.
 //
-// Roblox's render thread posts work to the main thread and waits on the
-// reply. That queue is inside libroblox.so, so polling is the only way to
-// observe it and a timed sleep cannot be woken by a post. A frame that misses
-// its slot slips a whole period, and the engine's slack before that happens
-// is under a millisecond, shorter than any sleep long enough to save CPU.
+// Roblox posts its per-frame main-thread step at a time of its own choosing
+// and nothing signals the host, so the pump is a poll. A frame that misses
+// its slot slips a whole period, and the engine's slack is under a
+// millisecond, so the pacer's 240 Hz cadence is far too coarse here. The rest
+// between polls is runtime/engine_pump_rest, bounded at 100 us.
 //
 // A throttled present blocks on vsync, so the pacer still runs there.
 TEST(InputPumpPacingRuntimeTest, UnthrottledPresentationDoesNotParkThePump) {