Linux: the GPU driver kept from the kernel's trace_marker (HALO_GPU_TRACE_MARKERS)

SteamOS keeps kernel tracing on for its GPU performance captures
(gpu-trace.service), and its Mesa, built with gpuvis, then writes a marker
into the trace buffer for each traced driver function. On the Steam Frame
that is some 480,000 writes a second: the driver's thread spent its time in
them, and the game ran at about 50 frames a second instead of the headset's
72 (#71 has the same for the arm64 build). The Steam Deck runs the same
gpu-trace service and the same Mesa, so it needs this too.

The executable now defines open() and openat() (and their 64-bit names),
exported so that the driver's calls reach them: they refuse trace_marker
and pass everything else on. gpuvis then skips its markers.
HALO_GPU_TRACE_MARKERS=1 lets the driver write them, to capture with gpuvis.
The link takes libdl for dlsym, which glibc before 2.34 keeps there.

port/linux/README.md

@@ -708,6 +708,14 @@ definition. Without this check, the linker gives the reference the address
   `d3d_find_flipcount`.
 - The build returns small structures and unions in registers
   (`-freg-struct-return`), as on Win32.
+- The GPU driver cannot open the kernel's `trace_marker`
+  (`src/posix_trace_marker.c`). SteamOS keeps kernel tracing on for its GPU
+  performance captures (`gpu-trace.service`), and its Mesa then writes a
+  marker for each traced driver function: on the Steam Frame, some 480,000
+  writes a second, which took the game from the headset's 72 Hz to about
+  50 frames a second. The Steam Deck runs the same service and Mesa.
+  `HALO_GPU_TRACE_MARKERS=1` lets the driver write them, to capture with
+  gpuvis.
 
 ### Game source changes
 

port/linux/port.json

@@ -68,6 +68,7 @@
   "libraries": [
     "SDL3",
     "m",
-    "pthread"
+    "pthread",
+    "dl"
   ]
 }

port/linux/src/posix_trace_marker.c

@@ -0,0 +1,91 @@
+/*
+POSIX_TRACE_MARKER.C
+
+The kernel's trace_marker kept from the GPU driver.
+
+Mesa built with gpuvis (SteamOS's) writes a marker into the kernel's trace
+buffer for each traced driver function while tracing is on, and SteamOS
+keeps tracing on for its GPU performance captures (gpu-trace.service). On the
+Steam Frame that is some 480,000 writes a second: the driver's thread spends
+its time in them, and the game ran at about 50 frames a second instead of
+the headset's 72. The Steam Deck runs the same service and the same Mesa. A
+trace_marker that cannot be opened turns the markers off (gpuvis then skips
+them).
+
+This executable defines open() and openat() (and their 64-bit names), which
+the link exports (tools/linux_build.py), so the driver's calls come here:
+they refuse trace_marker and pass everything else to the C library's.
+HALO_GPU_TRACE_MARKERS=1 lets the driver open it, to capture with gpuvis.
+Built with the host ABI.
+*/
+
+/* open and open64 as two functions, not open as glibc's open64 */
+#undef _FILE_OFFSET_BITS
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef int (*open_function)(const char *path, int flags, ...);
+typedef int (*openat_function)(int directory, const char *path, int flags, ...);
+
+static int refused(const char *path)
+{
+	static int allowed = -1;
+
+	if (!path || !strstr(path, "trace_marker"))
+		return 0;
+	if (allowed < 0)
+	{
+		const char *setting = getenv("HALO_GPU_TRACE_MARKERS");
+
+		allowed = setting && *setting && strcmp(setting, "0") != 0;
+	}
+	if (allowed)
+		return 0;
+	errno = EACCES;
+	return 1;
+}
+
+/* the mode, which follows only with O_CREAT or O_TMPFILE */
+#define MODE_ARGUMENT(flags, last, mode) \
+	do \
+	{ \
+		va_list arguments; \
+		va_start(arguments, last); \
+		mode = (flags & (O_CREAT | O_TMPFILE)) ? (mode_t)va_arg(arguments, int) : 0; \
+		va_end(arguments); \
+	} while (0)
+
+#define OPEN(name) \
+	int name(const char *path, int flags, ...) \
+	{ \
+		static open_function real; \
+		mode_t mode; \
+		MODE_ARGUMENT(flags, flags, mode); \
+		if (refused(path)) \
+			return -1; \
+		if (!real) \
+			real = (open_function)dlsym(RTLD_NEXT, #name); \
+		return real(path, flags, mode); \
+	}
+
+#define OPENAT(name) \
+	int name(int directory, const char *path, int flags, ...) \
+	{ \
+		static openat_function real; \
+		mode_t mode; \
+		MODE_ARGUMENT(flags, flags, mode); \
+		if (refused(path)) \
+			return -1; \
+		if (!real) \
+			real = (openat_function)dlsym(RTLD_NEXT, #name); \
+		return real(directory, path, flags, mode); \
+	}
+
+OPEN(open)
+OPEN(open64)
+OPENAT(openat)
+OPENAT(openat64)

tools/linux_build.py

@@ -496,7 +496,10 @@ def generate_linux_build(n: Writer, sln: Any) -> None:
             rule="linux_link",
             inputs=objects,
             variables={
-                "ldflags": " ".join(["--target=i686-linux-gnu", "-m32", "-no-pie", "-g", *extra_ldflags]),
+                "ldflags": " ".join(["--target=i686-linux-gnu", "-m32", "-no-pie", "-g", *extra_ldflags,
+                                     # (posix_trace_marker.c's, which the GPU driver's calls must reach)
+                                     *(f"-Wl,--export-dynamic-symbol={name}"
+                                       for name in ("open", "open64", "openat", "openat64"))]),
                 "libs": libs,
             },
             implicit=[Path("tools/linux_link_check.py")],