GPU: after the GPU stops, the next three launches record its breadcrumbs (DRED) for the crash report

The device-removal watcher always runs, and the fatal handler notes a DXGI stop in gpu-breadcrumbs.txt. Crash reports carry a gpu.breadcrumbs tag (protocol 5).

Engine/Core/Debug/client.cpp

@@ -132,6 +132,9 @@ void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept {
         copy_cut(shared->gpu_driver, driver);
     }
 }
+void set_gpu_breadcrumbs(bool on) noexcept {
+    if (auto* const shared = report) shared->gpu_breadcrumbs = on ? 1 : 0;
+}
 void stop() noexcept {
     // An exception already using these handles owns them until process exit.
     // Never unmap its report or close its events from a concurrent shutdown.

Engine/Core/Debug/crash_report.h

@@ -18,4 +18,6 @@ void capture_game_fatal(DWORD thread_id, EXCEPTION_POINTERS* exception) noexcept
 // reporting is off; too long a value is cut short.
 void set_destination(std::wstring_view destination) noexcept;
 void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept;
+// Whether the GPU's breadcrumbs (DRED) are recorded this launch: the report's gpu.breadcrumbs tag.
+void set_gpu_breadcrumbs(bool on) noexcept;
 }

Engine/Core/Debug/protocol.h

@@ -4,7 +4,7 @@
 #include <string>
 
 namespace dingosdk::crash_report {
-inline constexpr DWORD protocol_version = 4;
+inline constexpr DWORD protocol_version = 5;
 // Who asked for the dump: ReSkate's last-chance filter, or the game's own fatal handler about to
 // write its dump (which it does without ever reaching that filter).
 enum class CaptureSource : DWORD { unhandled_exception, game_fatal_handler };
@@ -24,6 +24,8 @@ struct SharedReport {
     std::array<wchar_t, 128> gpu_name{};
     std::array<wchar_t, 32> gpu_driver{};
     CaptureSource source{};
+    // Whether D3D12's record of the GPU's work (DRED) is on for this launch (gpu_breadcrumbs()).
+    DWORD gpu_breadcrumbs{};
 };
 // Explicitly inherited unnamed handles; no token or paths on the command line.
 enum HandleIndex : std::size_t { mapping, requested, captured, stopping, ready, parent, handle_count };

Extension/Boot/game_fatal_dump.cpp

@@ -4,6 +4,7 @@
 #include "Engine/Core/Platform/memory.h"
 #include "Engine/Game/Build/addresses.h"
 #include "Engine/Game/Build/20260929/game_fatal_dump.h"
+#include "Extension/UI/Overlay/gpu_diagnostics.h"
 
 #include <Windows.h>
 #include <DbgHelp.h>
@@ -25,8 +26,12 @@ std::mutex install_mutex;
 BOOL WINAPI write_dump(HANDLE process, DWORD process_id, HANDLE file, MINIDUMP_TYPE type,
     PMINIDUMP_EXCEPTION_INFORMATION exception, PMINIDUMP_USER_STREAM_INFORMATION streams,
     PMINIDUMP_CALLBACK_INFORMATION callback) {
-    if (exception && exception->ExceptionPointers && process_id == GetCurrentProcessId())
+    if (exception && exception->ExceptionPointers && process_id == GetCurrentProcessId()) {
+        // (A GPU that stopped: the next launches record what it was doing.)
+        if (const auto* record = exception->ExceptionPointers->ExceptionRecord)
+            overlay::note_gpu_stopped(record->ExceptionCode);
         crash_report::capture_game_fatal(exception->ThreadId, exception->ExceptionPointers);
+    }
     return original_write.load(std::memory_order_acquire)(process, process_id, file, type, exception, streams, callback);
 }
 

Extension/UI/Overlay/gpu_diagnostics.cpp

@@ -4,9 +4,11 @@
 #include "Engine/Core/Debug/crash_report.h"
 
 #include <dxgi1_4.h>
+#include <ShlObj.h>
 #include <wrl/client.h>
 #include <intrin.h>
 #include <format>
+#include <algorithm>
 #include <array>
 #include <atomic>
 #include <cstdarg>
@@ -25,8 +27,62 @@ namespace {
 using Microsoft::WRL::ComPtr;
 struct Log {
     std::atomic<bool> removal_recorded{};
+    std::atomic<bool> breadcrumbs{};
 };
 Log& log() { static auto* value = new Log; return *value; }
+
+// After the GPU stops responding, this many launches record its breadcrumbs: enough for one of them
+// to catch the next stop, where it comes every few sessions.
+constexpr int breadcrumb_launches = 3;
+// %LOCALAPPDATA%\ReSkate\gpu-breadcrumbs.txt: how many launches are still to record them.
+std::filesystem::path breadcrumbs_note() {
+    PWSTR local{};
+    std::filesystem::path path;
+    if (SUCCEEDED(SHGetKnownFolderPath(FOLDERID_LocalAppData, KF_FLAG_DONT_VERIFY, nullptr, &local)))
+        path = std::filesystem::path(local) / L"ReSkate" / L"gpu-breadcrumbs.txt";
+    CoTaskMemFree(local);
+    return path;
+}
+int breadcrumbs_left() {
+    const auto path = breadcrumbs_note();
+    int left{};
+    if (FILE* file{}; !path.empty() && _wfopen_s(&file, path.c_str(), L"r") == 0 && file) {
+        if (fscanf_s(file, "%d", &left) != 1) left = 0;
+        std::fclose(file);
+    }
+    return std::clamp(left, 0, breadcrumb_launches);
+}
+void set_breadcrumbs_left(int left) {
+    const auto path = breadcrumbs_note();
+    if (path.empty()) return;
+    std::error_code error;
+    if (left <= 0) {
+        std::filesystem::remove(path, error);
+        return;
+    }
+    std::filesystem::create_directories(path.parent_path(), error);
+    if (FILE* file{}; _wfopen_s(&file, path.c_str(), L"w") == 0 && file) {
+        std::fprintf(file, "%d\n", left);
+        std::fclose(file);
+    }
+}
+}
+
+bool gpu_breadcrumbs() noexcept { return log().breadcrumbs.load(); }
+
+void note_gpu_stopped(DWORD exception_code) noexcept {
+    const auto error = GetLastError();
+    switch (exception_code) {
+    case static_cast<DWORD>(DXGI_ERROR_DEVICE_REMOVED):
+    case static_cast<DWORD>(DXGI_ERROR_DEVICE_HUNG):
+    case static_cast<DWORD>(DXGI_ERROR_DEVICE_RESET):
+    case static_cast<DWORD>(DXGI_ERROR_DRIVER_INTERNAL_ERROR):
+        try { set_breadcrumbs_left(breadcrumb_launches); } catch (...) {}
+        break;
+    default:
+        break;
+    }
+    SetLastError(error);
 }
 
 bool graphics_option_enabled(const wchar_t* name) noexcept {
@@ -40,7 +96,17 @@ void initialize_graphics_diagnostics() noexcept {
     try {
         const bool diagnostic = graphics_option_enabled(L"RESKATE_GPU_DIAGNOSTICS");
         dingosdk::logging::printf(dingosdk::logging::Level::debug, dingosdk::logging::Channel::graphics, "Graphics startup: gpu_diagnostics=%d", diagnostic);
-        if (diagnostic) {
+        // The launches after one in which the GPU stopped responding record its breadcrumbs, for
+        // the report of the next stop. This one counts as one of them.
+        const int left = breadcrumbs_left();
+        if (left > 0) {
+            set_breadcrumbs_left(left - 1);
+            dingosdk::logging::printf(dingosdk::logging::Level::info, dingosdk::logging::Channel::graphics,
+                "GPU breadcrumbs are on for this launch: the GPU stopped responding in a recent session (%d more launch(es) after this one).",
+                left - 1);
+        }
+        log().breadcrumbs = diagnostic || left > 0;
+        if (log().breadcrumbs) {
             // DRED must be configured before the game creates its D3D12 device.
             // Do not enable the debug layer or change the game's device flags.
             ComPtr<ID3D12DeviceRemovedExtendedDataSettings> settings;
@@ -1018,6 +1084,7 @@ void record_gpu_identity(ID3D12Device* device) noexcept {
                 driver = std::format(L"{}.{}.{}.{}", HIWORD(version.HighPart), LOWORD(version.HighPart), HIWORD(version.LowPart),
                     LOWORD(version.LowPart));
             dingosdk::crash_report::set_gpu(description.Description, driver);
+            dingosdk::crash_report::set_gpu_breadcrumbs(gpu_breadcrumbs());
             dingosdk::logging::write(dingosdk::logging::Level::info, dingosdk::logging::Channel::graphics,
                 std::wstring(L"Graphics card: ") + description.Description + (driver.empty() ? L"" : L", driver " + driver) + L".");
         }
@@ -1028,7 +1095,7 @@ void record_gpu_identity(ID3D12Device* device) noexcept {
 void watch_device_removal(ID3D12Device* device) noexcept {
     static std::atomic<bool> watching{};
     const auto error = GetLastError();
-    if (!device || !graphics_option_enabled(L"RESKATE_GPU_DIAGNOSTICS") || watching.exchange(true)) {
+    if (!device || watching.exchange(true)) {
         SetLastError(error);
         return;
     }
@@ -1050,7 +1117,7 @@ void watch_device_removal(ID3D12Device* device) noexcept {
         dingosdk::logging::write(dingosdk::logging::Level::warning, dingosdk::logging::Channel::graphics,
             "GPU diagnostics: the device-removal watch could not start; failures are recorded when the overlay next draws.");
     }
-    try {
+    if (gpu_breadcrumbs()) try {
         std::thread(watch_video_memory, device->GetAdapterLuid()).detach();
     } catch (...) {}
     SetLastError(error);
@@ -1073,6 +1140,8 @@ void record_device_failure(ID3D12Device* device, HRESULT result,
         const auto reason = device->GetDeviceRemovedReason();
         dingosdk::logging::printf(Level::error, Channel::graphics, "GPU failure: operation=%s result=0x%08lx removed_reason=0x%08lx",
             operation, static_cast<unsigned long>(result), static_cast<unsigned long>(reason));
+        // The next launches record what the GPU was doing, for the report of the next stop.
+        if (reason != S_OK) set_breadcrumbs_left(breadcrumb_launches);
         ComPtr<ID3D12DeviceRemovedExtendedData1> dred;
         if (FAILED(device->QueryInterface(IID_PPV_ARGS(&dred)))) {
             dingosdk::logging::write(Level::error, Channel::graphics, "DRED is unavailable on this device.");
@@ -1085,7 +1154,8 @@ void record_device_failure(ID3D12Device* device, HRESULT result,
         D3D12_DRED_AUTO_BREADCRUMBS_OUTPUT1 breadcrumbs{};
         if (const auto status = dred->GetAutoBreadcrumbsOutput1(&breadcrumbs); FAILED(status)) {
             dingosdk::logging::printf(Level::error, Channel::graphics,
-                "DRED breadcrumbs unavailable (0x%08lx): start the game with --gpu-diagnostics to record them.", static_cast<unsigned long>(status));
+                "DRED breadcrumbs unavailable (0x%08lx): the next %d launches record them.", static_cast<unsigned long>(status),
+                breadcrumb_launches);
         } else {
             unsigned finished{}, running{}, waiting{};
             for (auto* node = breadcrumbs.pHeadAutoBreadcrumbNode; node; node = node->pNext) {

Extension/UI/Overlay/gpu_diagnostics.h

@@ -7,11 +7,21 @@
 namespace dingosdk::overlay {
 bool graphics_option_enabled(const wchar_t* name) noexcept;
 void initialize_graphics_diagnostics() noexcept;
+// Whether D3D12 records what the GPU was doing (DRED's breadcrumbs and page faults), so that when
+// it stops responding the log says which work it stopped in. On with RESKATE_GPU_DIAGNOSTICS, and
+// by itself (none of the diagnostics' other watches) for the next few launches after one in which
+// the GPU stopped: record_device_failure leaves a note for them. Decided by
+// initialize_graphics_diagnostics, before the game makes its device.
+bool gpu_breadcrumbs() noexcept;
+// The game is about to stop for an exception: one that is a DXGI device error (the GPU stopped
+// responding, its driver failed) leaves the note for the next launches, as record_device_failure
+// does, for when that came first.
+void note_gpu_stopped(DWORD exception_code) noexcept;
 void record_device_failure(ID3D12Device* device, HRESULT result,
     const char* operation) noexcept;
-// With RESKATE_GPU_DIAGNOSTICS on: records the failure the moment the device is removed, from a
-// thread of its own, instead of when the overlay next draws. The game's own error dialog can
-// otherwise come first and end the process before then.
+// Records the failure the moment the device is removed, from a thread of its own, instead of when
+// the overlay next draws: the game's own error dialog can otherwise come first and end the process
+// before then. Always on (a fence and a wait); with the breadcrumbs on, video memory is watched too.
 void watch_device_removal(ID3D12Device* device) noexcept;
 // With RESKATE_GPU_DIAGNOSTICS on: every DrawInstanced and DrawIndexedInstanced of the process (the
 // runtime's, found through this command list) is checked, and one of more than ten million

Launcher/crash_reporter.cpp

@@ -103,6 +103,7 @@ bool publish_dump(HANDLE process, const std::filesystem::path& temporary, const
     }
     if (const auto gpu = field(report.gpu_name); !gpu.empty()) attributes["gpu.name"] = gpu;
     if (const auto driver = field(report.gpu_driver); !driver.empty()) attributes["gpu.driver"] = driver;
+    attributes["gpu.breadcrumbs"] = report.gpu_breadcrumbs ? "on" : "off";
     const auto log = snapshot_log(std::filesystem::path(report.directory.data()).parent_path() / L"ReSkate.log", dump);
     attributes["attachment.log"] = log == LogSnapshot::complete ? "complete" : log == LogSnapshot::tail ? "tail" : "unavailable";
     // Where it crashed ("Skate.exe+0x3d9ed9d"): Sentry names game code by the nearest export, so a