diff --git a/Engine/Core/Debug/client.cpp b/Engine/Core/Debug/client.cpp index 5f2d0c8d..a9665503 100644 --- a/Engine/Core/Debug/client.cpp +++ b/Engine/Core/Debug/client.cpp @@ -132,6 +132,9 @@ void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept { copy_cut(shared->gpu_driver, driver); } } +void set_gpu_breadcrumbs(bool on) noexcept { + if (auto* const shared = report) shared->gpu_breadcrumbs = on ? 1 : 0; +} void stop() noexcept { // An exception already using these handles owns them until process exit. // Never unmap its report or close its events from a concurrent shutdown. diff --git a/Engine/Core/Debug/crash_report.h b/Engine/Core/Debug/crash_report.h index 231f4ca0..4c8e0cfd 100644 --- a/Engine/Core/Debug/crash_report.h +++ b/Engine/Core/Debug/crash_report.h @@ -18,4 +18,6 @@ void capture_game_fatal(DWORD thread_id, EXCEPTION_POINTERS* exception) noexcept // reporting is off; too long a value is cut short. void set_destination(std::wstring_view destination) noexcept; void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept; +// Whether the GPU's breadcrumbs (DRED) are recorded this launch: the report's gpu.breadcrumbs tag. +void set_gpu_breadcrumbs(bool on) noexcept; } diff --git a/Engine/Core/Debug/protocol.h b/Engine/Core/Debug/protocol.h index 902d2a23..fc56a090 100644 --- a/Engine/Core/Debug/protocol.h +++ b/Engine/Core/Debug/protocol.h @@ -4,7 +4,7 @@ #include namespace dingosdk::crash_report { -inline constexpr DWORD protocol_version = 4; +inline constexpr DWORD protocol_version = 5; // Who asked for the dump: ReSkate's last-chance filter, or the game's own fatal handler about to // write its dump (which it does without ever reaching that filter). enum class CaptureSource : DWORD { unhandled_exception, game_fatal_handler }; @@ -24,6 +24,8 @@ struct SharedReport { std::array gpu_name{}; std::array gpu_driver{}; CaptureSource source{}; + // Whether D3D12's record of the GPU's work (DRED) is on for this launch (gpu_breadcrumbs()). + DWORD gpu_breadcrumbs{}; }; // Explicitly inherited unnamed handles; no token or paths on the command line. enum HandleIndex : std::size_t { mapping, requested, captured, stopping, ready, parent, handle_count }; diff --git a/Extension/Boot/game_fatal_dump.cpp b/Extension/Boot/game_fatal_dump.cpp index bcf487fb..39ce4b1b 100644 --- a/Extension/Boot/game_fatal_dump.cpp +++ b/Extension/Boot/game_fatal_dump.cpp @@ -4,6 +4,7 @@ #include "Engine/Core/Platform/memory.h" #include "Engine/Game/Build/addresses.h" #include "Engine/Game/Build/20260929/game_fatal_dump.h" +#include "Extension/UI/Overlay/gpu_diagnostics.h" #include #include @@ -25,8 +26,12 @@ std::mutex install_mutex; BOOL WINAPI write_dump(HANDLE process, DWORD process_id, HANDLE file, MINIDUMP_TYPE type, PMINIDUMP_EXCEPTION_INFORMATION exception, PMINIDUMP_USER_STREAM_INFORMATION streams, PMINIDUMP_CALLBACK_INFORMATION callback) { - if (exception && exception->ExceptionPointers && process_id == GetCurrentProcessId()) + if (exception && exception->ExceptionPointers && process_id == GetCurrentProcessId()) { + // (A GPU that stopped: the next launches record what it was doing.) + if (const auto* record = exception->ExceptionPointers->ExceptionRecord) + overlay::note_gpu_stopped(record->ExceptionCode); crash_report::capture_game_fatal(exception->ThreadId, exception->ExceptionPointers); + } return original_write.load(std::memory_order_acquire)(process, process_id, file, type, exception, streams, callback); } diff --git a/Extension/UI/Overlay/gpu_diagnostics.cpp b/Extension/UI/Overlay/gpu_diagnostics.cpp index 358fe7db..3a4a6674 100644 --- a/Extension/UI/Overlay/gpu_diagnostics.cpp +++ b/Extension/UI/Overlay/gpu_diagnostics.cpp @@ -4,9 +4,11 @@ #include "Engine/Core/Debug/crash_report.h" #include +#include #include #include #include +#include #include #include #include @@ -25,8 +27,62 @@ namespace { using Microsoft::WRL::ComPtr; struct Log { std::atomic removal_recorded{}; + std::atomic breadcrumbs{}; }; Log& log() { static auto* value = new Log; return *value; } + +// After the GPU stops responding, this many launches record its breadcrumbs: enough for one of them +// to catch the next stop, where it comes every few sessions. +constexpr int breadcrumb_launches = 3; +// %LOCALAPPDATA%\ReSkate\gpu-breadcrumbs.txt: how many launches are still to record them. +std::filesystem::path breadcrumbs_note() { + PWSTR local{}; + std::filesystem::path path; + if (SUCCEEDED(SHGetKnownFolderPath(FOLDERID_LocalAppData, KF_FLAG_DONT_VERIFY, nullptr, &local))) + path = std::filesystem::path(local) / L"ReSkate" / L"gpu-breadcrumbs.txt"; + CoTaskMemFree(local); + return path; +} +int breadcrumbs_left() { + const auto path = breadcrumbs_note(); + int left{}; + if (FILE* file{}; !path.empty() && _wfopen_s(&file, path.c_str(), L"r") == 0 && file) { + if (fscanf_s(file, "%d", &left) != 1) left = 0; + std::fclose(file); + } + return std::clamp(left, 0, breadcrumb_launches); +} +void set_breadcrumbs_left(int left) { + const auto path = breadcrumbs_note(); + if (path.empty()) return; + std::error_code error; + if (left <= 0) { + std::filesystem::remove(path, error); + return; + } + std::filesystem::create_directories(path.parent_path(), error); + if (FILE* file{}; _wfopen_s(&file, path.c_str(), L"w") == 0 && file) { + std::fprintf(file, "%d\n", left); + std::fclose(file); + } +} +} + +bool gpu_breadcrumbs() noexcept { return log().breadcrumbs.load(); } + +void note_gpu_stopped(DWORD exception_code) noexcept { + const auto error = GetLastError(); + switch (exception_code) { + case static_cast(DXGI_ERROR_DEVICE_REMOVED): + case static_cast(DXGI_ERROR_DEVICE_HUNG): + case static_cast(DXGI_ERROR_DEVICE_RESET): + case static_cast(DXGI_ERROR_DRIVER_INTERNAL_ERROR): + try { set_breadcrumbs_left(breadcrumb_launches); } catch (...) {} + break; + default: + break; + } + SetLastError(error); } bool graphics_option_enabled(const wchar_t* name) noexcept { @@ -40,7 +96,17 @@ void initialize_graphics_diagnostics() noexcept { try { const bool diagnostic = graphics_option_enabled(L"RESKATE_GPU_DIAGNOSTICS"); dingosdk::logging::printf(dingosdk::logging::Level::debug, dingosdk::logging::Channel::graphics, "Graphics startup: gpu_diagnostics=%d", diagnostic); - if (diagnostic) { + // The launches after one in which the GPU stopped responding record its breadcrumbs, for + // the report of the next stop. This one counts as one of them. + const int left = breadcrumbs_left(); + if (left > 0) { + set_breadcrumbs_left(left - 1); + dingosdk::logging::printf(dingosdk::logging::Level::info, dingosdk::logging::Channel::graphics, + "GPU breadcrumbs are on for this launch: the GPU stopped responding in a recent session (%d more launch(es) after this one).", + left - 1); + } + log().breadcrumbs = diagnostic || left > 0; + if (log().breadcrumbs) { // DRED must be configured before the game creates its D3D12 device. // Do not enable the debug layer or change the game's device flags. ComPtr settings; @@ -1018,6 +1084,7 @@ void record_gpu_identity(ID3D12Device* device) noexcept { driver = std::format(L"{}.{}.{}.{}", HIWORD(version.HighPart), LOWORD(version.HighPart), HIWORD(version.LowPart), LOWORD(version.LowPart)); dingosdk::crash_report::set_gpu(description.Description, driver); + dingosdk::crash_report::set_gpu_breadcrumbs(gpu_breadcrumbs()); dingosdk::logging::write(dingosdk::logging::Level::info, dingosdk::logging::Channel::graphics, std::wstring(L"Graphics card: ") + description.Description + (driver.empty() ? L"" : L", driver " + driver) + L"."); } @@ -1028,7 +1095,7 @@ void record_gpu_identity(ID3D12Device* device) noexcept { void watch_device_removal(ID3D12Device* device) noexcept { static std::atomic watching{}; const auto error = GetLastError(); - if (!device || !graphics_option_enabled(L"RESKATE_GPU_DIAGNOSTICS") || watching.exchange(true)) { + if (!device || watching.exchange(true)) { SetLastError(error); return; } @@ -1050,7 +1117,7 @@ void watch_device_removal(ID3D12Device* device) noexcept { dingosdk::logging::write(dingosdk::logging::Level::warning, dingosdk::logging::Channel::graphics, "GPU diagnostics: the device-removal watch could not start; failures are recorded when the overlay next draws."); } - try { + if (gpu_breadcrumbs()) try { std::thread(watch_video_memory, device->GetAdapterLuid()).detach(); } catch (...) {} SetLastError(error); @@ -1073,6 +1140,8 @@ void record_device_failure(ID3D12Device* device, HRESULT result, const auto reason = device->GetDeviceRemovedReason(); dingosdk::logging::printf(Level::error, Channel::graphics, "GPU failure: operation=%s result=0x%08lx removed_reason=0x%08lx", operation, static_cast(result), static_cast(reason)); + // The next launches record what the GPU was doing, for the report of the next stop. + if (reason != S_OK) set_breadcrumbs_left(breadcrumb_launches); ComPtr dred; if (FAILED(device->QueryInterface(IID_PPV_ARGS(&dred)))) { dingosdk::logging::write(Level::error, Channel::graphics, "DRED is unavailable on this device."); @@ -1085,7 +1154,8 @@ void record_device_failure(ID3D12Device* device, HRESULT result, D3D12_DRED_AUTO_BREADCRUMBS_OUTPUT1 breadcrumbs{}; if (const auto status = dred->GetAutoBreadcrumbsOutput1(&breadcrumbs); FAILED(status)) { dingosdk::logging::printf(Level::error, Channel::graphics, - "DRED breadcrumbs unavailable (0x%08lx): start the game with --gpu-diagnostics to record them.", static_cast(status)); + "DRED breadcrumbs unavailable (0x%08lx): the next %d launches record them.", static_cast(status), + breadcrumb_launches); } else { unsigned finished{}, running{}, waiting{}; for (auto* node = breadcrumbs.pHeadAutoBreadcrumbNode; node; node = node->pNext) { diff --git a/Extension/UI/Overlay/gpu_diagnostics.h b/Extension/UI/Overlay/gpu_diagnostics.h index 7b9cbaf8..a5986ae6 100644 --- a/Extension/UI/Overlay/gpu_diagnostics.h +++ b/Extension/UI/Overlay/gpu_diagnostics.h @@ -7,11 +7,21 @@ namespace dingosdk::overlay { bool graphics_option_enabled(const wchar_t* name) noexcept; void initialize_graphics_diagnostics() noexcept; +// Whether D3D12 records what the GPU was doing (DRED's breadcrumbs and page faults), so that when +// it stops responding the log says which work it stopped in. On with RESKATE_GPU_DIAGNOSTICS, and +// by itself (none of the diagnostics' other watches) for the next few launches after one in which +// the GPU stopped: record_device_failure leaves a note for them. Decided by +// initialize_graphics_diagnostics, before the game makes its device. +bool gpu_breadcrumbs() noexcept; +// The game is about to stop for an exception: one that is a DXGI device error (the GPU stopped +// responding, its driver failed) leaves the note for the next launches, as record_device_failure +// does, for when that came first. +void note_gpu_stopped(DWORD exception_code) noexcept; void record_device_failure(ID3D12Device* device, HRESULT result, const char* operation) noexcept; -// With RESKATE_GPU_DIAGNOSTICS on: records the failure the moment the device is removed, from a -// thread of its own, instead of when the overlay next draws. The game's own error dialog can -// otherwise come first and end the process before then. +// Records the failure the moment the device is removed, from a thread of its own, instead of when +// the overlay next draws: the game's own error dialog can otherwise come first and end the process +// before then. Always on (a fence and a wait); with the breadcrumbs on, video memory is watched too. void watch_device_removal(ID3D12Device* device) noexcept; // With RESKATE_GPU_DIAGNOSTICS on: every DrawInstanced and DrawIndexedInstanced of the process (the // runtime's, found through this command list) is checked, and one of more than ten million diff --git a/Launcher/crash_reporter.cpp b/Launcher/crash_reporter.cpp index 5a46bfc4..eb9ff477 100644 --- a/Launcher/crash_reporter.cpp +++ b/Launcher/crash_reporter.cpp @@ -103,6 +103,7 @@ bool publish_dump(HANDLE process, const std::filesystem::path& temporary, const } if (const auto gpu = field(report.gpu_name); !gpu.empty()) attributes["gpu.name"] = gpu; if (const auto driver = field(report.gpu_driver); !driver.empty()) attributes["gpu.driver"] = driver; + attributes["gpu.breadcrumbs"] = report.gpu_breadcrumbs ? "on" : "off"; const auto log = snapshot_log(std::filesystem::path(report.directory.data()).parent_path() / L"ReSkate.log", dump); attributes["attachment.log"] = log == LogSnapshot::complete ? "complete" : log == LogSnapshot::tail ? "tail" : "unavailable"; // Where it crashed ("Skate.exe+0x3d9ed9d"): Sentry names game code by the nearest export, so a