mirror of
https://github.com/SpeedBreakerProject/speedbreaker.git
synced 2026-10-17 22:01:34 +02:00
d4c206ab98
Co-authored-by: love-gamer <340996855+love-gamer@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1283 lines
65 KiB
C++
1283 lines
65 KiB
C++
// SpeedBreaker runtime. GPL-3.0-or-later (see COPYING).
|
|
// PM4 packet semantics follow Xenia (gpu/command_processor.cc, BSD license).
|
|
#include <stdafx.h>
|
|
#include "command_processor.h"
|
|
|
|
#include <kernel/vmem.h>
|
|
#include <cpu/guest_thread.h>
|
|
#include <cpu/host_cpu.h>
|
|
#include <cpu/guest_time.h>
|
|
#include <kernel/function.h>
|
|
#include <video/presenter.h>
|
|
#include <video/frame_rate.h>
|
|
#include <video/vblank_lock.h>
|
|
#include <video/vblank_timer.h>
|
|
#include <video/display_gamma.h>
|
|
#include <platform/thermal.h>
|
|
#include <game/frame_time.h>
|
|
#include "renderer.h"
|
|
#include "gpu_clock.h"
|
|
#include "zpd_report.h"
|
|
#include <kernel/write_watch.h>
|
|
#include <debug/timeline.h>
|
|
#include <report/report.h>
|
|
#include <unordered_set>
|
|
|
|
namespace gpu
|
|
{
|
|
namespace
|
|
{
|
|
// Registers the sync logic touches (Xenia's register_table.inc).
|
|
constexpr uint32_t REG_CP_RB_WPTR = 0x01C5;
|
|
constexpr uint32_t REG_SCRATCH_UMSK = 0x01DC;
|
|
constexpr uint32_t REG_SCRATCH_ADDR = 0x01DD;
|
|
constexpr uint32_t REG_SCRATCH_REG0 = 0x0578;
|
|
constexpr uint32_t REG_SCRATCH_REG7 = 0x057F;
|
|
constexpr uint32_t REG_COHER_STATUS_HOST = 0x0A31;
|
|
constexpr uint32_t REG_VGT_EVENT_INITIATOR = 0x21F9;
|
|
constexpr uint32_t REG_RB_SAMPLE_COUNT_ADDR = 0x2325;
|
|
constexpr uint32_t REG_PA_SC_VIZ_QUERY_STATUS_0 = 0x0C44;
|
|
constexpr uint32_t REG_PA_SC_VIZ_QUERY_STATUS_1 = 0x0C45;
|
|
|
|
enum Opcode : uint32_t
|
|
{
|
|
PM4_ME_INIT = 0x48, PM4_NOP = 0x10, PM4_INDIRECT_BUFFER = 0x3F, PM4_INDIRECT_BUFFER_PFD = 0x37,
|
|
PM4_WAIT_FOR_IDLE = 0x26, PM4_WAIT_REG_MEM = 0x3C, PM4_REG_RMW = 0x21, PM4_REG_TO_MEM = 0x3E,
|
|
PM4_MEM_WRITE = 0x3D, PM4_COND_WRITE = 0x45, PM4_EVENT_WRITE = 0x46, PM4_EVENT_WRITE_SHD = 0x58,
|
|
PM4_EVENT_WRITE_EXT = 0x5A, PM4_EVENT_WRITE_ZPD = 0x5B, PM4_DRAW_INDX = 0x22, PM4_DRAW_INDX_2 = 0x36,
|
|
PM4_VIZ_QUERY = 0x23, PM4_SET_CONSTANT = 0x2D, PM4_SET_CONSTANT2 = 0x55, PM4_SET_SHADER_CONSTANTS = 0x56,
|
|
PM4_LOAD_ALU_CONSTANT = 0x2F, PM4_IM_LOAD = 0x27, PM4_IM_LOAD_IMMEDIATE = 0x2B,
|
|
PM4_INVALIDATE_STATE = 0x3B, PM4_SET_BIN_MASK = 0x50, PM4_SET_BIN_SELECT = 0x51,
|
|
PM4_CONTEXT_UPDATE = 0x5E, PM4_INTERRUPT = 0x54, PM4_XE_SWAP = 0x64,
|
|
PM4_SET_BIN_MASK_LO = 0x60, PM4_SET_BIN_MASK_HI = 0x61, PM4_SET_BIN_SELECT_LO = 0x62, PM4_SET_BIN_SELECT_HI = 0x63,
|
|
};
|
|
|
|
// Page-aligned, as is s_history: where the hot arrays fall within a
|
|
// 4 KB page otherwise depends on every global linked before them, and
|
|
// loads that share their low 12 address bits with a recent store wait
|
|
// for it (4K aliasing: ls_bad_status2.stli_other). The play-test
|
|
// report code's buffers moved them and the command processor went
|
|
// from 12.5 to 15 ms in the fly-in, the same instructions at an IPC
|
|
// of 0.94 instead of 1.28.
|
|
alignas(4096) uint32_t s_regs[REGISTER_COUNT];
|
|
|
|
std::atomic<uint32_t> s_primaryBuffer{ 0 }; // physical
|
|
uint32_t s_primarySize = 0; // bytes
|
|
uint32_t s_readIndex = 0; // dwords
|
|
std::atomic<uint32_t> s_readWriteback{ 0 }; // physical, 0 = off
|
|
std::atomic<uint32_t> s_interruptCallback{ 0 };
|
|
std::atomic<uint32_t> s_interruptData{ 0 };
|
|
std::atomic<uint64_t> s_frames{ 0 };
|
|
std::atomic<uint64_t> s_draws{ 0 };
|
|
std::atomic<uint32_t> s_counter{ 0 }; // Xenia's counter_: swaps + vblanks (two threads count)
|
|
uint64_t s_binMask = ~0ull, s_binSelect = ~0ull;
|
|
std::atomic<bool> s_started{ false };
|
|
std::atomic<uint32_t> s_vblanks{ 0 }; // guest vblank interrupts raised
|
|
// The WAIT_REG_MEM being polled past its first check, for the hang
|
|
// watchdog (WaitingSince, DescribeState): since when (steady clock,
|
|
// ns; 0: none), and what it waits for.
|
|
std::atomic<int64_t> s_waitSince{ 0 };
|
|
std::atomic<uint32_t> s_waitInfo{ 0 }, s_waitPoll{ 0 }, s_waitRef{ 0 }, s_waitMask{ 0 };
|
|
|
|
// Busy time (renderer::g_cpBusyNs) is ring execution not blocked in a
|
|
// wait, accounted at each batch's end and at each swap (mid-batch),
|
|
// so a frame's share of it is exact for hitch reports.
|
|
std::chrono::steady_clock::time_point s_busyFrom;
|
|
uint64_t s_busyWaitFrom = 0;
|
|
void AccountBusy()
|
|
{
|
|
auto now = std::chrono::steady_clock::now();
|
|
uint64_t ns = uint64_t(std::chrono::duration_cast<std::chrono::nanoseconds>(now - s_busyFrom).count());
|
|
renderer::g_cpBusyNs += ns - std::min(ns, renderer::g_cpWaitNs - s_busyWaitFrom);
|
|
s_busyFrom = now;
|
|
s_busyWaitFrom = renderer::g_cpWaitNs;
|
|
}
|
|
|
|
uint8_t* Physical(uint32_t address)
|
|
{
|
|
return static_cast<uint8_t*>(g_memory.Translate(vmem::Physical().Base() | (address & 0x1FFFFFFF)));
|
|
}
|
|
|
|
be<uint32_t>* MmioRegister(uint32_t index)
|
|
{
|
|
return static_cast<be<uint32_t>*>(g_memory.Translate(MMIO_BASE + index * 4));
|
|
}
|
|
|
|
uint32_t GpuSwap(uint32_t value, uint32_t endian)
|
|
{
|
|
switch (endian & 3)
|
|
{
|
|
case 1: return ((value << 8) & 0xFF00FF00) | ((value >> 8) & 0x00FF00FF); // 8in16
|
|
case 2: return ByteSwap(value); // 8in32
|
|
case 3: return (value >> 16) | (value << 16); // 16in32
|
|
default: return value;
|
|
}
|
|
}
|
|
|
|
// Stores `value` so that the guest (big-endian) reads it after the
|
|
// GPU's endian swap, as xe::store(TranslatePhysical(...)) does in Xenia.
|
|
// Guest stores by the command processor (and the completion thread)
|
|
// go around the write-watch GPU guard: these threads never wait on
|
|
// the GPU they feed.
|
|
void GuestStore(uint32_t address, const void* data, size_t size)
|
|
{
|
|
writewatch::GpuThreadStore(Physical(address), data, size);
|
|
}
|
|
|
|
void StoreSwapped(uint32_t address, uint32_t value)
|
|
{
|
|
uint32_t v = GpuSwap(value, address & 3);
|
|
GuestStore(address & ~3u, &v, 4);
|
|
}
|
|
|
|
uint32_t LoadSwapped(uint32_t address)
|
|
{
|
|
uint32_t v;
|
|
memcpy(&v, Physical(address & ~3u), 4);
|
|
return GpuSwap(v, address & 3);
|
|
}
|
|
|
|
// NFSMW_OCCLUSION_FAKE=<n> (diagnostic, never shipped): every
|
|
// occlusion query END reports n samples passed, so nothing is ever
|
|
// occluded (the sun glare's hypothesis test; 0 or unset: zeros). It
|
|
// overrides NFSMW_OCCLUSION=1, whose [zpd] lines then count nothing:
|
|
// logged.
|
|
const uint32_t s_occlusionFake = [] {
|
|
const char* v = std::getenv("NFSMW_OCCLUSION_FAKE");
|
|
uint32_t samples = v ? uint32_t(std::strtoul(v, nullptr, 0)) : 0u;
|
|
if (samples)
|
|
fprintf(stderr, "[gpu] NFSMW_OCCLUSION_FAKE=%u: every occlusion query END reports %u samples (a diagnostic; "
|
|
"it overrides NFSMW_OCCLUSION=1)\n", samples, samples);
|
|
return samples;
|
|
}();
|
|
|
|
// An END report of `samples`, in the order D3D's GetData needs
|
|
// (gpu/zpd_report.h: B lanes, ZPass_A, then the other A lanes).
|
|
void StoreZpdEnd(uint32_t address, uint32_t samples)
|
|
{
|
|
uint8_t* host = Physical(address);
|
|
writewatch::GpuThreadWrite(host, 32, [host, samples] {
|
|
zpd::StoreEnd(samples, [host](uint32_t offset, uint32_t value) { memcpy(host + offset, &value, 4); });
|
|
});
|
|
}
|
|
|
|
// Memory stores of the command stream are deferred to GPU completion
|
|
// (renderer::QueueWrite). The ones still pending are tracked, so that
|
|
// a WAIT_REG_MEM on a value the stream itself will store (D3D's
|
|
// pipeline-drain idiom: EVENT_WRITE_SHD then wait for it) can go on
|
|
// at once: everything recorded after it executes after it in queue
|
|
// order, and its CPU-visible effects are deferred behind it too.
|
|
// Waiting instead drains the whole host GPU queue (5-10 ms, several
|
|
// times a frame). NFSMW_SELF_WAITS=0 waits like the hardware.
|
|
struct PendingStore { uint32_t value; uint64_t sequence; };
|
|
std::mutex s_pendingMutex;
|
|
std::unordered_map<uint32_t, PendingStore> s_pendingStores; // by physical address
|
|
uint64_t s_pendingSequence = 0;
|
|
const bool s_selfWaits = [] { const char* v = std::getenv("NFSMW_SELF_WAITS"); return !v || v[0] != '0'; }();
|
|
|
|
uint64_t AddPending(uint32_t address, uint32_t value)
|
|
{
|
|
std::lock_guard lock(s_pendingMutex);
|
|
uint64_t sequence = ++s_pendingSequence;
|
|
s_pendingStores[address & 0x1FFFFFFC] = { value, sequence };
|
|
return sequence;
|
|
}
|
|
|
|
void ClearPending(uint32_t address, uint64_t sequence)
|
|
{
|
|
std::lock_guard lock(s_pendingMutex);
|
|
auto it = s_pendingStores.find(address & 0x1FFFFFFC);
|
|
if (it != s_pendingStores.end() && it->second.sequence == sequence)
|
|
s_pendingStores.erase(it);
|
|
}
|
|
|
|
void QueueStores(uint32_t address, std::vector<uint32_t> values)
|
|
{
|
|
std::vector<uint64_t> sequences(values.size());
|
|
{
|
|
std::lock_guard lock(s_pendingMutex);
|
|
for (size_t i = 0; i < values.size(); i++)
|
|
{
|
|
sequences[i] = ++s_pendingSequence;
|
|
s_pendingStores[(address + uint32_t(i) * 4) & 0x1FFFFFFC] = { values[i], sequences[i] };
|
|
}
|
|
}
|
|
renderer::QueueWrite([address, values = std::move(values), sequences = std::move(sequences)] {
|
|
std::lock_guard lock(s_pendingMutex);
|
|
for (size_t i = 0; i < values.size(); i++)
|
|
{
|
|
uint32_t a = address + uint32_t(i) * 4;
|
|
StoreSwapped(a, values[i]);
|
|
auto it = s_pendingStores.find(a & 0x1FFFFFFC);
|
|
if (it != s_pendingStores.end() && it->second.sequence == sequences[i])
|
|
s_pendingStores.erase(it);
|
|
}
|
|
});
|
|
}
|
|
|
|
void QueueStore(uint32_t address, uint32_t value)
|
|
{
|
|
QueueStores(address, { value });
|
|
}
|
|
|
|
// The value the stream will have stored at `address` once the GPU
|
|
// catches up, if a store to it is pending.
|
|
std::optional<uint32_t> PendingValue(uint32_t address)
|
|
{
|
|
std::lock_guard lock(s_pendingMutex);
|
|
auto it = s_pendingStores.find(address & 0x1FFFFFFC);
|
|
if (it == s_pendingStores.end())
|
|
return std::nullopt;
|
|
return it->second.value;
|
|
}
|
|
|
|
void WriteRegister(uint32_t index, uint32_t value)
|
|
{
|
|
if (index >= REGISTER_COUNT)
|
|
return;
|
|
s_regs[index] = value;
|
|
if (index >= 0x4000 && index < 0x4928) // float/fetch/bool/loop constants
|
|
renderer::g_constantsDirty.store(true, std::memory_order_relaxed);
|
|
if (index >= REG_SCRATCH_REG0 && index <= REG_SCRATCH_REG7)
|
|
{
|
|
// Scratch registers are fences: the CPU sees them (MMIO and
|
|
// the optional memory copy) once the work before is done.
|
|
uint32_t n = index - REG_SCRATCH_REG0;
|
|
bool toMemory = ((1u << n) & s_regs[REG_SCRATCH_UMSK]) != 0;
|
|
uint32_t address = s_regs[REG_SCRATCH_ADDR] + n * 4;
|
|
uint64_t sequence = toMemory ? AddPending(address, value) : 0;
|
|
renderer::QueueWrite([=] {
|
|
if (toMemory)
|
|
{
|
|
be<uint32_t> big = value;
|
|
GuestStore(address, &big, 4);
|
|
ClearPending(address, sequence);
|
|
}
|
|
*MmioRegister(index) = value;
|
|
});
|
|
return;
|
|
}
|
|
else if (index == REG_COHER_STATUS_HOST)
|
|
{
|
|
s_regs[index] |= 0x80000000u;
|
|
}
|
|
else if (video::display_gamma::IsLutRegister(index))
|
|
{
|
|
// The display gamma ramp (video/display_gamma.h): stored as
|
|
// any register, and followed for the presenter.
|
|
video::display_gamma::OnRegisterWrite(index, value);
|
|
}
|
|
if (index != REG_CP_RB_WPTR) // the game owns that one
|
|
*MmioRegister(index) = s_regs[index];
|
|
}
|
|
|
|
// A run of registers WriteRegister has nothing special for (it treats
|
|
// only the scratch fences, COHER_STATUS_HOST, the write pointer and
|
|
// the display gamma ramp's registers specially) can be written in bulk.
|
|
bool BulkSafe(uint32_t base, uint32_t count)
|
|
{
|
|
uint32_t end = base + count;
|
|
if (!(renderer::g_cpOpt & renderer::CP_OPT_BULK_REGISTERS) || end > REGISTER_COUNT)
|
|
return false;
|
|
auto hits = [&](uint32_t lo, uint32_t hi) { return base <= hi && end > lo; };
|
|
return !hits(REG_CP_RB_WPTR, REG_CP_RB_WPTR) && !hits(REG_SCRATCH_REG0, REG_SCRATCH_REG7) &&
|
|
!hits(REG_COHER_STATUS_HOST, REG_COHER_STATUS_HOST) &&
|
|
!hits(video::display_gamma::kRegRwMode, video::display_gamma::kRegLutaControl);
|
|
}
|
|
|
|
// WriteRegister for `count` registers from `src`, big-endian dwords
|
|
// (the ring's or guest memory's bytes, which is also how the MMIO
|
|
// mirror holds them). A heavy frame writes ~600k registers, mostly
|
|
// shader constants in runs.
|
|
void WriteRegistersBulk(uint32_t base, const uint8_t* src, uint32_t count)
|
|
{
|
|
memcpy(MmioRegister(base), src, size_t(count) * 4);
|
|
for (uint32_t m = 0; m < count; m++)
|
|
{
|
|
uint32_t v;
|
|
memcpy(&v, src + size_t(m) * 4, 4);
|
|
s_regs[base + m] = ByteSwap(v);
|
|
}
|
|
if (base < 0x4928 && base + count > 0x4000) // float/fetch/bool/loop constants
|
|
renderer::g_constantsDirty.store(true, std::memory_order_relaxed);
|
|
}
|
|
|
|
// One host thread per interrupt source, each with its own guest
|
|
// context, as Xenia runs the callback on the GPU/vsync threads.
|
|
void DispatchInterrupt(PPCContext& ctx, uint32_t source, uint32_t cpu)
|
|
{
|
|
timeline::Mark(timeline::Interrupt, source | cpu << 8);
|
|
uint32_t callback = s_interruptCallback;
|
|
if (callback == 0)
|
|
return;
|
|
if (ImportTraceEnabled())
|
|
fprintf(stderr, "[trace t%u] GPU interrupt source %u cpu %u -> %08X\n",
|
|
GuestThread::GetCurrentThreadId(), source, cpu, callback);
|
|
// Run the handler "on" the target CPU, as Xenia's SetActiveCpu does:
|
|
// the PCR's processor number (r13 + 0x10C) selects D3D's per-CPU state.
|
|
*static_cast<uint8_t*>(g_memory.Translate(ctx.r13.u32 + 0x10C)) = uint8_t(cpu);
|
|
ctx.r3.u64 = source;
|
|
ctx.r4.u64 = s_interruptData;
|
|
g_memory.FindFunction(callback)(ctx, g_memory.base);
|
|
}
|
|
|
|
struct Reader
|
|
{
|
|
uint8_t* base;
|
|
uint32_t size; // bytes (ring size for the primary buffer)
|
|
uint32_t offset; // bytes
|
|
uint32_t end; // bytes (write offset)
|
|
|
|
uint32_t Available() const { return (end + size - offset) % size + (end == offset ? 0 : 0); }
|
|
uint32_t Read()
|
|
{
|
|
uint32_t v;
|
|
memcpy(&v, base + offset, 4);
|
|
offset += 4;
|
|
if (offset >= size)
|
|
offset = 0;
|
|
return ByteSwap(v);
|
|
}
|
|
void Skip(uint32_t dwords) { offset = (offset + dwords * 4) % size; }
|
|
uint8_t* Here() const { return base + offset; }
|
|
// The next `dwords` as one contiguous run (nullptr if they wrap);
|
|
// consumes them.
|
|
const uint8_t* Take(uint32_t dwords)
|
|
{
|
|
if (offset + dwords * 4 > size)
|
|
return nullptr;
|
|
const uint8_t* p = base + offset;
|
|
offset += dwords * 4;
|
|
if (offset >= size)
|
|
offset = 0;
|
|
return p;
|
|
}
|
|
};
|
|
|
|
void ExecuteBuffer(Reader& r, PPCContext& ctx);
|
|
|
|
// NFSMW_DUMP_SHADERS=<dir>: save each distinct shader's microcode
|
|
// (guest byte order) as <dir>/<vs|ps>_<hash>.bin for offline work.
|
|
void DumpShader(uint32_t type, const uint32_t* words, uint32_t dwords)
|
|
{
|
|
static const char* dir = std::getenv("NFSMW_DUMP_SHADERS");
|
|
if (dir == nullptr || dwords == 0)
|
|
return;
|
|
static std::mutex mutex;
|
|
static std::unordered_set<uint64_t> seen;
|
|
uint64_t h = 0xCBF29CE484222325ull ^ type;
|
|
for (uint32_t i = 0; i < dwords; i++)
|
|
h = (h ^ words[i]) * 0x100000001B3ull;
|
|
std::lock_guard lock(mutex);
|
|
if (!seen.insert(h).second)
|
|
return;
|
|
char path[512];
|
|
snprintf(path, sizeof(path), "%s/%s_%016llx.bin", dir, type == 0 ? "vs" : "ps", (unsigned long long)h);
|
|
if (FILE* f = fopen(path, "wb"))
|
|
{
|
|
fwrite(words, 4, dwords, f);
|
|
fclose(f);
|
|
}
|
|
}
|
|
|
|
// Last packets executed, for diagnosing stuck waits.
|
|
struct PacketRecord { uint32_t header; uint32_t args[4]; };
|
|
alignas(4096) PacketRecord s_history[64]; // (see s_regs)
|
|
uint32_t s_historyPos = 0;
|
|
|
|
void Record(uint32_t header, const Reader& r)
|
|
{
|
|
PacketRecord& rec = s_history[s_historyPos++ % 64];
|
|
rec.header = header;
|
|
uint32_t count = (header >> 30) == 3 ? ((header >> 16) & 0x3FFF) + 1 : 0;
|
|
uint32_t n = std::min(count, 4u);
|
|
if (r.offset + n * 4 <= r.size)
|
|
{
|
|
for (uint32_t i = 0; i < 4; i++)
|
|
{
|
|
uint32_t v = 0;
|
|
if (i < n)
|
|
memcpy(&v, r.base + r.offset + i * 4, 4);
|
|
rec.args[i] = ByteSwap(v);
|
|
}
|
|
return;
|
|
}
|
|
Reader peek = r;
|
|
for (uint32_t i = 0; i < 4; i++)
|
|
rec.args[i] = i < count ? peek.Read() : 0;
|
|
}
|
|
|
|
// (Also read by a hang report on the watchdog's thread, racily: by
|
|
// then this thread is stuck and the records hold still.)
|
|
std::string FormatHistory()
|
|
{
|
|
std::string text = "[gpu] last packets (oldest first):\n";
|
|
for (uint32_t i = 0; i < 64; i++)
|
|
{
|
|
const PacketRecord& rec = s_history[(s_historyPos + i) % 64];
|
|
if (rec.header == 0)
|
|
continue;
|
|
uint32_t type = rec.header >> 30;
|
|
if (type == 3)
|
|
text += std::format(" type3 op {:02X} count {}: {:08X} {:08X} {:08X} {:08X}\n", (rec.header >> 8) & 0x7F,
|
|
((rec.header >> 16) & 0x3FFF) + 1, rec.args[0], rec.args[1], rec.args[2], rec.args[3]);
|
|
else if (type == 0)
|
|
text += std::format(" type0 reg {:04X} count {}\n", rec.header & 0x7FFF, ((rec.header >> 16) & 0x3FFF) + 1);
|
|
}
|
|
return text;
|
|
}
|
|
|
|
void DumpHistory()
|
|
{
|
|
fputs(FormatHistory().c_str(), stderr);
|
|
}
|
|
|
|
bool ExecuteType3(Reader& r, uint32_t packet, PPCContext& ctx)
|
|
{
|
|
uint32_t opcode = (packet >> 8) & 0x7F;
|
|
uint32_t count = ((packet >> 16) & 0x3FFF) + 1;
|
|
|
|
if ((packet & 1) && ((s_binSelect & s_binMask) == 0 || opcode == PM4_XE_SWAP))
|
|
{
|
|
r.Skip(count); // predicated packet that doesn't apply
|
|
return true;
|
|
}
|
|
|
|
auto compare = [](uint32_t func, uint32_t value, uint32_t ref) {
|
|
switch (func & 7)
|
|
{
|
|
case 1: return value < ref;
|
|
case 2: return value <= ref;
|
|
case 3: return value == ref;
|
|
case 4: return value != ref;
|
|
case 5: return value >= ref;
|
|
case 6: return value > ref;
|
|
case 7: return true;
|
|
default: return false;
|
|
}
|
|
};
|
|
|
|
switch (opcode)
|
|
{
|
|
case PM4_ME_INIT:
|
|
case PM4_NOP:
|
|
case PM4_INVALIDATE_STATE:
|
|
case PM4_CONTEXT_UPDATE:
|
|
case PM4_WAIT_FOR_IDLE: // we execute in order; idle is implied
|
|
r.Skip(count);
|
|
return true;
|
|
|
|
case PM4_INTERRUPT:
|
|
{
|
|
// Raised when the GPU reaches this point: dispatched by the
|
|
// renderer's completion thread (with a guest context of its
|
|
// own) once the work before it has finished, not by draining.
|
|
uint32_t cpuMask = r.Read();
|
|
r.Skip(count - 1);
|
|
renderer::QueueWrite([cpuMask] {
|
|
// Runs on the completion thread, or right here when the GPU
|
|
// is idle (this thread already has a guest context).
|
|
PPCContext* guest = GetPPCContext();
|
|
if (!guest)
|
|
{
|
|
static thread_local GuestThreadContext* interruptThread = new GuestThreadContext(2);
|
|
guest = &interruptThread->ppcContext;
|
|
}
|
|
for (uint32_t n = 0; n < 6; n++)
|
|
if (cpuMask & (1u << n))
|
|
DispatchInterrupt(*guest, 1, n);
|
|
});
|
|
return true;
|
|
}
|
|
|
|
case PM4_XE_SWAP:
|
|
{
|
|
r.Read(); // 'SWAP'
|
|
r.Read(); // front buffer (physical); also in the fetch constant
|
|
uint32_t width = r.Read(), height = r.Read();
|
|
r.Skip(count - 4);
|
|
// VdSwap wrote the front buffer's fetch constant into fetch slot 0.
|
|
// Presented from the GPU without waiting; frame dumps (and the
|
|
// headless case) take the CPU path, which needs the GPU idle.
|
|
static const bool dumping = std::getenv("NFSMW_DUMP_FRAMES") != nullptr;
|
|
if (dumping || !renderer::PresentFrontBuffer(&s_regs[0x4800], width, height))
|
|
{
|
|
renderer::Flush();
|
|
video::SubmitFrontBuffer(&s_regs[0x4800], width, height);
|
|
}
|
|
s_counter++;
|
|
timeline::Mark(timeline::Swap);
|
|
uint64_t frame = ++s_frames;
|
|
renderer::OnSwap(frame);
|
|
// Every 120 frames: frame rate and per-frame costs since the last line.
|
|
// Frame times are guest time (guesttime::NowNs: the host's
|
|
// on desktop), so a frame across a suspension isn't a hitch
|
|
// and the frame rate counts only the time the game ran.
|
|
static int64_t lastTime = guesttime::NowNs();
|
|
static renderer::Stats last{};
|
|
static uint64_t lastDraws = 0;
|
|
static writewatch::GuardStats lastGuard{};
|
|
// Stutter: the longest gap between two swaps in the window.
|
|
static int64_t lastSwap = guesttime::NowNs();
|
|
static int64_t overlayTime = guesttime::NowNs(); // the last 30-swap mark
|
|
static double worstMs = 0;
|
|
{
|
|
AccountBusy();
|
|
int64_t swapNow = guesttime::NowNs();
|
|
double frameMs = double(swapNow - lastSwap) / 1e6;
|
|
worstMs = std::max(worstMs, frameMs);
|
|
static uint32_t lastVblanks = 0;
|
|
uint32_t vblanks = s_vblanks.load();
|
|
renderer::HitchReport(frame, frameMs, vblanks - lastVblanks);
|
|
lastVblanks = vblanks;
|
|
lastSwap = swapNow;
|
|
}
|
|
if (frame % 30 == 0)
|
|
{
|
|
static uint64_t overlayBusyNs = 0, overlayDraws = 0, overlayGpuUs = 0;
|
|
// GPU: execution time from timestamps where the device has them.
|
|
auto st = renderer::GetStats();
|
|
uint64_t gpuUs = st.gpuExecUs ? st.gpuExecUs : st.gpuBusyUs, draws = s_draws.load();
|
|
float cpMs = float(double(renderer::g_cpBusyNs - overlayBusyNs) / 1e6 / 30);
|
|
float gpuMs = float(double(gpuUs - overlayGpuUs) / 1000.0 / 30);
|
|
float drawsPerFrame = float(double(draws - overlayDraws) / 30);
|
|
renderer::PublishPerfNumbers({ cpMs, gpuMs, drawsPerFrame });
|
|
overlayBusyNs = renderer::g_cpBusyNs;
|
|
overlayDraws = draws;
|
|
overlayGpuUs = gpuUs;
|
|
// The same numbers, with the game time they took, for
|
|
// Frame Rate's Auto (every half second at 60).
|
|
int64_t costNow = guesttime::NowNs();
|
|
video::NoteFrameCosts({ double(costNow - overlayTime) / 1e9, 30, cpMs, gpuMs, drawsPerFrame });
|
|
overlayTime = costNow;
|
|
}
|
|
if (frame % 120 == 0)
|
|
{
|
|
auto st = renderer::GetStats();
|
|
auto guard = writewatch::GetGuardStats();
|
|
// The scaled data path, and the presenter's GPU time per
|
|
// game frame it post-processed (NFSMW_GPU_TIMING=1): with
|
|
// the renderer's executing time, the whole GPU frame.
|
|
auto scaled = renderer::GetScaledStats();
|
|
auto memory = renderer::GetMemoryStats();
|
|
auto post = video::GetPostTiming();
|
|
static renderer::ScaledStats lastScaled{};
|
|
static renderer::MemoryStats lastMemory{};
|
|
static video::PostTiming lastPost{};
|
|
double postFrames = double(std::max<uint64_t>(post.frames - lastPost.frames, 1));
|
|
int64_t now = guesttime::NowNs();
|
|
double secs = double(now - lastTime) / 1e9;
|
|
// CP busy: time in ring batches not blocked on the GPU or the game.
|
|
static uint64_t lastBusyNs = 0;
|
|
double cpBusyMs = double(renderer::g_cpBusyNs - lastBusyNs) / 1e6 / 120;
|
|
double drawsPerFrame = double(s_draws.load() - lastDraws) / 120;
|
|
lastBusyNs = renderer::g_cpBusyNs;
|
|
// On screen: frames the presenter dropped or kept up
|
|
// past their vblanks; the world's steps per game frame.
|
|
static video::PresentStats lastShown{};
|
|
static game::FrameTimeStats lastWorld{};
|
|
auto shown = video::GetPresentStats();
|
|
auto world = game::GetFrameTimeStats();
|
|
// The GPU's average clock, where the kernel keeps it (the Frame).
|
|
double mhz = gpu::clock::AverageMHzSinceLast();
|
|
std::string clockText = mhz > 0.0 ? std::format(", at {:.0f} MHz", mhz) : std::string();
|
|
// At the end, so parsers of the fields before are untouched:
|
|
// the device's thermal state (Apple, or NFSMW_THERMAL_SIM),
|
|
// and Frame Rate Auto's rate.
|
|
platform::thermal::State thermal = platform::thermal::Current();
|
|
std::string tailText = (thermal == platform::thermal::State::Unknown ? std::string()
|
|
: std::format(" | thermal {}", platform::thermal::Name(thermal))) + video::AutoFrameRatePerfText();
|
|
// NFSMW_WATCH_SUBPAGES in use: its snapshots (write_watch.cpp).
|
|
// Pool-full evictions, whole-page fallbacks and guest pages
|
|
// marked for a GPU read are where it loses its gain (which,
|
|
// for a free-roam run); untile waits, what it costs instead
|
|
// (gave up: the wait hit its bound, see WaitForGpu).
|
|
std::string subpagesText = !guard.subpages ? std::string()
|
|
: std::format(", subpages: {:.1f} opened, {:.1f} clean, {:.1f} changed, {:.1f} evicted ({:.1f} pool full), {:.1f} whole (GPU),"
|
|
" {:.1f} marked (GPU read), {:.2f} untile waits ({:.2f} gave up)",
|
|
double(guard.snapOpened - lastGuard.snapOpened) / 120, double(guard.snapClean - lastGuard.snapClean) / 120,
|
|
double(guard.snapChanged - lastGuard.snapChanged) / 120, double(guard.snapEvicted - lastGuard.snapEvicted) / 120,
|
|
double(guard.snapFull - lastGuard.snapFull) / 120, double(guard.snapWhole - lastGuard.snapWhole) / 120,
|
|
double(guard.snapReading - lastGuard.snapReading) / 120, double(guard.snapWaits - lastGuard.snapWaits) / 120,
|
|
double(guard.snapGaveUp - lastGuard.snapGaveUp) / 120);
|
|
fprintf(stderr, "[perf] %.1f fps (worst frame %.1f ms) | CP busy %.1f ms/frame (%.2f us/draw) | per frame: %.0f draws (%.0f skipped as later tiles, %.0f constant uploads), %.1f passes, %.1f resolves, %.1f submits, %.1f uploads, GPU busy %.1f ms (executing %.1f ms%s) | guard: %.1f faults (%.1f opened), %.2f stalls, %.2f ms stalled%s | %.0f KB snapshots | shadow: %.0f KB up, %.0f KB back, %.2f waits"
|
|
" | scaled: %.1f loads, %.0f MB untiled, %.0f MB resolved, %.2f allocations, %.2f evictions, %.2f fallbacks | post-processing GPU %.2f ms (AA %.2f ms)"
|
|
" | display: %llu dropped, %llu repeated | world: %llu frames, %llu without a step, %llu with 2+ | late %llu"
|
|
" | memory: %.0f MB (targets %.0f in %u, textures %.0f in %u, scaled %.0f, front %.0f), device %.0f of %.0f MB,"
|
|
" image blocks %u (%.0f MB: %.0f MB in %u images), %u spare (%.0f MB), %u images alone (%.0f MB)"
|
|
" | images: %llu made (%.0f us each), %llu allocations (%.1f ms); blocks: %llu from the spare, %llu allocated here,"
|
|
" %llu by the memory thread (%.1f ms each), %llu freed%s\n",
|
|
120.0 / secs, worstMs, cpBusyMs, drawsPerFrame > 0 ? cpBusyMs * 1000.0 / drawsPerFrame : 0.0,
|
|
drawsPerFrame, double(st.tileSkips - last.tileSkips) / 120,
|
|
double(st.constantUploads - last.constantUploads) / 120,
|
|
double(st.passes - last.passes) / 120, double(st.resolves - last.resolves) / 120,
|
|
double(st.submits - last.submits) / 120, double(st.uploads - last.uploads) / 120,
|
|
double(st.gpuBusyUs - last.gpuBusyUs) / 1000.0 / 120, double(st.gpuExecUs - last.gpuExecUs) / 1000.0 / 120,
|
|
clockText.c_str(), double(guard.faults - lastGuard.faults) / 120, double(guard.opened - lastGuard.opened) / 120, double(guard.stalls - lastGuard.stalls) / 120,
|
|
double(guard.stallUs - lastGuard.stallUs) / 1000.0 / 120, subpagesText.c_str(),
|
|
double(st.snapshotBytes - last.snapshotBytes) / 1024.0 / 120,
|
|
double(guard.uploadBytes - lastGuard.uploadBytes) / 1024.0 / 120, double(guard.readbackBytes - lastGuard.readbackBytes) / 1024.0 / 120,
|
|
double(st.shadowWaits - last.shadowWaits) / 120, double(scaled.loads - lastScaled.loads) / 120,
|
|
double(scaled.untiledBytes - lastScaled.untiledBytes) / 1048576.0 / 120,
|
|
double(scaled.resolvedBytes - lastScaled.resolvedBytes) / 1048576.0 / 120,
|
|
double(scaled.allocations - lastScaled.allocations) / 120, double(scaled.evictions - lastScaled.evictions) / 120,
|
|
double(scaled.fallbacks - lastScaled.fallbacks) / 120, double(post.totalNs - lastPost.totalNs) / 1e6 / postFrames,
|
|
double(post.aaNs - lastPost.aaNs) / 1e6 / postFrames,
|
|
(unsigned long long)(shown.dropped - lastShown.dropped), (unsigned long long)(shown.repeated - lastShown.repeated),
|
|
(unsigned long long)(world.frames - lastWorld.frames), (unsigned long long)(world.noStep - lastWorld.noStep),
|
|
(unsigned long long)(world.multiStep - lastWorld.multiStep), (unsigned long long)(st.lateFrames - last.lateFrames),
|
|
double(memory.targets + memory.textures + memory.scaled + memory.front) / 1048576.0, double(memory.targets) / 1048576.0,
|
|
memory.targetCount, double(memory.textures) / 1048576.0, memory.textureCount, double(memory.scaled) / 1048576.0,
|
|
double(memory.front) / 1048576.0, double(memory.deviceUsed) / 1048576.0, double(memory.deviceBudget) / 1048576.0,
|
|
memory.blocks, double(memory.blockBytes) / 1048576.0, double(memory.rangeBytes) / 1048576.0, memory.ranges,
|
|
memory.spares, double(memory.spareBytes) / 1048576.0, memory.own, double(memory.ownBytes) / 1048576.0,
|
|
(unsigned long long)(memory.images - lastMemory.images),
|
|
memory.images > lastMemory.images ? double(memory.imageNs - lastMemory.imageNs) / 1000.0 / double(memory.images - lastMemory.images) : 0.0,
|
|
(unsigned long long)(memory.allocations - lastMemory.allocations), double(memory.allocationNs - lastMemory.allocationNs) / 1e6,
|
|
(unsigned long long)(memory.fromSpare - lastMemory.fromSpare), (unsigned long long)(memory.blocksHere - lastMemory.blocksHere),
|
|
(unsigned long long)(memory.spareBlocks - lastMemory.spareBlocks),
|
|
memory.spareBlocks > lastMemory.spareBlocks ? double(memory.spareNs - lastMemory.spareNs) / 1e6 / double(memory.spareBlocks - lastMemory.spareBlocks) : 0.0,
|
|
(unsigned long long)(memory.blocksFreed - lastMemory.blocksFreed), tailText.c_str());
|
|
lastMemory = memory;
|
|
lastShown = shown;
|
|
lastWorld = world;
|
|
lastGuard = guard;
|
|
lastScaled = scaled;
|
|
lastPost = post;
|
|
worstMs = 0;
|
|
lastTime = now;
|
|
last = st;
|
|
lastDraws = s_draws.load();
|
|
}
|
|
if (frame == 1 || frame % 300 == 0)
|
|
{
|
|
auto st = renderer::GetStats();
|
|
fprintf(stderr, "[gpu] frame %llu (%llu draws so far; rendered %llu, skipped %llu, resolves %llu, "
|
|
"submits %llu, pipelines %llu, uploads %llu, shaders from cache %llu)\n", (unsigned long long)frame, (unsigned long long)s_draws.load(),
|
|
(unsigned long long)st.draws, (unsigned long long)st.skipped, (unsigned long long)st.resolves,
|
|
(unsigned long long)st.submits, (unsigned long long)st.pipelines, (unsigned long long)st.uploads,
|
|
(unsigned long long)st.shaderCacheHits);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
case PM4_INDIRECT_BUFFER:
|
|
case PM4_INDIRECT_BUFFER_PFD:
|
|
{
|
|
uint32_t ptr = r.Read() & 0x1FFFFFFF;
|
|
uint32_t length = r.Read() & 0xFFFFF;
|
|
r.Skip(count - 2);
|
|
Reader ib{ Physical(ptr), length * 4 + 4, 0, length * 4 }; // +4: never wrap
|
|
ExecuteBuffer(ib, ctx);
|
|
return true;
|
|
}
|
|
|
|
case PM4_WAIT_REG_MEM:
|
|
{
|
|
uint32_t info = r.Read(), poll = r.Read(), ref = r.Read(), mask = r.Read(), wait = r.Read();
|
|
r.Skip(count - 5);
|
|
bool memory = (info & 0x10) != 0;
|
|
std::optional<renderer::CpWaitTimer> waitTimer; // from the first poll after the submit
|
|
auto waitStart = std::chrono::steady_clock::now();
|
|
// Suspensions so far: one that begins during the wait, even
|
|
// one spent parked in the GPU gate inside Submit() (it closes
|
|
// after the count moves), restarts the wait's clock.
|
|
uint64_t waitEpoch = guesttime::Epoch();
|
|
bool reported = false, submitted = false;
|
|
// NFSMW_TEST_HANG: no wait passes (report/watchdog.cpp).
|
|
if (memory && s_selfWaits && !report::g_testHang.load(std::memory_order_relaxed))
|
|
if (auto pending = PendingValue(poll); pending && compare(info, *pending & mask, ref))
|
|
return true;
|
|
while (true)
|
|
{
|
|
uint32_t value = memory ? LoadSwapped(poll) : s_regs[poll % REGISTER_COUNT];
|
|
if (!memory && poll == REG_COHER_STATUS_HOST)
|
|
{
|
|
s_regs[poll] = 0; // MakeCoherent: nothing to flush in a null GPU
|
|
value = 0;
|
|
}
|
|
if (compare(info, value & mask, ref) && !report::g_testHang.load(std::memory_order_relaxed))
|
|
{
|
|
if (submitted)
|
|
{
|
|
timeline::Mark(timeline::RegMemEnd, value);
|
|
s_waitSince.store(0, std::memory_order_relaxed);
|
|
}
|
|
break;
|
|
}
|
|
// Deferred fence writes may be what the CPU is waiting for
|
|
// before it produces what we wait for: get them moving.
|
|
// (Most of these waits, MakeCoherent's, pass at once and
|
|
// submitted nothing: ~30 command buffers a frame.)
|
|
if (!submitted)
|
|
{
|
|
submitted = true;
|
|
timeline::Mark(timeline::RegMemBegin, poll);
|
|
s_waitInfo.store(info, std::memory_order_relaxed);
|
|
s_waitPoll.store(poll, std::memory_order_relaxed);
|
|
s_waitRef.store(ref, std::memory_order_relaxed);
|
|
s_waitMask.store(mask, std::memory_order_relaxed);
|
|
s_waitSince.store(std::chrono::duration_cast<std::chrono::nanoseconds>(
|
|
waitStart.time_since_epoch()).count(), std::memory_order_release);
|
|
static const bool logWaits = std::getenv("NFSMW_LOG_REGMEM") != nullptr;
|
|
static int logged = 0;
|
|
if (logWaits && logged < 40)
|
|
{
|
|
logged++;
|
|
auto pending = PendingValue(poll);
|
|
fprintf(stderr, "[regmem] %s %08X func %u value %08X mask %08X ref %08X pending %s%08X | last packets:\n",
|
|
memory ? "mem" : "reg", poll, info & 7, value, mask, ref, pending ? "" : "none ", pending.value_or(0));
|
|
DumpHistory();
|
|
}
|
|
renderer::Submit();
|
|
waitTimer.emplace(renderer::CP_WAIT_REGMEM);
|
|
continue;
|
|
}
|
|
if (guesttime::Suspended() || guesttime::Epoch() != waitEpoch)
|
|
{
|
|
// What it waits for comes from game threads or the
|
|
// vblank, which stand still while the game is
|
|
// suspended: wait for the resume instead of polling,
|
|
// then count the wait (its stuck report, the hang
|
|
// watchdog's) from there.
|
|
waitTimer.reset();
|
|
{
|
|
renderer::CpWaitTimer held(renderer::CP_WAIT_SUSPENDED);
|
|
guesttime::WaitWhileSuspended();
|
|
}
|
|
waitEpoch = guesttime::Epoch();
|
|
waitTimer.emplace(renderer::CP_WAIT_REGMEM);
|
|
waitStart = std::chrono::steady_clock::now();
|
|
reported = false;
|
|
s_waitSince.store(std::chrono::duration_cast<std::chrono::nanoseconds>(
|
|
waitStart.time_since_epoch()).count(), std::memory_order_release);
|
|
continue;
|
|
}
|
|
if (!reported && std::chrono::steady_clock::now() - waitStart > std::chrono::seconds(1))
|
|
{
|
|
reported = true;
|
|
fprintf(stderr, "[gpu] WAIT_REG_MEM stuck >1s: %s %08X, func %u, value %08X & mask %08X vs ref %08X\n",
|
|
memory ? "memory" : "register", poll, info & 7, value, mask, ref);
|
|
DumpHistory();
|
|
}
|
|
// The guest's wait interval is a hint for hardware polling;
|
|
// sleeping whole milliseconds per poll added a millisecond
|
|
// or more to every wait. Spin with yields for 2 ms, then
|
|
// back off to short sleeps.
|
|
(void)wait;
|
|
if (std::chrono::steady_clock::now() - waitStart < std::chrono::milliseconds(2))
|
|
std::this_thread::yield();
|
|
else
|
|
std::this_thread::sleep_for(std::chrono::microseconds(100));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
case PM4_REG_RMW:
|
|
{
|
|
uint32_t info = r.Read(), andMask = r.Read(), orMask = r.Read();
|
|
r.Skip(count - 3);
|
|
uint32_t value = s_regs[info & 0x1FFF];
|
|
value &= (info >> 31) & 1 ? s_regs[andMask & 0x1FFF] : andMask;
|
|
value |= (info >> 30) & 1 ? s_regs[orMask & 0x1FFF] : orMask;
|
|
WriteRegister(info & 0x1FFF, value);
|
|
return true;
|
|
}
|
|
|
|
case PM4_REG_TO_MEM:
|
|
{
|
|
uint32_t reg = r.Read(), address = r.Read();
|
|
r.Skip(count - 2);
|
|
uint32_t value = s_regs[reg % REGISTER_COUNT];
|
|
QueueStore(address, value);
|
|
return true;
|
|
}
|
|
|
|
case PM4_MEM_WRITE:
|
|
{
|
|
uint32_t address = r.Read();
|
|
std::vector<uint32_t> values(count - 1);
|
|
for (uint32_t& v : values)
|
|
v = r.Read();
|
|
QueueStores(address, std::move(values));
|
|
return true;
|
|
}
|
|
|
|
case PM4_COND_WRITE:
|
|
{
|
|
uint32_t info = r.Read(), poll = r.Read(), ref = r.Read(), mask = r.Read();
|
|
uint32_t writeAddr = r.Read(), data = r.Read();
|
|
r.Skip(count - 6);
|
|
uint32_t value = (info & 0x10) ? LoadSwapped(poll) : s_regs[poll % REGISTER_COUNT];
|
|
if (compare(info, value & mask, ref))
|
|
{
|
|
if (info & 0x100)
|
|
QueueStore(writeAddr, data);
|
|
else
|
|
WriteRegister(writeAddr, data);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
case PM4_EVENT_WRITE:
|
|
WriteRegister(REG_VGT_EVENT_INITIATOR, r.Read() & 0x3F);
|
|
r.Skip(count - 1);
|
|
return true;
|
|
|
|
case PM4_EVENT_WRITE_SHD:
|
|
{
|
|
uint32_t initiator = r.Read(), address = r.Read(), value = r.Read();
|
|
r.Skip(count - 3);
|
|
static const bool logFences = std::getenv("NFSMW_LOG_D3D_WAIT") != nullptr;
|
|
if (logFences)
|
|
{
|
|
static std::unordered_set<uint32_t> seen;
|
|
if (seen.insert(address).second)
|
|
fprintf(stderr, "[d3dwait] EVENT_WRITE_SHD fence address %08X (read pointer write-back %08X)\n",
|
|
address, uint32_t(s_readWriteback));
|
|
}
|
|
WriteRegister(REG_VGT_EVENT_INITIATOR, initiator & 0x3F);
|
|
QueueStore(address, (initiator >> 31) & 1 ? s_counter.load() : value);
|
|
return true;
|
|
}
|
|
|
|
case PM4_EVENT_WRITE_EXT:
|
|
{
|
|
uint32_t initiator = r.Read(), address = r.Read();
|
|
r.Skip(count - 2);
|
|
WriteRegister(REG_VGT_EVENT_INITIATOR, initiator & 0x3F);
|
|
renderer::QueueWrite([=] {
|
|
// Screen extents (8in16): the whole 8192x8192 range, z 0..1.
|
|
const uint16_t extents[] = { 0, 8192 >> 3, 0, 8192 >> 3, 0, 1 };
|
|
be<uint16_t> out[6];
|
|
for (int i = 0; i < 6; i++)
|
|
out[i] = extents[i];
|
|
GuestStore(address & ~3u, out, sizeof(out));
|
|
});
|
|
return true;
|
|
}
|
|
|
|
case PM4_EVENT_WRITE_ZPD:
|
|
{
|
|
// Occlusion queries: report "finished" with zero samples,
|
|
// unless they are counted.
|
|
WriteRegister(REG_VGT_EVENT_INITIATOR, r.Read() & 0x3F);
|
|
r.Skip(count - 1);
|
|
uint32_t addr = s_regs[REG_RB_SAMPLE_COUNT_ADDR];
|
|
// D3D's reports are 64-byte blocks: BEGIN at +0x20, END at +0.
|
|
// NFSMW_OCCLUSION=1 counts them (an END's report is then the
|
|
// renderer's, stored at GPU completion); the fake overrides it.
|
|
if (addr != 0 && s_occlusionFake && zpd::Classify(addr) == zpd::Kind::End)
|
|
renderer::QueueWrite([=] { StoreZpdEnd(addr, s_occlusionFake); });
|
|
else if (addr != 0 && (s_occlusionFake || !renderer::ZpassDone(addr)))
|
|
renderer::QueueWrite([=] {
|
|
const uint8_t zeros[32] = {};
|
|
GuestStore(addr, zeros, sizeof(zeros));
|
|
});
|
|
return true;
|
|
}
|
|
|
|
case PM4_DRAW_INDX:
|
|
case PM4_DRAW_INDX_2:
|
|
{
|
|
s_draws++;
|
|
// DRAW_INDX: viz query id, VGT_DRAW_INITIATOR, then VGT_DMA_BASE
|
|
// and VGT_DMA_SIZE for DMA indices. DRAW_INDX_2 has no viz dword.
|
|
uint32_t used = 0;
|
|
if (opcode == PM4_DRAW_INDX)
|
|
{
|
|
r.Read();
|
|
used++;
|
|
}
|
|
uint32_t initiator = r.Read();
|
|
used++;
|
|
WriteRegister(0x21FC, initiator); // VGT_DRAW_INITIATOR
|
|
uint32_t dmaBase = 0, dmaSize = 0;
|
|
if (((initiator >> 6) & 3) == 0 && count >= used + 2) // SourceSelect::kDMA
|
|
{
|
|
dmaBase = r.Read();
|
|
dmaSize = r.Read();
|
|
used += 2;
|
|
WriteRegister(0x21FA, dmaBase); // VGT_DMA_BASE
|
|
WriteRegister(0x21FB, dmaSize); // VGT_DMA_SIZE
|
|
}
|
|
r.Skip(count - used);
|
|
renderer::Draw(initiator, dmaBase, dmaSize);
|
|
return true;
|
|
}
|
|
|
|
case PM4_VIZ_QUERY:
|
|
{
|
|
uint32_t d = r.Read();
|
|
r.Skip(count - 1);
|
|
uint32_t id = d & 0x3F;
|
|
if (d & 0x100)
|
|
s_regs[id < 32 ? REG_PA_SC_VIZ_QUERY_STATUS_0 : REG_PA_SC_VIZ_QUERY_STATUS_1] |= 1u << (id & 31);
|
|
return true;
|
|
}
|
|
|
|
case PM4_SET_CONSTANT:
|
|
case PM4_LOAD_ALU_CONSTANT:
|
|
{
|
|
uint32_t address = opcode == PM4_LOAD_ALU_CONSTANT ? (r.Read() & 0x3FFFFFFF) : 0;
|
|
uint32_t offsetType = r.Read();
|
|
uint32_t size = opcode == PM4_LOAD_ALU_CONSTANT ? (r.Read() & 0xFFF) : count - 1;
|
|
uint32_t index = offsetType & 0x7FF;
|
|
static constexpr uint32_t bases[] = { 0x4000, 0x4800, 0x4900, 0x4908, 0x2000 };
|
|
uint32_t type = (offsetType >> 16) & 0xFF;
|
|
if (type > 4)
|
|
{
|
|
r.Skip(opcode == PM4_LOAD_ALU_CONSTANT ? count - 3 : count - 1);
|
|
return true;
|
|
}
|
|
index += bases[type];
|
|
if (BulkSafe(index, size))
|
|
{
|
|
if (opcode == PM4_LOAD_ALU_CONSTANT)
|
|
{
|
|
WriteRegistersBulk(index, Physical(address), size);
|
|
r.Skip(count - 3);
|
|
return true;
|
|
}
|
|
if (const uint8_t* run = r.Take(size))
|
|
{
|
|
WriteRegistersBulk(index, run, size);
|
|
return true;
|
|
}
|
|
}
|
|
for (uint32_t n = 0; n < size; n++, index++)
|
|
WriteRegister(index, opcode == PM4_LOAD_ALU_CONSTANT
|
|
? uint32_t(*reinterpret_cast<be<uint32_t>*>(Physical(address + n * 4)))
|
|
: r.Read());
|
|
if (opcode == PM4_LOAD_ALU_CONSTANT)
|
|
r.Skip(count - 3);
|
|
return true;
|
|
}
|
|
|
|
case PM4_SET_CONSTANT2:
|
|
case PM4_SET_SHADER_CONSTANTS:
|
|
{
|
|
uint32_t index = r.Read() & 0xFFFF;
|
|
if (BulkSafe(index, count - 1))
|
|
if (const uint8_t* run = r.Take(count - 1))
|
|
{
|
|
WriteRegistersBulk(index, run, count - 1);
|
|
return true;
|
|
}
|
|
for (uint32_t n = 0; n < count - 1; n++, index++)
|
|
WriteRegister(index, r.Read());
|
|
return true;
|
|
}
|
|
|
|
case PM4_IM_LOAD:
|
|
{
|
|
uint32_t addrType = r.Read(), startSize = r.Read();
|
|
r.Skip(count - 2);
|
|
const auto* words = reinterpret_cast<const uint32_t*>(Physical(addrType & ~3u));
|
|
DumpShader(addrType & 3, words, startSize & 0xFFFF);
|
|
renderer::LoadShader(addrType & 3, words, startSize & 0xFFFF);
|
|
return true;
|
|
}
|
|
case PM4_IM_LOAD_IMMEDIATE:
|
|
{
|
|
uint32_t type = r.Read(), startSize = r.Read();
|
|
uint32_t size = startSize & 0xFFFF;
|
|
std::vector<uint32_t> words(size);
|
|
for (uint32_t i = 0; i < size; i++)
|
|
{
|
|
uint32_t v = r.Read();
|
|
words[i] = ByteSwap(v); // keep guest (big-endian) order, like IM_LOAD memory
|
|
}
|
|
r.Skip(count - 2 - size);
|
|
DumpShader(type & 3, words.data(), size);
|
|
renderer::LoadShader(type & 3, words.data(), size);
|
|
return true;
|
|
}
|
|
|
|
case PM4_SET_BIN_MASK_LO: s_binMask = (s_binMask & ~0xFFFFFFFFull) | r.Read(); r.Skip(count - 1); return true;
|
|
case PM4_SET_BIN_MASK_HI: s_binMask = (s_binMask & 0xFFFFFFFFull) | (uint64_t(r.Read()) << 32); r.Skip(count - 1); return true;
|
|
case PM4_SET_BIN_SELECT_LO: s_binSelect = (s_binSelect & ~0xFFFFFFFFull) | r.Read(); r.Skip(count - 1); return true;
|
|
case PM4_SET_BIN_SELECT_HI: s_binSelect = (s_binSelect & 0xFFFFFFFFull) | (uint64_t(r.Read()) << 32); r.Skip(count - 1); return true;
|
|
case PM4_SET_BIN_MASK: { uint64_t hi = r.Read(); s_binMask = (hi << 32) | r.Read(); r.Skip(count - 2); return true; }
|
|
case PM4_SET_BIN_SELECT: { uint64_t hi = r.Read(); s_binSelect = (hi << 32) | r.Read(); r.Skip(count - 2); return true; }
|
|
}
|
|
|
|
static uint32_t s_reported[128];
|
|
if (!s_reported[opcode]++)
|
|
fprintf(stderr, "[gpu] PM4 opcode %02X not handled (skipped, %u dwords)\n", opcode, count);
|
|
r.Skip(count);
|
|
return true;
|
|
}
|
|
|
|
void ExecuteBuffer(Reader& r, PPCContext& ctx)
|
|
{
|
|
while (r.offset != r.end)
|
|
{
|
|
uint32_t packet = r.Read();
|
|
if (packet == 0)
|
|
continue;
|
|
if ((packet >> 30) != 2)
|
|
Record(packet, r);
|
|
switch (packet >> 30)
|
|
{
|
|
case 0: // write `count` registers from `base` (or one register repeatedly)
|
|
{
|
|
uint32_t count = ((packet >> 16) & 0x3FFF) + 1;
|
|
uint32_t base = packet & 0x7FFF;
|
|
bool one = (packet >> 15) & 1;
|
|
if (!one && BulkSafe(base, count))
|
|
if (const uint8_t* run = r.Take(count))
|
|
{
|
|
WriteRegistersBulk(base, run, count);
|
|
break;
|
|
}
|
|
for (uint32_t m = 0; m < count; m++)
|
|
WriteRegister(one ? base : base + m, r.Read());
|
|
break;
|
|
}
|
|
case 1: // two registers
|
|
{
|
|
uint32_t a = r.Read(), b = r.Read();
|
|
WriteRegister(packet & 0x7FF, a);
|
|
WriteRegister((packet >> 11) & 0x7FF, b);
|
|
break;
|
|
}
|
|
case 2: // filler
|
|
break;
|
|
case 3:
|
|
ExecuteType3(r, packet, ctx);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void Worker()
|
|
{
|
|
#ifdef __APPLE__
|
|
// The idle poll sleeps 100 us at a time; at the default QoS macOS
|
|
// coalesces those timers into ~10 ms wake-ups, leaving the GPU
|
|
// starved with recorded work pending.
|
|
pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0);
|
|
#endif
|
|
writewatch::t_gpuThread = true;
|
|
renderer::t_commandProcessor = true;
|
|
SetHostThreadName("nfsmw-cp");
|
|
// Dense scenes are limited by this thread. On the Steam Machine the
|
|
// scheduler left it on a 3.5 GHz core, or on the sibling of a
|
|
// spinning game thread, while those held the 4.8 GHz cores.
|
|
hostcpu::UseReservedCore("command processor");
|
|
GuestThreadContext thread(2);
|
|
PPCContext& ctx = thread.ppcContext;
|
|
renderer::Initialize(s_regs);
|
|
auto lastWork = std::chrono::steady_clock::now();
|
|
while (true)
|
|
{
|
|
uint32_t primary = s_primaryBuffer;
|
|
uint32_t writeIndex = *MmioRegister(REG_CP_RB_WPTR);
|
|
if (primary == 0 || writeIndex == s_readIndex || writeIndex * 4 >= s_primarySize)
|
|
{
|
|
// Kicks come in bursts and the game often waits on their
|
|
// results: poll with yields for 2 ms after the last work,
|
|
// then back off to short sleeps.
|
|
// No new commands: whatever is recorded goes out now (the
|
|
// game may be waiting on a fence in it).
|
|
if (writewatch::g_submitRequested.exchange(false))
|
|
renderer::Submit(); // a guest thread waits on recorded work
|
|
renderer::SubmitIfDue();
|
|
// The game is suspended: its threads stand still, so
|
|
// wait for the resume rather than poll all the while.
|
|
if (guesttime::Suspended())
|
|
{
|
|
guesttime::WaitWhileSuspended();
|
|
lastWork = std::chrono::steady_clock::now();
|
|
continue;
|
|
}
|
|
if (std::chrono::steady_clock::now() - lastWork < std::chrono::milliseconds(2))
|
|
std::this_thread::yield();
|
|
else
|
|
std::this_thread::sleep_for(std::chrono::microseconds(100));
|
|
continue;
|
|
}
|
|
s_regs[REG_CP_RB_WPTR] = writeIndex;
|
|
Reader r{ Physical(primary), s_primarySize, s_readIndex * 4, writeIndex * 4 };
|
|
timeline::Mark(timeline::BatchBegin, (writeIndex - s_readIndex) & (s_primarySize / 4 - 1));
|
|
s_busyFrom = std::chrono::steady_clock::now();
|
|
s_busyWaitFrom = renderer::g_cpWaitNs;
|
|
ExecuteBuffer(r, ctx);
|
|
AccountBusy();
|
|
timeline::Mark(timeline::BatchEnd);
|
|
if (writewatch::g_submitRequested.exchange(false))
|
|
renderer::Submit();
|
|
renderer::SubmitIfIdle();
|
|
s_readIndex = writeIndex;
|
|
// The read pointer is fetch progress (as on the hardware), not
|
|
// completion: everything the batch needs has been consumed or
|
|
// copied, so its ring space is free now. D3D tracks resource
|
|
// lifetimes with fences, which stay deferred to GPU completion.
|
|
if (uint32_t wb = s_readWriteback)
|
|
*reinterpret_cast<be<uint32_t>*>(Physical(wb)) = writeIndex;
|
|
lastWork = std::chrono::steady_clock::now();
|
|
}
|
|
}
|
|
|
|
// One line for the vblank lock's state (on changes, and each minute),
|
|
// and how late the last minute's vblanks fired.
|
|
void LogVblank(const video::VblankLock::Status& s, const video::VblankTiming& timing, const char* timer)
|
|
{
|
|
const video::RefreshEstimate& d = s.display;
|
|
std::string line;
|
|
if (s.locked)
|
|
line = std::format("locked to the display: {:.4f} Hz (fit {} presents over {:.1f} s, rms {:.2f} ms, {} set aside); "
|
|
"vblank {:.2f} ms before its refresh, error {:+.3f} ms; vblank to screen {:.1f} ms (median), to present {:.1f} ms (99%); {} slips",
|
|
1e9 / d.periodNs, d.samples, d.spanNs / 1e9, d.rmsNs / 1e6, d.rejected, s.leadNs / 1e6, s.errorNs / 1e6,
|
|
s.latencyNs / 1e6, s.readyNs / 1e6, s.slips);
|
|
else if (s.follows)
|
|
line = std::format("free-running every {:.3f} ms: the display follows the vblank (variable refresh?), "
|
|
"so there is no refresh to lock to", s.periodNs / 1e6);
|
|
else if (d.samples)
|
|
line = std::format("free-running every {:.3f} ms; display {}: {:.4f} Hz (fit {} presents over {:.1f} s, rms {:.2f} ms, {} set aside)",
|
|
s.periodNs / 1e6, d.valid ? "measured (lock off)" : "not accepted", 1e9 / d.periodNs, d.samples, d.spanNs / 1e9,
|
|
d.rmsNs / 1e6, d.rejected);
|
|
else
|
|
line = std::format("free-running every {:.3f} ms", s.periodNs / 1e6);
|
|
if (video::VblankTiming::Summary t = timing.Summarize(); t.count)
|
|
line += std::format("; timer {}: vblanks late {:.2f} ms (median), {:.2f} ms (99%), {:.2f} ms at most, handler {:.2f} ms (99%)",
|
|
timer, t.lateMedianNs / 1e6, t.late99Ns / 1e6, t.lateMaxNs / 1e6, t.handler99Ns / 1e6);
|
|
fprintf(stderr, "[vblank] %s\n", line.c_str());
|
|
}
|
|
|
|
void Vsync()
|
|
{
|
|
SetHostThreadName("nfsmw-vsync"); // (Linux would give it its creator's name, nfsmw-main)
|
|
// Woken on time (video/vblank_timer.h): on macOS an ordinary
|
|
// thread's sleeps ran up to 9 ms late.
|
|
const video::VblankTimer timer = video::ChosenVblankTimer();
|
|
const bool realtime = video::PrepareVblankThread(timer, 16'666'667);
|
|
const char* timerName = video::VblankTimerName(timer, realtime);
|
|
video::VblankTiming timing;
|
|
GuestThreadContext thread(2);
|
|
using namespace std::chrono;
|
|
video::VblankLock& clock = video::GuestVblank();
|
|
auto now = [] { return int64_t(duration_cast<nanoseconds>(steady_clock::now().time_since_epoch()).count()); };
|
|
// The time suspended that `next` has been moved on for. Kept
|
|
// across vblanks and changed only with a shift, so a suspension
|
|
// that begins and ends while this thread is elsewhere (in the
|
|
// guest's interrupt handler, say) still moves the next vblank.
|
|
// Both taken while running (a boot thread can start this one in
|
|
// a suspension): `next` is a host time at which `paused` held.
|
|
int64_t next, paused;
|
|
do
|
|
{
|
|
guesttime::WaitWhileSuspended();
|
|
paused = guesttime::PausedNs();
|
|
next = now();
|
|
} while (guesttime::Suspended() || guesttime::PausedNs() != paused);
|
|
bool locked = false;
|
|
uint64_t slips = 0, count = 0;
|
|
while (true)
|
|
{
|
|
next = clock.Next(next);
|
|
for (int k = video::GuestVblankDivider(); k > 1; k--) // Frame Rate 30: every other refresh
|
|
next = clock.Next(next);
|
|
// To the vblank in guest time: a suspension before it moves
|
|
// it on by exactly the time suspended, so the guest sees one
|
|
// ordinary period across it, and no vblank fires, queues or
|
|
// is replayed meanwhile.
|
|
while (true)
|
|
{
|
|
std::this_thread::sleep_until(steady_clock::time_point(nanoseconds(next)));
|
|
guesttime::WaitWhileSuspended();
|
|
int64_t nowPaused = guesttime::PausedNs();
|
|
if (nowPaused == paused)
|
|
break;
|
|
next += nowPaused - paused;
|
|
paused = nowPaused;
|
|
clock.Resumed();
|
|
}
|
|
const int64_t fired = now();
|
|
video::NoteGuestVblank(fired);
|
|
s_counter++;
|
|
s_vblanks++;
|
|
DispatchInterrupt(thread.ppcContext, 0, 2);
|
|
timing.Add(fired - next, now() - fired);
|
|
auto status = clock.GetStatus();
|
|
if (status.locked != locked || status.slips != slips || ++count % 3600 == 0)
|
|
{
|
|
LogVblank(status, timing, timerName);
|
|
locked = status.locked;
|
|
slips = status.slips;
|
|
}
|
|
}
|
|
}
|
|
|
|
void StartOnce()
|
|
{
|
|
if (s_started.exchange(true))
|
|
return;
|
|
// Registers Xenia returns fixed values for (GraphicsSystem::ReadRegister).
|
|
WriteRegister(0x0F00, 0x08100748); // RB_EDRAM_TIMING
|
|
WriteRegister(0x0F01, 0x0000200E); // RB_BC_CONTROL
|
|
WriteRegister(0x194C, 0x000002D0); // D1MODE_V_COUNTER
|
|
WriteRegister(0x1951, 0x00000001); // interrupt status: vblank
|
|
WriteRegister(0x1961, 0x050002D0); // D1MODE_VIEWPORT_SIZE
|
|
std::thread(Worker).detach();
|
|
std::thread(Vsync).detach();
|
|
}
|
|
}
|
|
|
|
void InitializeRingBuffer(uint32_t guestAddress, uint32_t sizeLog2)
|
|
{
|
|
s_primarySize = 1u << (sizeLog2 + 3);
|
|
s_readIndex = 0;
|
|
s_primaryBuffer = guestAddress & 0x1FFFFFFF;
|
|
fprintf(stderr, "[gpu] ring buffer %08X, %u bytes\n", guestAddress, s_primarySize);
|
|
StartOnce();
|
|
}
|
|
|
|
void EnableReadPointerWriteBack(uint32_t guestAddress, uint32_t blockSizeLog2)
|
|
{
|
|
s_readWriteback = guestAddress & 0x1FFFFFFF;
|
|
}
|
|
|
|
void SetInterruptCallback(uint32_t callback, uint32_t userData)
|
|
{
|
|
s_interruptData = userData;
|
|
s_interruptCallback = callback;
|
|
fprintf(stderr, "[gpu] interrupt callback %08X (data %08X)\n", callback, userData);
|
|
}
|
|
|
|
uint64_t FrameCount()
|
|
{
|
|
return s_frames;
|
|
}
|
|
|
|
int64_t WaitingSince()
|
|
{
|
|
return s_waitSince.load(std::memory_order_acquire);
|
|
}
|
|
|
|
std::string DescribeState()
|
|
{
|
|
std::string text;
|
|
if (int64_t since = s_waitSince.load(std::memory_order_acquire))
|
|
{
|
|
uint32_t info = s_waitInfo.load(std::memory_order_relaxed), poll = s_waitPoll.load(std::memory_order_relaxed);
|
|
bool memory = (info & 0x10) != 0;
|
|
uint32_t value = memory ? LoadSwapped(poll) : s_regs[poll % REGISTER_COUNT];
|
|
static const char* const kFunc[8] = { "never", "<", "<=", "==", "!=", ">=", ">", "always" };
|
|
double seconds = double(std::chrono::duration_cast<std::chrono::nanoseconds>(
|
|
std::chrono::steady_clock::now().time_since_epoch()).count() - since) / 1e9;
|
|
text += std::format("In a WAIT_REG_MEM for {:.1f} s: {} {:08X}: value {:08X} & mask {:08X} {} ref {:08X}{}\n",
|
|
seconds, memory ? "memory" : "register", poll, value, s_waitMask.load(), kFunc[info & 7], s_waitRef.load(),
|
|
report::g_testHang.load() ? " (NFSMW_TEST_HANG)" : "");
|
|
}
|
|
else
|
|
text += "Not in a WAIT_REG_MEM.\n";
|
|
text += std::format("Frames {}, draws {}, guest vblanks {}; ring buffer {:08X} ({} bytes), read index {} of write index {}\n",
|
|
s_frames.load(), s_draws.load(), s_vblanks.load(), s_primaryBuffer.load(), s_primarySize, s_readIndex,
|
|
uint32_t(*MmioRegister(REG_CP_RB_WPTR)));
|
|
text += FormatHistory();
|
|
return text;
|
|
}
|
|
}
|