mirror of
https://github.com/SpeedBreakerProject/speedbreaker.git
synced 2026-10-17 21:01:26 +02:00
SpeedBreaker ProjectandClaude Opus 5.5
a6ade41b9f
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1338 lines
64 KiB
C++
1338 lines
64 KiB
C++
// SpeedBreaker runtime. GPL-3.0-or-later (see COPYING). See write_watch.h.
|
|
#include <stdafx.h>
|
|
#include "write_watch.h"
|
|
#include "memory.h"
|
|
|
|
#include <cpu/guest_time.h>
|
|
#include <sys/mman.h>
|
|
#include <unistd.h>
|
|
#include <time.h>
|
|
#include <dlfcn.h>
|
|
|
|
namespace writewatch
|
|
{
|
|
std::atomic<bool> g_submitRequested{ false };
|
|
thread_local bool t_gpuThread = false;
|
|
std::atomic<uint32_t> g_snapsOpen{ 0 };
|
|
|
|
namespace
|
|
{
|
|
constexpr uint32_t kPhysicalSize = 0x20000000;
|
|
constexpr uint32_t kWindows[3] = { 0xA0000000, 0xC0000000, 0xE0000000 };
|
|
// GPU hazards are tracked per guest page (4 KB): host pages are 16 KB
|
|
// on Apple silicon, and neighbouring guest resources (the two 64x64
|
|
// luminance targets NFSMW reads back, a vertex buffer next to the
|
|
// command ring) would otherwise wait on each other's GPU work.
|
|
constexpr uint32_t kSubShift = 12;
|
|
constexpr uint32_t kSubCount = kPhysicalSize >> kSubShift;
|
|
|
|
size_t s_pageSize = 0;
|
|
uint32_t s_pageShift = 0;
|
|
uint32_t s_pageCount = 0;
|
|
uint32_t s_subsPerPage = 1;
|
|
std::atomic<uint64_t> s_sequence{ 1 };
|
|
// Texture watches (per host page), and the write sequence of each
|
|
// guest page (4 KB). A CPU write is only seen per host page (the
|
|
// whole page is unprotected at once), so it marks every guest page
|
|
// of it (unless NFSMW_WATCH_SUBPAGES, see s_watchSubpages); a resolve
|
|
// marks just the guest pages it wrote. On Apple's
|
|
// 16 KB pages a sequence per host page made each resolve dirty its
|
|
// neighbours (the two shadow maps, the bloom chain share pages):
|
|
// ~17 textures a frame untiled again, ~40 MB, where a 4 KB-page
|
|
// Steam Machine reloads 3 in the same race.
|
|
std::unique_ptr<std::atomic<uint8_t>[]> s_watched;
|
|
std::unique_ptr<std::atomic<uint64_t>[]> s_written;
|
|
// GPU guard: the last submissions reading and writing each guest
|
|
// page. A pending GPU write makes the host page PROT_NONE (CPU reads
|
|
// would be stale); a pending read, PROT_READ. s_prot is what is set.
|
|
std::unique_ptr<std::atomic<uint64_t>[]> s_gpuRead, s_gpuWrite;
|
|
std::unique_ptr<std::atomic<uint8_t>[]> s_prot; // 0 RW, 1 read-only, 2 none
|
|
std::atomic<uint64_t> s_completed{ 0 };
|
|
// Host pages opened for an access to a guest page without hazards
|
|
// while another of its guest pages still has one: protected again
|
|
// at the next Rearm (submission boundary).
|
|
std::vector<uint32_t> s_rearm;
|
|
// NFSMW_LOG_GUARD=1: what each guest page was guarded for, logged on stalls.
|
|
const bool s_logGuard = std::getenv("NFSMW_LOG_GUARD") != nullptr;
|
|
std::unique_ptr<std::atomic<const char*>[]> s_gpuTag;
|
|
std::unique_ptr<std::atomic<uint32_t>[]> s_gpuRange[2];
|
|
std::atomic<uint64_t> s_stalls{ 0 }, s_stallUs{ 0 }, s_faults{ 0 }, s_opened{ 0 };
|
|
// Shadow mode (a renderer that can't import guest memory, e.g. on a
|
|
// discrete GPU): the GPU reads and writes its own copy of physical
|
|
// memory. Per host page: the write sequence the copy was made at
|
|
// (0 = never), and whether the GPU wrote it last (resolves), which
|
|
// makes the guest's copy stale until a CPU access reads it back.
|
|
uint8_t* s_shadow = nullptr;
|
|
std::unique_ptr<std::atomic<uint64_t>[]> s_shadowSeq;
|
|
std::unique_ptr<std::atomic<uint8_t>[]> s_gpuOwned;
|
|
// Highest write sequence per 256 KB block: clean ranges are skipped
|
|
// without looking at their pages.
|
|
constexpr uint32_t kBlockShift = 18;
|
|
std::unique_ptr<std::atomic<uint64_t>[]> s_blockWritten;
|
|
// Per 256 KB block, for the command processor's per-draw checks
|
|
// (GpuWritePending, AnyGpuOwned): the highest submission that writes
|
|
// any of its guest pages (never lowered), and how many of its pages
|
|
// are GPU-owned. A block without either is clean at a glance; the
|
|
// page arrays are 1 MB each, and a look there is mostly a cache miss.
|
|
std::unique_ptr<std::atomic<uint64_t>[]> s_blockGpuWrite;
|
|
std::unique_ptr<std::atomic<uint32_t>[]> s_blockOwned;
|
|
// NFSMW_WATCH_RANGES=0: the bookkeeping as before (see Put, Change,
|
|
// MarkWritten, WrittenSince), for A/B.
|
|
const bool s_ranges = [] { const char* v = std::getenv("NFSMW_WATCH_RANGES"); return !v || v[0] != '0'; }();
|
|
|
|
// Per-page state that only changes under s_protectMutex, or only on
|
|
// the command processor, is stored plainly. A sequentially consistent
|
|
// store or exchange is a locked instruction on x86 (about 20 cycles):
|
|
// a resolve made eight of them per page (write sequence, block, epoch,
|
|
// guard, protection twice, ownership).
|
|
template <class T>
|
|
void Put(std::atomic<T>& a, T value)
|
|
{
|
|
if (s_ranges)
|
|
a.store(value, std::memory_order_relaxed);
|
|
else
|
|
a.store(value);
|
|
}
|
|
// Put, returning whether the value changed.
|
|
template <class T>
|
|
bool Change(std::atomic<T>& a, T value)
|
|
{
|
|
if (!s_ranges)
|
|
return a.exchange(value) != value;
|
|
if (a.load(std::memory_order_relaxed) == value)
|
|
return false;
|
|
a.store(value, std::memory_order_relaxed);
|
|
return true;
|
|
}
|
|
// NFSMW_LOG_UPLOADS: per guest page, what last bumped its write
|
|
// sequence (kind << 60 | frame << 32 | tag), only allocated then.
|
|
const bool s_logWriters = std::getenv("NFSMW_LOG_UPLOADS") != nullptr;
|
|
std::unique_ptr<std::atomic<uint64_t>[]> s_lastWriter;
|
|
std::unique_ptr<std::atomic<uint32_t>[]> s_lastWriterPc; // a fault's t_faultPc - s_imageBase
|
|
std::atomic<uint64_t> s_writerFrame{ 0 };
|
|
const uint8_t* s_imageBase = nullptr;
|
|
void NoteWriter(uint32_t firstSub, uint32_t count, Writer kind, uint32_t tag)
|
|
{
|
|
uint64_t v = uint64_t(kind) << 60 | (s_writerFrame.load(std::memory_order_relaxed) & 0x0FFFFFFF) << 32 | tag;
|
|
auto pc = static_cast<const uint8_t*>(t_faultPc);
|
|
uint32_t offset = kind == Writer::kCpuFault && pc > s_imageBase && pc - s_imageBase < 0xFFFFFFFF ? uint32_t(pc - s_imageBase) : 0;
|
|
for (uint32_t i = 0; i < count; i++)
|
|
{
|
|
s_lastWriter[firstSub + i].store(v, std::memory_order_relaxed);
|
|
s_lastWriterPc[firstSub + i].store(offset, std::memory_order_relaxed);
|
|
}
|
|
}
|
|
|
|
// NFSMW_WATCH_SUBPAGES=1, on hosts whose pages hold several guest
|
|
// pages (16 KB on Apple; off elsewhere): a CPU write fault marks just
|
|
// the guest page it hit. The game rewrites a per-bind record every
|
|
// frame in the 4 KB page next to many textures (and in the one before
|
|
// HUD ones): marking the whole host page reloaded ~10 of them a frame
|
|
// on Apple, each ending a render pass, where 4 KB-page hosts reload
|
|
// none. The fault copies the host page first (still read-only, so
|
|
// the copy is what the watch last vouched for) and opens it; its
|
|
// other guest pages are pending until anything relies on the page
|
|
// again: protecting it (the hook in SetProt) or reading sequences of
|
|
// it (Settle). Either makes it read-only, then marks the pending
|
|
// pages whose bytes changed (a store after that faults); if all of
|
|
// them changed, the watch ends there, as at a whole-page fault. A
|
|
// snapshot open for kSnapAge submissions (~8 frames: no texture there
|
|
// is in use), on a page a host write opens, or the oldest when a
|
|
// fault finds every slot taken, marks its pending pages unseen and
|
|
// leaves the page open, as a fault always did. A fault waits for a
|
|
// texture's untile still reading another guest page of it (see
|
|
// OpenSnapshot); a guest page another GPU read is pending on is
|
|
// marked at the fault, not compared (as is the untile's, if that
|
|
// wait gives up: see WaitForGpu). Only watched pages get one: a
|
|
// page protected just for the GPU's use (vertex data) has no texture
|
|
// to keep (every path that ends a watch marks all its guest pages).
|
|
// Unsure cases mark the whole host page as before. In use with host
|
|
// pages of 2 to 32 guest pages: by default on Apple (iPad race:
|
|
// 13.9 -> 0 texture reloads a frame with the renderer's fusion
|
|
// toggles), only when set elsewhere; =0 turns it off (null arrays).
|
|
#ifdef __APPLE__
|
|
constexpr bool kSubpagesByDefault = true;
|
|
#else
|
|
constexpr bool kSubpagesByDefault = false;
|
|
#endif
|
|
const bool s_watchSubpages = [] { const char* v = std::getenv("NFSMW_WATCH_SUBPAGES"); return v ? v[0] == '1' : kSubpagesByDefault; }();
|
|
constexpr uint32_t kSnapSlots = 128, kSnapAge = 32;
|
|
static_assert(kSnapSlots < 256, "s_snapOf holds a slot + 1 in a byte");
|
|
// Under s_protectMutex: the slots (pending: the guest pages not
|
|
// marked yet, a bit each; opened: s_rearms then; tag: the fault;
|
|
// serial: s_snapSerial then, the oldest goes first when all are taken).
|
|
struct Snap { uint32_t page = UINT32_MAX, pending = 0, tag = 0; uint64_t opened = 0, serial = 0; };
|
|
Snap s_snap[kSnapSlots];
|
|
uint32_t s_snapFree[kSnapSlots], s_snapFreeCount = 0;
|
|
uint64_t s_rearms = 0, s_snapSerial = 0;
|
|
uint32_t s_allSubs = 0; // a pending mask with every guest page of a host page
|
|
std::unique_ptr<uint8_t[]> s_snapData; // the copies, a host page per slot
|
|
// Per host page, its slot + 1 (0: none; the array is null when off);
|
|
// how many are open per 256 KB block (SettleRange's quick check; in
|
|
// all: g_snapsOpen, the sequence readers' inline one).
|
|
std::unique_ptr<std::atomic<uint8_t>[]> s_snapOf;
|
|
std::unique_ptr<std::atomic<uint16_t>[]> s_blockSnaps;
|
|
// Per guest page, the last submission whose texture untile reads it
|
|
// (GpuRead's `untile`; looked at while its s_gpuRead is pending).
|
|
std::unique_ptr<std::atomic<uint64_t>[]> s_gpuUntile;
|
|
std::atomic<uint64_t> s_snapOpened{ 0 }, s_snapClean{ 0 }, s_snapChanged{ 0 }, s_snapEvicted{ 0 }, s_snapFull{ 0 }, s_snapWhole{ 0 },
|
|
s_snapReading{ 0 }, s_snapWaits{ 0 }, s_snapGaveUp{ 0 };
|
|
|
|
std::atomic<uint64_t> s_uploadBytes{ 0 }, s_readbackBytes{ 0 }, s_protects{ 0 }, s_protectNs{ 0 }, s_hostWriteBytes{ 0 },
|
|
s_firstUploadBytes{ 0 }, s_uploadNs{ 0 };
|
|
std::mutex s_protectMutex; // protection changes; the fault path takes it after any wait
|
|
|
|
// Host address of physical page `page` in window `w`, or nullptr when
|
|
// the page isn't visible there (the shifted 0xE window on 4 KB hosts).
|
|
uint8_t* PageHost(uint32_t w, uint32_t page)
|
|
{
|
|
uint64_t physical = uint64_t(page) << s_pageShift;
|
|
uint32_t shift = w == 2 ? g_memory.eWindowShift : 0;
|
|
if (physical < shift)
|
|
return nullptr;
|
|
return g_memory.base + kWindows[w] + (physical - shift);
|
|
}
|
|
|
|
// Pages [first, first + count) in every window that shows them (a
|
|
// window shows a suffix of physical memory: the 0xE one is shifted).
|
|
void ProtectPages(uint32_t first, uint32_t count, int prot)
|
|
{
|
|
auto start = std::chrono::steady_clock::now();
|
|
for (uint32_t w = 0; w < 3; w++)
|
|
{
|
|
uint32_t page = first;
|
|
while (page < first + count && !PageHost(w, page))
|
|
page++;
|
|
if (page == first + count)
|
|
continue;
|
|
mprotect(PageHost(w, page), size_t(first + count - page) << s_pageShift, prot);
|
|
s_protects.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
s_protectNs.fetch_add(uint64_t(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - start).count()),
|
|
std::memory_order_relaxed);
|
|
}
|
|
|
|
// Protection changes to consecutive pages, made with one mprotect per
|
|
// window for each run that gets the same protection. Page by page,
|
|
// the career intro's frame where 9.4 MB of new textures come in made
|
|
// 7,275 calls (a write-protect flushes the TLB of every core running
|
|
// the game): 11 ms of a 36 ms frame. In runs: 747 calls, 2.4 ms, and
|
|
// the frame takes 25 ms (Steam Machine). NFSMW_PROTECT_RUNS=0: one
|
|
// call per page. Flushed when it goes out of scope: declare it after
|
|
// the s_protectMutex lock.
|
|
const bool s_protectRuns = [] { const char* v = std::getenv("NFSMW_PROTECT_RUNS"); return !v || v[0] != '0'; }();
|
|
void CloseSnapshot(uint32_t page, bool compare);
|
|
struct ProtectRun
|
|
{
|
|
uint32_t first = 0, count = 0;
|
|
int prot = 0;
|
|
// NFSMW_WATCH_SUBPAGES: pages added read-only with a snapshot
|
|
// open, compared once the run has protected them (SetProt).
|
|
uint32_t compares[32], compareCount = 0;
|
|
void Add(uint32_t page, int p)
|
|
{
|
|
if (count && (page != first + count || p != prot || !s_protectRuns))
|
|
Flush();
|
|
if (count == 0)
|
|
{
|
|
first = page;
|
|
prot = p;
|
|
}
|
|
count++;
|
|
}
|
|
// After Add(page).
|
|
void Compare(uint32_t page)
|
|
{
|
|
if (compareCount == std::size(compares))
|
|
Flush(); // protects `page` too: it is compared at the next flush
|
|
compares[compareCount++] = page;
|
|
}
|
|
void Flush()
|
|
{
|
|
if (count)
|
|
ProtectPages(first, count, prot);
|
|
count = 0;
|
|
for (uint32_t i = 0; i < compareCount; i++)
|
|
CloseSnapshot(compares[i], true);
|
|
compareCount = 0;
|
|
}
|
|
~ProtectRun() { Flush(); }
|
|
};
|
|
|
|
// Physical address of a host address inside one of the windows, or -1.
|
|
int64_t PhysicalOf(const void* host)
|
|
{
|
|
auto* p = static_cast<const uint8_t*>(host);
|
|
for (uint32_t w = 0; w < 3; w++)
|
|
{
|
|
const uint8_t* start = g_memory.base + kWindows[w];
|
|
if (p >= start && p < start + kPhysicalSize)
|
|
{
|
|
uint64_t physical = uint64_t(p - start) + (w == 2 ? g_memory.eWindowShift : 0);
|
|
return physical < kPhysicalSize ? int64_t(physical) : -1;
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
int64_t PageOf(const void* host)
|
|
{
|
|
int64_t physical = PhysicalOf(host);
|
|
return physical < 0 ? -1 : physical >> s_pageShift;
|
|
}
|
|
|
|
// A CPU write to `page` (textures in it reload; its shadow copy is stale).
|
|
// Bumped after each BumpWritten: unchanged means no page was written
|
|
// (with NFSMW_WATCH_SUBPAGES, none a reader compared: see WriteEpoch).
|
|
std::atomic<uint64_t> s_writeEpoch{ 1 };
|
|
|
|
// A CPU write to host page `page`: every guest page of it.
|
|
void BumpWritten(uint32_t page, Writer kind, uint32_t tag)
|
|
{
|
|
uint64_t seq = s_sequence.fetch_add(1) + 1;
|
|
// The block first: a block's sequence is never below one of its
|
|
// pages', even while a write is being recorded (WrittenSince).
|
|
auto& block = s_blockWritten[(uint64_t(page) << s_pageShift) >> kBlockShift];
|
|
uint64_t old = block.load();
|
|
while (old < seq && !block.compare_exchange_weak(old, seq)) {}
|
|
for (uint32_t i = 0, sub = page * s_subsPerPage; i < s_subsPerPage; i++, sub++)
|
|
s_written[sub].store(seq);
|
|
if (s_logWriters)
|
|
NoteWriter(page * s_subsPerPage, s_subsPerPage, kind, tag);
|
|
s_writeEpoch.fetch_add(1, std::memory_order_release);
|
|
}
|
|
|
|
// Guest pages [firstSub, lastSub] at once, with one sequence (a
|
|
// resolve: up to ~3,600 guest pages, ~25 a frame). Readers compare
|
|
// sequences with a Current() taken earlier, which every page's new
|
|
// one exceeds. Only the command processor reads them, so plain
|
|
// stores do; the epoch, bumped last, publishes them to any other
|
|
// reader.
|
|
uint64_t BumpWrittenRange(uint32_t firstSub, uint32_t lastSub, Writer kind, uint32_t tag)
|
|
{
|
|
uint64_t seq = s_sequence.fetch_add(1) + 1;
|
|
for (uint64_t b = (uint64_t(firstSub) << kSubShift) >> kBlockShift; b <= (uint64_t(lastSub) << kSubShift) >> kBlockShift; b++)
|
|
{
|
|
auto& block = s_blockWritten[b]; // first, as in BumpWritten
|
|
uint64_t old = block.load();
|
|
while (old < seq && !block.compare_exchange_weak(old, seq)) {}
|
|
}
|
|
std::atomic<uint64_t>* written = s_written.get(); // not reloaded after every store
|
|
for (uint32_t sub = firstSub; sub <= lastSub; sub++)
|
|
written[sub].store(seq, std::memory_order_relaxed);
|
|
if (s_logWriters)
|
|
NoteWriter(firstSub, lastSub - firstSub + 1, kind, tag);
|
|
s_writeEpoch.fetch_add(1, std::memory_order_release);
|
|
return seq;
|
|
}
|
|
|
|
// NFSMW_WATCH_SUBPAGES: guest pages `subs` (a bit each) of host page
|
|
// `page`, with one sequence and one epoch bump, as BumpWrittenRange.
|
|
// A host page (at most 128 KB, aligned) lies in one 256 KB block.
|
|
// kCompared notes each guest page's own address as its tag.
|
|
void BumpSubs(uint32_t page, uint32_t subs, Writer kind, uint32_t tag)
|
|
{
|
|
uint64_t seq = s_sequence.fetch_add(1) + 1;
|
|
auto& block = s_blockWritten[(uint64_t(page) << s_pageShift) >> kBlockShift]; // first, as in BumpWritten
|
|
uint64_t old = block.load();
|
|
while (old < seq && !block.compare_exchange_weak(old, seq)) {}
|
|
for (; subs; subs &= subs - 1)
|
|
{
|
|
uint32_t sub = page * s_subsPerPage + uint32_t(__builtin_ctz(subs));
|
|
s_written[sub].store(seq, std::memory_order_relaxed);
|
|
if (s_logWriters)
|
|
NoteWriter(sub, 1, kind, kind == Writer::kCompared ? sub << kSubShift : tag);
|
|
}
|
|
s_writeEpoch.fetch_add(1, std::memory_order_release);
|
|
}
|
|
|
|
// The newest write to any guest page of host page `page`.
|
|
uint64_t PageWritten(uint32_t page)
|
|
{
|
|
uint64_t highest = 0;
|
|
for (uint32_t i = 0, sub = page * s_subsPerPage; i < s_subsPerPage; i++, sub++)
|
|
highest = std::max(highest, s_written[sub].load());
|
|
return highest;
|
|
}
|
|
|
|
uint8_t* GuestPage(uint32_t page)
|
|
{
|
|
return g_memory.base + kWindows[0] + (uint64_t(page) << s_pageShift);
|
|
}
|
|
|
|
// Under s_protectMutex: close host page `page`'s snapshot (see
|
|
// s_watchSubpages). Compared: the page is read-only first (made so
|
|
// here, or by the caller's run), so a store either landed before and
|
|
// is compared, or faults after and waits for the lock; the pending
|
|
// guest pages whose bytes changed are marked. The page stays watched
|
|
// while one compared clean (its next store must fault); with all of
|
|
// them changed, all are marked and nothing is left to vouch for, so
|
|
// the watch ends as at a whole-page fault (a Watch closing it has set
|
|
// it again already: never cleared here). Not compared: every pending
|
|
// guest page is marked and the page is left as it is (open, as a
|
|
// fault left it before). An open snapshot's page is RW (prot 0): every
|
|
// change to its protection closes it (SetProt: at once, or once the
|
|
// run that makes it read-only is flushed).
|
|
void CloseSnapshot(uint32_t page, bool compare)
|
|
{
|
|
uint32_t slot = s_snapOf[page].load(std::memory_order_relaxed) - 1u;
|
|
Snap& snap = s_snap[slot];
|
|
if (compare && s_prot[page].load(std::memory_order_relaxed) == 0)
|
|
{
|
|
Put(s_prot[page], uint8_t(1));
|
|
ProtectPages(page, 1, PROT_READ);
|
|
}
|
|
const uint8_t* copy = s_snapData.get() + (size_t(slot) << s_pageShift);
|
|
const uint8_t* now = GuestPage(page);
|
|
uint32_t changed = snap.pending;
|
|
if (compare)
|
|
for (uint32_t pending = snap.pending; pending; pending &= pending - 1)
|
|
{
|
|
uint32_t i = uint32_t(__builtin_ctz(pending));
|
|
size_t offset = size_t(i) << kSubShift;
|
|
if (memcmp(now + offset, copy + offset, size_t(1) << kSubShift) == 0)
|
|
changed &= ~(1u << i);
|
|
}
|
|
if (changed)
|
|
BumpSubs(page, changed, compare ? Writer::kCompared : Writer::kUnverified, snap.tag);
|
|
if (compare && changed != snap.pending)
|
|
s_watched[page].store(1);
|
|
// Release: a reader that sees the page closed sees its marks.
|
|
s_snapOf[page].store(0, std::memory_order_release);
|
|
s_blockSnaps[(uint64_t(page) << s_pageShift) >> kBlockShift].fetch_sub(1, std::memory_order_relaxed);
|
|
g_snapsOpen.fetch_sub(1, std::memory_order_relaxed);
|
|
snap.page = UINT32_MAX;
|
|
s_snapFree[s_snapFreeCount++] = slot;
|
|
(!compare ? s_snapEvicted : changed ? s_snapChanged : s_snapClean).fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
|
|
// Under s_protectMutex. With `run`, the change is made when the run
|
|
// is flushed.
|
|
void SetProt(uint32_t page, uint8_t prot, ProtectRun* run = nullptr)
|
|
{
|
|
// Protected again (a watch, a GPU use, a rearm, a shadow sync, a
|
|
// settle): the page's snapshot is compared once it is read-only.
|
|
// Every path that protects a page comes through here. In a run,
|
|
// after the run's flush (one mprotect per window for the run, not
|
|
// one per page); one going to PROT_NONE is compared read-only
|
|
// first (its bytes can't be read after).
|
|
bool compare = prot != 0 && s_snapOf && s_snapOf[page].load(std::memory_order_relaxed);
|
|
if (compare && (!run || prot == 2))
|
|
{
|
|
if (run)
|
|
run->Flush(); // the run's earlier pages first: CloseSnapshot protects this one itself
|
|
// (Unless the flush compared it: the run made it read-only first.)
|
|
if (s_snapOf[page].load(std::memory_order_relaxed))
|
|
CloseSnapshot(page, true);
|
|
compare = false;
|
|
}
|
|
if (!Change(s_prot[page], prot))
|
|
return; // (not with `compare`: an open snapshot's page is prot 0)
|
|
int os = prot == 2 ? PROT_NONE : prot == 1 ? PROT_READ : PROT_READ | PROT_WRITE;
|
|
if (run)
|
|
{
|
|
run->Add(page, os);
|
|
if (compare)
|
|
run->Compare(page);
|
|
}
|
|
else
|
|
ProtectPages(page, 1, os);
|
|
}
|
|
|
|
// The protection the page's watch and guard state call for.
|
|
uint8_t WantedProt(uint32_t page)
|
|
{
|
|
// Shadow mode: GPU reads use the copy, so only GPU-written pages
|
|
// (stale here) and write tracking need protection.
|
|
if (s_shadow)
|
|
return s_gpuOwned[page].load() ? 2 : s_watched[page].load() ? 1 : 0;
|
|
uint64_t completed = s_completed.load();
|
|
uint8_t prot = s_watched[page].load() ? 1 : 0;
|
|
for (uint32_t i = 0, sub = page * s_subsPerPage; i < s_subsPerPage; i++, sub++)
|
|
{
|
|
if (s_gpuWrite[sub].load() > completed)
|
|
return 2;
|
|
if (s_gpuRead[sub].load() > completed)
|
|
prot = 1;
|
|
}
|
|
return prot;
|
|
}
|
|
|
|
// Shadow mode, under s_protectMutex: copy a GPU-written page back to
|
|
// the guest (its GPU writes must be complete). The copy is then
|
|
// current again, and further CPU writes are tracked.
|
|
void ReadBackLocked(uint32_t page)
|
|
{
|
|
if (!s_shadow || !s_gpuOwned[page].load())
|
|
return;
|
|
SetProt(page, 0);
|
|
memcpy(GuestPage(page), s_shadow + (uint64_t(page) << s_pageShift), s_pageSize);
|
|
s_readbackBytes += s_pageSize;
|
|
s_gpuOwned[page].store(0);
|
|
s_blockOwned[(uint64_t(page) << s_pageShift) >> kBlockShift].fetch_sub(1, std::memory_order_release); // after the copy
|
|
s_shadowSeq[page].store(s_sequence.load());
|
|
s_watched[page].store(1);
|
|
}
|
|
|
|
// Under s_protectMutex.
|
|
void Reprotect(uint32_t page, ProtectRun* run = nullptr)
|
|
{
|
|
SetProt(page, WantedProt(page), run);
|
|
}
|
|
|
|
// Under s_protectMutex, for a CPU store that faulted on guest page
|
|
// `sub` of host page `page` (`tag`: its address): mark just that
|
|
// guest page and keep a copy of the host page to compare its other
|
|
// guest pages with later (see s_watchSubpages). False: mark the whole
|
|
// host page instead, or, with `untile` set (a guest page), wait for
|
|
// that page's untile and fault again (HandleFault). `markUntiles`:
|
|
// such a wait gave up, so pending untiles are marked as other reads.
|
|
bool OpenSnapshot(uint32_t page, uint32_t sub, uint32_t tag, bool markUntiles, uint32_t& untile)
|
|
{
|
|
if (!s_snapOf || t_gpuThread)
|
|
return false;
|
|
uint32_t bit = 1u << (sub - page * s_subsPerPage);
|
|
if (uint32_t open = s_snapOf[page].load(std::memory_order_relaxed))
|
|
{
|
|
// Another thread's fault opened the page while this one
|
|
// waited for the lock: this guest page is marked now, the
|
|
// others stay pending.
|
|
s_snap[open - 1].pending &= ~bit;
|
|
BumpSubs(page, bit, Writer::kCpuFault, tag);
|
|
return true;
|
|
}
|
|
// Not watched (protected for a GPU read only): no texture relies
|
|
// on its guest pages, so marking them all costs nothing, and a
|
|
// copy here would be made under the lock for nothing.
|
|
if (!s_watched[page].load(std::memory_order_relaxed))
|
|
return false;
|
|
// The copy must hold what the watch last vouched for: the page
|
|
// still read-only (prot 0: another thread opened it whole), and
|
|
// no GPU write to it pending, which changes its bytes without a
|
|
// fault (a read fault can open such a page to read-only, see
|
|
// HandleFault). In shadow mode PROT_NONE is a GPU-owned page.
|
|
// A guest page with a GPU read pending can't be compared: stores
|
|
// to it stop waiting for the GPU once the page is open, so the GPU
|
|
// may read bytes the CPU changes and then restores, which a
|
|
// compare can't see. A texture's untile is waited for first (as a
|
|
// 4 KB host's store to the texture waits): marked, the texture
|
|
// would reload, its new untile be pending at the next store to
|
|
// its neighbour, and so on every frame. Only while the untile is
|
|
// its guest page's newest GPU read: with a later one pending
|
|
// there (vertex data in the texture's last page), the page would
|
|
// be marked after the wait anyway. Any other GPU read (no
|
|
// texture kept: vertex data) leaves its guest page marked now, as
|
|
// the whole host page would be. (Not in shadow mode: the GPU reads
|
|
// its copy. The renderer's NFSMW_EAGER_WRITES=0 records no GPU
|
|
// uses: there fences wait for the GPU, so the game doesn't rewrite
|
|
// what an untile still reads.)
|
|
uint8_t prot = s_prot[page].load(std::memory_order_relaxed);
|
|
uint64_t completed = s_completed.load();
|
|
bool gpuWriting = false;
|
|
uint32_t reading = 0, waitFor = UINT32_MAX;
|
|
uint64_t untileUse = completed;
|
|
for (uint32_t i = 0, s = page * s_subsPerPage; i < s_subsPerPage; i++, s++)
|
|
{
|
|
gpuWriting = gpuWriting || s_gpuWrite[s].load(std::memory_order_relaxed) > completed;
|
|
uint64_t read = s_shadow ? 0 : s_gpuRead[s].load(std::memory_order_relaxed);
|
|
if (read > completed)
|
|
{
|
|
reading |= 1u << i; // (never `sub`: HandleFault waited for its GPU uses)
|
|
uint64_t use = s_gpuUntile[s].load(std::memory_order_relaxed);
|
|
if (!markUntiles && use == read && use > untileUse)
|
|
{
|
|
untileUse = use;
|
|
waitFor = s; // the last to complete
|
|
}
|
|
}
|
|
}
|
|
if (prot == 0 || (prot == 2 && s_shadow) || gpuWriting)
|
|
{
|
|
s_snapWhole.fetch_add(1, std::memory_order_relaxed);
|
|
return false;
|
|
}
|
|
if (waitFor != UINT32_MAX)
|
|
{
|
|
untile = waitFor; // (only here: the whole-page cases above wait for nothing)
|
|
return false;
|
|
}
|
|
uint32_t pending = s_allSubs & ~bit & ~reading;
|
|
if (pending == 0)
|
|
{
|
|
s_snapWhole.fetch_add(1, std::memory_order_relaxed);
|
|
return false;
|
|
}
|
|
// Every slot taken: the oldest snapshot closes unseen (as when it
|
|
// ages out), which marks its pending guest pages just as marking
|
|
// this whole host page would mark this one's.
|
|
if (s_snapFreeCount == 0)
|
|
{
|
|
const Snap* oldest = &s_snap[0];
|
|
for (const Snap& snap : s_snap)
|
|
if (snap.serial < oldest->serial)
|
|
oldest = &snap;
|
|
CloseSnapshot(oldest->page, false);
|
|
s_snapFull.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
// PROT_NONE left by a GPU write that has completed (nothing lowers
|
|
// it before a CPU access): readable now, and stores still fault.
|
|
if (prot == 2)
|
|
SetProt(page, 1);
|
|
uint32_t slot = s_snapFree[--s_snapFreeCount];
|
|
memcpy(s_snapData.get() + (size_t(slot) << s_pageShift), GuestPage(page), s_pageSize);
|
|
s_snap[slot] = { page, pending, tag, s_rearms, ++s_snapSerial };
|
|
s_snapOf[page].store(uint8_t(slot + 1), std::memory_order_relaxed);
|
|
s_blockSnaps[(uint64_t(page) << s_pageShift) >> kBlockShift].fetch_add(1, std::memory_order_relaxed);
|
|
g_snapsOpen.fetch_add(1, std::memory_order_relaxed);
|
|
s_snapOpened.fetch_add(1, std::memory_order_relaxed);
|
|
s_snapReading.fetch_add(uint32_t(__builtin_popcount(reading)), std::memory_order_relaxed);
|
|
// Last: its release bump of the epoch publishes the snapshot, and
|
|
// every texture checked before it is checked again (Settle first).
|
|
BumpSubs(page, bit | reading, Writer::kCpuFault, tag);
|
|
return true;
|
|
}
|
|
|
|
// Before a write sequence of physical [start, end) is read (start <
|
|
// end), with a snapshot open somewhere (Settle checks inline): compare
|
|
// the snapshots open on its host pages, so that their pending guest
|
|
// pages' sequences are current. A snapshot opened after this is
|
|
// published by an epoch bump (OpenSnapshot): such a reader checks
|
|
// again. Every sequence reader settles first (Settle, Settled). Per
|
|
// 256 KB block, one lock, and the open pages made read-only in runs
|
|
// (one mprotect per window for a run of them), then compared. (Out of
|
|
// line: one copy for SettleOpen and every Settled reader.)
|
|
[[gnu::noinline]] void SettleRange(uint64_t start, uint64_t end)
|
|
{
|
|
for (uint64_t b = start >> kBlockShift; b <= (end - 1) >> kBlockShift; b++)
|
|
{
|
|
if (s_blockSnaps[b].load(std::memory_order_acquire) == 0)
|
|
continue;
|
|
uint64_t from = std::max(start, b << kBlockShift), to = std::min(end, (b + 1) << kBlockShift);
|
|
uint32_t page = uint32_t(from >> s_pageShift), last = uint32_t((to - 1) >> s_pageShift);
|
|
while (page <= last && !s_snapOf[page].load(std::memory_order_acquire))
|
|
page++;
|
|
if (page > last)
|
|
continue;
|
|
std::lock_guard lock(s_protectMutex);
|
|
ProtectRun run;
|
|
for (; page <= last; page++)
|
|
if (s_snapOf[page].load(std::memory_order_relaxed)) // (still open: not closed meanwhile)
|
|
SetProt(page, 1, &run);
|
|
}
|
|
}
|
|
|
|
// The sequence readers' scans of the range (size > 0), current once
|
|
// the range is settled.
|
|
uint64_t SequenceOf(uint32_t physical, uint32_t size)
|
|
{
|
|
uint32_t first = (physical & (kPhysicalSize - 1)) >> kSubShift;
|
|
uint32_t last = std::min<uint64_t>(kSubCount - 1, (uint64_t(physical & (kPhysicalSize - 1)) + size - 1) >> kSubShift);
|
|
uint64_t highest = 0;
|
|
for (uint32_t sub = first; sub <= last; sub++)
|
|
highest = std::max(highest, s_written[sub].load(std::memory_order_relaxed));
|
|
return highest;
|
|
}
|
|
|
|
uint32_t PagesWrittenAfter(uint32_t physical, uint32_t size, uint64_t sequence, uint32_t* pages, uint32_t max)
|
|
{
|
|
uint32_t first = (physical & (kPhysicalSize - 1)) >> kSubShift;
|
|
uint32_t last = std::min<uint64_t>(kSubCount - 1, (uint64_t(physical & (kPhysicalSize - 1)) + size - 1) >> kSubShift);
|
|
uint32_t count = 0;
|
|
for (uint32_t sub = first; sub <= last; sub++)
|
|
if (s_written[sub].load(std::memory_order_relaxed) > sequence)
|
|
{
|
|
if (count < max)
|
|
pages[count] = sub << kSubShift;
|
|
count++;
|
|
}
|
|
return count;
|
|
}
|
|
|
|
bool WrittenAfter(uint32_t physical, uint32_t size, uint64_t sequence)
|
|
{
|
|
if (!s_ranges)
|
|
return SequenceOf(physical, size) > sequence;
|
|
uint32_t first = (physical & (kPhysicalSize - 1)) >> kSubShift;
|
|
uint32_t last = std::min<uint64_t>(kSubCount - 1, (uint64_t(physical & (kPhysicalSize - 1)) + size - 1) >> kSubShift);
|
|
// A block's sequence is at least each of its pages' (BumpWritten
|
|
// raises it first): a block not written since is skipped.
|
|
uint32_t blockSubs = 1u << (kBlockShift - kSubShift);
|
|
for (uint32_t sub = first; sub <= last;)
|
|
{
|
|
uint32_t blockLast = std::min(last, sub | (blockSubs - 1));
|
|
if (s_blockWritten[(uint64_t(sub) << kSubShift) >> kBlockShift].load(std::memory_order_relaxed) > sequence)
|
|
for (uint32_t p = sub; p <= blockLast; p++)
|
|
if (s_written[p].load(std::memory_order_relaxed) > sequence)
|
|
return true;
|
|
sub = blockLast + 1;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool BlocksWrittenAfter(uint32_t physical, uint32_t size, uint64_t sequence)
|
|
{
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
uint64_t end = std::min<uint64_t>(kPhysicalSize, start + size);
|
|
for (uint64_t block = start >> kBlockShift; block <= (end - 1) >> kBlockShift; block++)
|
|
if (s_blockWritten[block].load() > sequence)
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
// A sequence reader with a snapshot open anywhere: Read(physical,
|
|
// size, ...) once the range is settled (before the blocks: a compare
|
|
// raises them). Out of line and tail-called, so that the readers'
|
|
// common path, none open (always when off), is a load and a branch:
|
|
// no call, no stack frame.
|
|
template <auto Read, class... Args>
|
|
[[gnu::noinline]] auto Settled(uint32_t physical, uint32_t size, Args... args)
|
|
{
|
|
SettleOpen(physical, size);
|
|
return Read(physical, size, args...);
|
|
}
|
|
|
|
// The submission a CPU access to guest page `sub` must wait for: GPU
|
|
// writes for a read, any GPU use for a write.
|
|
uint64_t Hazard(uint32_t sub, bool write)
|
|
{
|
|
uint64_t use = s_gpuWrite[sub].load();
|
|
if (s_shadow)
|
|
return use; // GPU reads use the shadow copy
|
|
return write ? std::max(use, s_gpuRead[sub].load()) : use;
|
|
}
|
|
|
|
thread_local uint32_t t_faultPhysical = 0;
|
|
thread_local bool t_faultWrite = false;
|
|
|
|
// Wait (spinning; this runs in the fault handler) until the GPU is
|
|
// done with guest page `sub` for this access (`untile`: with its
|
|
// texture untile, see OpenSnapshot). False: an untile wait gave up.
|
|
// That one waits for a guest page the store doesn't touch, which the
|
|
// guest never ordered against the GPU: the storing thread may hold
|
|
// what the command processor needs before it next submits (with
|
|
// eager writes it runs the guest's interrupt handler itself, which
|
|
// takes guest locks, and the kernel's dispatcher lock in KeSetEvent).
|
|
// It gives up after kUntileWait and the untile's page is marked
|
|
// instead (always correct: the texture reloads). The bound is well
|
|
// above a frame of GPU latency: below it, each give-up would reload
|
|
// the texture and the next store would wait again, every frame.
|
|
constexpr auto kUntileWait = std::chrono::milliseconds(100);
|
|
bool WaitForGpu(uint32_t sub, bool write, bool untile = false)
|
|
{
|
|
uint64_t use = untile ? s_gpuUntile[sub].load() : Hazard(sub, write);
|
|
if (use <= s_completed.load() || t_gpuThread)
|
|
return true;
|
|
auto start = std::chrono::steady_clock::now();
|
|
bool done = true;
|
|
while (s_completed.load() < use)
|
|
{
|
|
if (untile && std::chrono::steady_clock::now() - start > kUntileWait)
|
|
{
|
|
done = false;
|
|
break;
|
|
}
|
|
g_submitRequested.store(true);
|
|
// While the game is suspended the GPU gate holds the
|
|
// submission back until it's resumed: poll every 1 ms instead.
|
|
// A signal handler can't wait on the gate, only read the flag
|
|
// itself (a lock-free atomic; Suspended()'s seqlock would spin
|
|
// forever in a fault that interrupted Suspend or Resume).
|
|
struct timespec ts{ 0, guesttime::g_state.suspended.load(std::memory_order_relaxed) ? 1'000'000 : 20'000 };
|
|
nanosleep(&ts, nullptr);
|
|
}
|
|
s_stalls++;
|
|
uint64_t us = uint64_t(std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start).count());
|
|
s_stallUs += us;
|
|
if (s_logGuard)
|
|
{
|
|
static std::atomic<uint32_t> logged{ 0 };
|
|
Dl_info dl{};
|
|
dladdr(t_faultPc, &dl);
|
|
if (logged++ < 400)
|
|
fprintf(stderr, "[guard] stall %llu us (%s %08X): %s %08X+%X (submission %llu%s) in %s\n", (unsigned long long)us,
|
|
t_faultWrite ? "write" : "read", t_faultPhysical, s_gpuTag[sub].load(), s_gpuRange[0][sub].load(),
|
|
s_gpuRange[1][sub].load(), (unsigned long long)use, done ? "" : ", gave up", dl.dli_sname ? dl.dli_sname : "?");
|
|
}
|
|
return done;
|
|
}
|
|
|
|
void Guard(uint32_t physical, uint32_t size, uint64_t submission, bool write, const char* tag, bool untile = false)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return;
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
uint64_t end = std::min<uint64_t>(kPhysicalSize, start + size);
|
|
uint32_t firstSub = uint32_t(start >> kSubShift), lastSub = uint32_t((end - 1) >> kSubShift);
|
|
std::lock_guard lock(s_protectMutex);
|
|
if (write)
|
|
for (uint64_t b = start >> kBlockShift; b <= (end - 1) >> kBlockShift; b++)
|
|
if (s_blockGpuWrite[b].load(std::memory_order_relaxed) < submission)
|
|
s_blockGpuWrite[b].store(submission, std::memory_order_relaxed);
|
|
// The array in a local: a store in the loop would reload the
|
|
// global (-fno-strict-aliasing).
|
|
std::atomic<uint64_t>* uses = (write ? s_gpuWrite : s_gpuRead).get();
|
|
for (uint32_t sub = firstSub; sub <= lastSub; sub++)
|
|
{
|
|
auto& use = uses[sub];
|
|
if (use.load(std::memory_order_relaxed) < submission)
|
|
Put(use, submission);
|
|
if (s_logGuard)
|
|
{
|
|
s_gpuTag[sub].store(tag);
|
|
s_gpuRange[0][sub].store(physical);
|
|
s_gpuRange[1][sub].store(size);
|
|
}
|
|
}
|
|
if (untile && s_gpuUntile)
|
|
for (uint32_t sub = firstSub; sub <= lastSub; sub++)
|
|
if (s_gpuUntile[sub].load(std::memory_order_relaxed) < submission)
|
|
Put(s_gpuUntile[sub], submission);
|
|
if (s_shadow && !write)
|
|
return; // the GPU reads its copy: nothing to protect, the use is recorded for SyncShadow
|
|
ProtectRun run;
|
|
for (uint32_t page = firstSub / s_subsPerPage, lastPage = lastSub / s_subsPerPage; page <= lastPage; page++)
|
|
SetProt(page, std::max(s_prot[page].load(), uint8_t(write ? 2 : 1)), &run); // only tightens
|
|
}
|
|
}
|
|
|
|
void Initialize()
|
|
{
|
|
s_pageSize = size_t(sysconf(_SC_PAGESIZE));
|
|
s_pageShift = uint32_t(__builtin_ctzll(s_pageSize));
|
|
s_pageCount = kPhysicalSize >> s_pageShift;
|
|
s_subsPerPage = s_pageShift > kSubShift ? 1u << (s_pageShift - kSubShift) : 1u;
|
|
s_watched.reset(new std::atomic<uint8_t>[s_pageCount]);
|
|
s_written.reset(new std::atomic<uint64_t>[kSubCount]);
|
|
s_prot.reset(new std::atomic<uint8_t>[s_pageCount]);
|
|
for (uint32_t i = 0; i < s_pageCount; i++)
|
|
{
|
|
s_watched[i].store(0);
|
|
s_prot[i].store(0);
|
|
}
|
|
for (uint32_t i = 0; i < kSubCount; i++)
|
|
s_written[i].store(0);
|
|
s_shadowSeq.reset(new std::atomic<uint64_t>[s_pageCount]);
|
|
s_gpuOwned.reset(new std::atomic<uint8_t>[s_pageCount]);
|
|
for (uint32_t i = 0; i < s_pageCount; i++)
|
|
{
|
|
s_shadowSeq[i].store(0);
|
|
s_gpuOwned[i].store(0);
|
|
}
|
|
s_blockWritten.reset(new std::atomic<uint64_t>[kPhysicalSize >> kBlockShift]);
|
|
s_blockGpuWrite.reset(new std::atomic<uint64_t>[kPhysicalSize >> kBlockShift]);
|
|
s_blockOwned.reset(new std::atomic<uint32_t>[kPhysicalSize >> kBlockShift]);
|
|
for (uint32_t i = 0; i < (kPhysicalSize >> kBlockShift); i++)
|
|
{
|
|
s_blockWritten[i].store(0);
|
|
s_blockGpuWrite[i].store(0);
|
|
s_blockOwned[i].store(0);
|
|
}
|
|
s_gpuRead.reset(new std::atomic<uint64_t>[kSubCount]);
|
|
s_gpuWrite.reset(new std::atomic<uint64_t>[kSubCount]);
|
|
for (uint32_t i = 0; i < kSubCount; i++)
|
|
{
|
|
s_gpuRead[i].store(0);
|
|
s_gpuWrite[i].store(0);
|
|
}
|
|
if (s_logWriters)
|
|
{
|
|
s_lastWriter.reset(new std::atomic<uint64_t>[kSubCount]);
|
|
s_lastWriterPc.reset(new std::atomic<uint32_t>[kSubCount]);
|
|
for (uint32_t i = 0; i < kSubCount; i++)
|
|
{
|
|
s_lastWriter[i].store(0);
|
|
s_lastWriterPc[i].store(0);
|
|
}
|
|
Dl_info dl{};
|
|
if (dladdr(reinterpret_cast<const void*>(&Initialize), &dl))
|
|
s_imageBase = static_cast<const uint8_t*>(dl.dli_fbase);
|
|
}
|
|
if (s_logGuard)
|
|
{
|
|
s_gpuTag.reset(new std::atomic<const char*>[kSubCount]);
|
|
s_gpuRange[0].reset(new std::atomic<uint32_t>[kSubCount]);
|
|
s_gpuRange[1].reset(new std::atomic<uint32_t>[kSubCount]);
|
|
}
|
|
// NFSMW_WATCH_SUBPAGES: a pending mask holds a host page's guest
|
|
// pages, a bit each. The copies are touched now, not first in the
|
|
// fault handler.
|
|
if (s_watchSubpages && s_subsPerPage > 1 && s_subsPerPage <= 32)
|
|
{
|
|
s_snapData = std::make_unique<uint8_t[]>(size_t(kSnapSlots) << s_pageShift);
|
|
s_snapOf.reset(new std::atomic<uint8_t>[s_pageCount]);
|
|
for (uint32_t i = 0; i < s_pageCount; i++)
|
|
s_snapOf[i].store(0, std::memory_order_relaxed);
|
|
s_blockSnaps.reset(new std::atomic<uint16_t>[kPhysicalSize >> kBlockShift]);
|
|
for (uint32_t i = 0; i < (kPhysicalSize >> kBlockShift); i++)
|
|
s_blockSnaps[i].store(0, std::memory_order_relaxed);
|
|
s_gpuUntile.reset(new std::atomic<uint64_t>[kSubCount]);
|
|
for (uint32_t i = 0; i < kSubCount; i++)
|
|
s_gpuUntile[i].store(0, std::memory_order_relaxed);
|
|
for (uint32_t i = 0; i < kSnapSlots; i++)
|
|
s_snapFree[i] = kSnapSlots - 1 - i;
|
|
s_snapFreeCount = kSnapSlots;
|
|
s_allSubs = s_subsPerPage == 32 ? ~0u : (1u << s_subsPerPage) - 1;
|
|
}
|
|
}
|
|
|
|
uint64_t Current()
|
|
{
|
|
return s_sequence.load();
|
|
}
|
|
|
|
void Watch(uint32_t physical, uint32_t size)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return;
|
|
uint32_t first = (physical & (kPhysicalSize - 1)) >> s_pageShift;
|
|
uint32_t last = std::min<uint64_t>(s_pageCount - 1, (uint64_t(physical & (kPhysicalSize - 1)) + size - 1) >> s_pageShift);
|
|
std::lock_guard lock(s_protectMutex);
|
|
ProtectRun run;
|
|
for (uint32_t page = first; page <= last; page++)
|
|
if (Change(s_watched[page], uint8_t(1)))
|
|
SetProt(page, std::max<uint8_t>(s_prot[page].load(), 1), &run);
|
|
}
|
|
|
|
uint64_t WriteEpoch()
|
|
{
|
|
return s_writeEpoch.load(std::memory_order_acquire);
|
|
}
|
|
|
|
void SettleOpen(uint32_t physical, uint32_t size)
|
|
{
|
|
if (!s_snapOf || size == 0)
|
|
return;
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
SettleRange(start, std::min<uint64_t>(kPhysicalSize, start + size));
|
|
}
|
|
|
|
uint64_t WriteSequence(uint32_t physical, uint32_t size)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return 0;
|
|
if (g_snapsOpen.load(std::memory_order_acquire))
|
|
return Settled<SequenceOf>(physical, size);
|
|
return SequenceOf(physical, size);
|
|
}
|
|
|
|
void SetWriterFrame(uint64_t frame)
|
|
{
|
|
s_writerFrame.store(frame, std::memory_order_relaxed);
|
|
}
|
|
|
|
LastWrite LastWriter(uint32_t physical)
|
|
{
|
|
if (!s_lastWriter)
|
|
return {};
|
|
uint32_t sub = (physical & (kPhysicalSize - 1)) >> kSubShift;
|
|
uint64_t v = s_lastWriter[sub].load(std::memory_order_relaxed);
|
|
return { Writer(v >> 60), uint32_t(v >> 32) & 0x0FFFFFFF, uint32_t(v), s_lastWriterPc[sub].load(std::memory_order_relaxed) };
|
|
}
|
|
|
|
uint32_t WrittenPages(uint32_t physical, uint32_t size, uint64_t sequence, uint32_t* pages, uint32_t max)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return 0;
|
|
if (g_snapsOpen.load(std::memory_order_acquire))
|
|
return Settled<PagesWrittenAfter>(physical, size, sequence, pages, max);
|
|
return PagesWrittenAfter(physical, size, sequence, pages, max);
|
|
}
|
|
|
|
uint64_t MarkWritten(uint32_t physical, uint32_t size, Writer kind, uint32_t tag)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return 0;
|
|
if (s_ranges)
|
|
{
|
|
uint32_t firstSub = (physical & (kPhysicalSize - 1)) >> kSubShift;
|
|
uint32_t lastSub = std::min<uint64_t>(kSubCount - 1, (uint64_t(physical & (kPhysicalSize - 1)) + size - 1) >> kSubShift);
|
|
return BumpWrittenRange(firstSub, lastSub, kind, tag);
|
|
}
|
|
uint32_t first = (physical & (kPhysicalSize - 1)) >> s_pageShift;
|
|
uint32_t last = std::min<uint64_t>(s_pageCount - 1, (uint64_t(physical & (kPhysicalSize - 1)) + size - 1) >> s_pageShift);
|
|
for (uint32_t page = first; page <= last; page++)
|
|
BumpWritten(page, kind, tag);
|
|
return 0;
|
|
}
|
|
|
|
bool WrittenSince(uint32_t physical, uint32_t size, uint64_t sequence)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return false;
|
|
if (g_snapsOpen.load(std::memory_order_acquire))
|
|
return Settled<WrittenAfter>(physical, size, sequence);
|
|
return WrittenAfter(physical, size, sequence);
|
|
}
|
|
|
|
void BeforeHostWrite(const void* host, size_t size)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return;
|
|
int64_t start = PhysicalOf(host);
|
|
int64_t end = PhysicalOf(static_cast<const uint8_t*>(host) + size - 1);
|
|
if (start < 0 || end < start)
|
|
return;
|
|
s_hostWriteBytes += size;
|
|
for (int64_t sub = start >> kSubShift; sub <= end >> kSubShift; sub++)
|
|
WaitForGpu(uint32_t(sub), true);
|
|
std::lock_guard lock(s_protectMutex);
|
|
ProtectRun run;
|
|
for (int64_t page = start >> s_pageShift; page <= end >> s_pageShift; page++)
|
|
{
|
|
ReadBackLocked(uint32_t(page));
|
|
// An open snapshot ends unseen (all its guest pages are marked
|
|
// next): a compare would protect the page under the kernel's write.
|
|
if (s_snapOf && s_snapOf[page].load(std::memory_order_relaxed))
|
|
CloseSnapshot(uint32_t(page), false);
|
|
BumpWritten(uint32_t(page), Writer::kHostWrite, uint32_t(start));
|
|
s_watched[page].store(0);
|
|
// The kernel can't fault: open the page whatever else it holds.
|
|
SetProt(uint32_t(page), 0, &run);
|
|
if (WantedProt(uint32_t(page)) != 0)
|
|
s_rearm.push_back(uint32_t(page));
|
|
}
|
|
}
|
|
|
|
bool HandleFault(const void* host, bool write)
|
|
{
|
|
if (!s_pageCount)
|
|
return false;
|
|
int64_t physical = PhysicalOf(host);
|
|
if (physical < 0)
|
|
return false;
|
|
uint32_t page = uint32_t(physical >> s_pageShift);
|
|
uint32_t sub = uint32_t(physical >> kSubShift);
|
|
s_faults++;
|
|
t_faultWrite = write;
|
|
t_faultPhysical = uint32_t(physical);
|
|
// Accessible again in every mirror (a racing thread may have done it
|
|
// already; retrying the access is correct either way) once the GPU
|
|
// is done with this guest page. The command processor may guard it
|
|
// again while we wait: check under the lock.
|
|
bool markUntiles = false; // a wait for another guest page's untile gave up
|
|
while (true)
|
|
{
|
|
WaitForGpu(sub, write);
|
|
std::unique_lock lock(s_protectMutex);
|
|
if (!t_gpuThread && Hazard(sub, write) > s_completed.load())
|
|
continue;
|
|
if (t_gpuThread)
|
|
{
|
|
// Never waits (GPU threads store through GpuThreadStore; this
|
|
// is a fallback): the guest page's guard is dropped.
|
|
s_gpuRead[sub].store(0);
|
|
s_gpuWrite[sub].store(0);
|
|
}
|
|
if (s_shadow && s_gpuOwned[page].load())
|
|
{
|
|
// The GPU wrote this page (a resolve): once all of its writes
|
|
// are done, bring the guest's copy up to date.
|
|
uint32_t pendingSub = UINT32_MAX;
|
|
for (uint32_t i = 0, s2 = page * s_subsPerPage; i < s_subsPerPage; i++, s2++)
|
|
if (!t_gpuThread && s_gpuWrite[s2].load() > s_completed.load())
|
|
pendingSub = s2;
|
|
if (pendingSub != UINT32_MAX)
|
|
{
|
|
lock.unlock();
|
|
WaitForGpu(pendingSub, false);
|
|
continue;
|
|
}
|
|
ReadBackLocked(page);
|
|
}
|
|
if (write)
|
|
{
|
|
// (NFSMW_WATCH_SUBPAGES: just this guest page, the others
|
|
// compared later, once no texture untile reads them, or
|
|
// marked if a wait for one gives up.)
|
|
uint32_t untile = UINT32_MAX;
|
|
if (!OpenSnapshot(page, sub, uint32_t(physical), markUntiles, untile))
|
|
{
|
|
if (untile != UINT32_MAX)
|
|
{
|
|
lock.unlock();
|
|
s_snapWaits.fetch_add(1, std::memory_order_relaxed);
|
|
if (!WaitForGpu(untile, true, true))
|
|
{
|
|
s_snapGaveUp.fetch_add(1, std::memory_order_relaxed);
|
|
markUntiles = true;
|
|
}
|
|
continue;
|
|
}
|
|
BumpWritten(page, Writer::kCpuFault, uint32_t(physical));
|
|
}
|
|
s_watched[page].store(0);
|
|
}
|
|
uint8_t wanted = WantedProt(page);
|
|
uint8_t allowed = write ? 0 : 1;
|
|
if (wanted <= allowed)
|
|
{
|
|
SetProt(page, wanted);
|
|
return true;
|
|
}
|
|
// Another guest page of this host page still has a GPU hazard:
|
|
// open it for this access and protect it again soon.
|
|
SetProt(page, allowed);
|
|
s_rearm.push_back(page);
|
|
s_opened++;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
void Rearm()
|
|
{
|
|
std::lock_guard lock(s_protectMutex);
|
|
// NFSMW_WATCH_SUBPAGES: snapshots nothing compared for kSnapAge
|
|
// submissions give their slots back (pending pages marked unseen),
|
|
// or pages the game writes and no texture checks would hold them all.
|
|
s_rearms++;
|
|
if (s_snapOf && g_snapsOpen.load(std::memory_order_relaxed))
|
|
for (const Snap& snap : s_snap)
|
|
if (snap.page != UINT32_MAX && s_rearms - snap.opened > kSnapAge)
|
|
CloseSnapshot(snap.page, false);
|
|
if (s_rearm.empty())
|
|
return;
|
|
std::sort(s_rearm.begin(), s_rearm.end());
|
|
s_rearm.erase(std::unique(s_rearm.begin(), s_rearm.end()), s_rearm.end());
|
|
ProtectRun run;
|
|
for (uint32_t page : s_rearm)
|
|
Reprotect(page, &run);
|
|
run.Flush();
|
|
s_rearm.clear();
|
|
}
|
|
|
|
void GpuRead(uint32_t physical, uint32_t size, uint64_t submission, const char* tag, bool untile)
|
|
{
|
|
Guard(physical, size, submission, false, tag, untile);
|
|
}
|
|
|
|
void GpuWrite(uint32_t physical, uint32_t size, uint64_t submission, const char* tag)
|
|
{
|
|
Guard(physical, size, submission, true, tag);
|
|
}
|
|
|
|
bool GpuWritePending(uint32_t physical, uint32_t size)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return false;
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
uint64_t end = std::min<uint64_t>(kPhysicalSize, start + size);
|
|
uint64_t completed = s_completed.load();
|
|
if (s_ranges)
|
|
{
|
|
// Blocks no pending GPU write reaches are skipped (see s_blockGpuWrite).
|
|
for (uint64_t b = start >> kBlockShift; b <= (end - 1) >> kBlockShift; b++)
|
|
if (s_blockGpuWrite[b].load(std::memory_order_relaxed) > completed)
|
|
{
|
|
uint64_t from = std::max(start, b << kBlockShift), to = std::min(end, (b + 1) << kBlockShift);
|
|
for (uint64_t sub = from >> kSubShift; sub <= (to - 1) >> kSubShift; sub++)
|
|
if (s_gpuWrite[sub].load(std::memory_order_relaxed) > completed)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
for (uint64_t sub = start >> kSubShift; sub <= (end - 1) >> kSubShift; sub++)
|
|
if (s_gpuWrite[sub].load() > completed)
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
void SetCompleted(uint64_t submission)
|
|
{
|
|
s_completed.store(submission);
|
|
}
|
|
|
|
void Quiesce()
|
|
{
|
|
// Every recording thread holds the lock from taking its sequence
|
|
// until its pages have it: one that took it before this waits here
|
|
// or is done, and one that takes it after takes a newer one.
|
|
std::lock_guard lock(s_protectMutex);
|
|
}
|
|
|
|
bool CpuReadable(uint32_t physical, uint32_t size)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return true;
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
uint64_t end = std::min<uint64_t>(kPhysicalSize, start + size);
|
|
for (uint64_t page = start >> s_pageShift; page <= (end - 1) >> s_pageShift; page++)
|
|
if (s_prot[page].load(std::memory_order_relaxed) == 2)
|
|
return false;
|
|
return true;
|
|
}
|
|
|
|
void GpuThreadWrite(void* host, size_t size, void (*write)(const void* context), const void* context)
|
|
{
|
|
int64_t first = PageOf(host);
|
|
int64_t last = PageOf(static_cast<const uint8_t*>(host) + size - 1);
|
|
if (first < 0 || last < first)
|
|
{
|
|
write(context);
|
|
return;
|
|
}
|
|
// Temporarily writable, stored, then protected as before; the pages
|
|
// count as written (textures in them reload).
|
|
std::lock_guard lock(s_protectMutex);
|
|
bool open = true;
|
|
for (int64_t page = first; page <= last; page++)
|
|
open = open && s_prot[page].load() == 0;
|
|
if (open)
|
|
{
|
|
write(context);
|
|
return;
|
|
}
|
|
for (int64_t page = first; page <= last; page++)
|
|
{
|
|
ReadBackLocked(uint32_t(page));
|
|
SetProt(uint32_t(page), 0);
|
|
}
|
|
write(context);
|
|
for (int64_t page = first; page <= last; page++)
|
|
{
|
|
if (s_snapOf && s_snapOf[page].load(std::memory_order_relaxed))
|
|
CloseSnapshot(uint32_t(page), false); // (an open page of the range: marked whole next, as BeforeHostWrite)
|
|
BumpWritten(uint32_t(page), Writer::kGpuStore, uint32_t(PhysicalOf(host)));
|
|
s_watched[page].store(0);
|
|
Reprotect(uint32_t(page));
|
|
}
|
|
}
|
|
|
|
void GpuThreadStore(void* host, const void* data, size_t size)
|
|
{
|
|
GpuThreadWrite(host, size, [&] { memcpy(host, data, size); });
|
|
}
|
|
|
|
void EnableShadow(uint8_t* shadow)
|
|
{
|
|
s_shadow = shadow;
|
|
}
|
|
|
|
bool RangeWrittenSince(uint32_t physical, uint32_t size, uint64_t sequence)
|
|
{
|
|
if (!s_pageCount || size == 0)
|
|
return false;
|
|
if (g_snapsOpen.load(std::memory_order_acquire))
|
|
return Settled<BlocksWrittenAfter>(physical, size, sequence);
|
|
return BlocksWrittenAfter(physical, size, sequence);
|
|
}
|
|
|
|
uint64_t SyncShadow(uint32_t physical, uint32_t size, uint64_t since)
|
|
{
|
|
if (!s_shadow || !s_pageCount || size == 0)
|
|
return 0;
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
uint64_t end = std::min<uint64_t>(kPhysicalSize, start + size);
|
|
std::lock_guard lock(s_protectMutex);
|
|
// Protect before copying: a store after this faults and makes the
|
|
// page dirty again; one before it is in the copy. Up to 256 pages
|
|
// are protected (in runs) and then copied at a time.
|
|
ProtectRun run;
|
|
uint32_t pending[256], pendingCount = 0;
|
|
uint64_t sequence = s_sequence.load();
|
|
auto copyPending = [&] {
|
|
run.Flush();
|
|
if (pendingCount == 0 && s_ranges)
|
|
return; // most calls copy nothing: no clock reads
|
|
auto copyStart = std::chrono::steady_clock::now();
|
|
for (uint32_t i = 0; i < pendingCount; i++)
|
|
{
|
|
if (s_shadowSeq[pending[i]].load() == 0)
|
|
s_firstUploadBytes += s_pageSize;
|
|
memcpy(s_shadow + (uint64_t(pending[i]) << s_pageShift), GuestPage(pending[i]), s_pageSize);
|
|
Put(s_shadowSeq[pending[i]], sequence);
|
|
}
|
|
s_uploadNs += uint64_t(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - copyStart).count());
|
|
s_uploadBytes += uint64_t(pendingCount) * s_pageSize;
|
|
pendingCount = 0;
|
|
sequence = s_sequence.load();
|
|
};
|
|
uint32_t blockPages = 1u << (kBlockShift - s_pageShift);
|
|
for (uint64_t page = start >> s_pageShift; page <= (end - 1) >> s_pageShift; page++)
|
|
{
|
|
// Every page of the range was copied, clean or GPU-owned when it
|
|
// was last synced at `since`: a 256 KB block nothing was written
|
|
// to since then has nothing to copy.
|
|
if (since && s_ranges && s_blockWritten[(page << s_pageShift) >> kBlockShift].load(std::memory_order_relaxed) <= since)
|
|
{
|
|
page |= blockPages - 1;
|
|
continue;
|
|
}
|
|
if (s_gpuOwned[page].load())
|
|
continue; // the GPU's copy is the newer one
|
|
uint64_t copied = s_shadowSeq[page].load();
|
|
if (copied != 0 && PageWritten(uint32_t(page)) <= copied)
|
|
continue;
|
|
// Work in flight may still read the old copy: wait for it first.
|
|
for (uint32_t i = 0, sub = uint32_t(page) * s_subsPerPage; i < s_subsPerPage; i++, sub++)
|
|
if (s_gpuRead[sub].load() > s_completed.load())
|
|
{
|
|
copyPending();
|
|
return s_gpuRead[sub].load();
|
|
}
|
|
Put(s_watched[page], uint8_t(1));
|
|
Reprotect(uint32_t(page), &run);
|
|
pending[pendingCount++] = uint32_t(page);
|
|
if (pendingCount == std::size(pending))
|
|
copyPending();
|
|
}
|
|
copyPending();
|
|
return 0;
|
|
}
|
|
|
|
void MarkGpuOwned(uint32_t physical, uint32_t size)
|
|
{
|
|
if (!s_shadow || !s_pageCount || size == 0)
|
|
return;
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
uint64_t end = std::min<uint64_t>(kPhysicalSize, start + size);
|
|
std::lock_guard lock(s_protectMutex);
|
|
ProtectRun run;
|
|
for (uint64_t page = start >> s_pageShift; page <= (end - 1) >> s_pageShift; page++)
|
|
{
|
|
bool owned = s_gpuOwned[page].load(std::memory_order_relaxed);
|
|
if (!owned)
|
|
s_blockOwned[(page << s_pageShift) >> kBlockShift].fetch_add(1, std::memory_order_relaxed);
|
|
else if (s_ranges && s_prot[page].load(std::memory_order_relaxed) == 2)
|
|
continue; // owned and protected already (a target resolved every frame)
|
|
Put(s_gpuOwned[page], uint8_t(1));
|
|
Reprotect(uint32_t(page), &run);
|
|
}
|
|
}
|
|
|
|
bool AnyGpuOwned(uint32_t physical, uint32_t size)
|
|
{
|
|
if (!s_shadow || !s_pageCount || size == 0)
|
|
return false;
|
|
uint64_t start = physical & (kPhysicalSize - 1);
|
|
uint64_t end = std::min<uint64_t>(kPhysicalSize, start + size);
|
|
if (s_ranges)
|
|
{
|
|
// Blocks without a GPU-owned page are skipped. Pages become owned
|
|
// only in MarkGpuOwned, on the command processor, so a count read
|
|
// here is never below the truth; a read-back on another thread
|
|
// lowers it after its copy (acquire: the copy is then visible).
|
|
for (uint64_t b = start >> kBlockShift; b <= (end - 1) >> kBlockShift; b++)
|
|
if (s_blockOwned[b].load(std::memory_order_acquire))
|
|
{
|
|
uint64_t from = std::max(start, b << kBlockShift), to = std::min(end, (b + 1) << kBlockShift);
|
|
for (uint64_t page = from >> s_pageShift; page <= (to - 1) >> s_pageShift; page++)
|
|
if (s_gpuOwned[page].load())
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
for (uint64_t page = start >> s_pageShift; page <= (end - 1) >> s_pageShift; page++)
|
|
if (s_gpuOwned[page].load())
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
GuardStats GetGuardStats()
|
|
{
|
|
return { s_stalls.load(), s_stallUs.load(), s_faults.load(), s_opened.load(), s_uploadBytes.load(), s_readbackBytes.load(),
|
|
s_protects.load(), s_protectNs.load(), s_hostWriteBytes.load(), s_firstUploadBytes.load(), s_uploadNs.load(),
|
|
s_snapOf != nullptr, s_snapOpened.load(), s_snapClean.load(), s_snapChanged.load(), s_snapEvicted.load(), s_snapFull.load(),
|
|
s_snapWhole.load(), s_snapReading.load(), s_snapWaits.load(), s_snapGaveUp.load() };
|
|
}
|
|
}
|