SpeedBreakerProject--speedbreaker/runtime/gpu/renderer.cpp

// SpeedBreaker runtime. GPL-3.0-or-later (see COPYING).
// See renderer.h. State translation follows Xenia (BSD): draw_util.cc and
// vulkan/vulkan_pipeline_cache.cc.
#include <stdafx.h>
#include "renderer.h"
#include <cpu/guest_thread.h>
#include <cpu/host_cpu.h>
#include <cpu/guest_time.h>
#include <debug/timeline.h>
#include <unordered_set>
#include "block_ranges.h"
#include "edge_bars.h"
#include "memory_plan.h"
#include "pipeline_cache_header.h"
#include "shader_translator.h"
#include "zpd_report.h"
#include "depth24_glsl.h"
#include "depth_only.h"
#include "render_scale.h"
#include "shared_memory_glsl.h"
#include "xenos/registers.h"

#include <kernel/vmem.h>
#include <kernel/write_watch.h>
#include <video/presenter.h>
#include <video/picture_fit.h>
#include <video/scale_budget.h>
#include <user/paths.h>
#include <user/settings.h>
#include <game/ultrawide.h>

#include <glslang/Include/glslang_c_interface.h>
#include <glslang/Public/resource_limits_c.h>

#include <array>
#include <cmath>
#include <condition_variable>
#include <cstdarg>
#include <deque>
#include <map>
#include <mutex>
#include <set>
#include <thread>
#include <sys/resource.h>
#include <time.h>
#include <unistd.h>
#include <memory>
#include <unordered_map>

namespace gpu::renderer
{
    namespace
    {
        using namespace xe::gpu;

        const video::VulkanContext* s_vk = nullptr;
        VkDevice s_dev = VK_NULL_HANDLE;
        const uint32_t* s_regs = nullptr;
        PFN_vkCmdPushDescriptorSetKHR s_pushDescriptorSet = nullptr;
        Stats s_stats{};
        // NFSMW_GPU_EXP bitmask, for performance experiments: 1 skip draw
        // calls, 2 skip resolve copies, 4 skip texture loads, 8 bind 1x1
        // placeholders instead of textures.
        const uint32_t s_exp = [] { const char* v = std::getenv("NFSMW_GPU_EXP"); return v ? uint32_t(std::atoi(v)) : 0u; }();
        // Fewer render passes and texture reloads (NFSMW_PASS_MERGE,
        // NFSMW_CLEAR_FLUSH_RECT, NFSMW_DIRECT_CUBE, NFSMW_FUSE_PARTIAL): on
        // by default where passes cost most, the tile GPUs: under MoltenVK
        // (iPad race frame: 46 -> 20 passes; heavy scenes ~4 ms less GPU
        // time) and Turnip on arm64 Linux (Steam Frame: 37 -> 21 passes,
        // -0.3 ms GPU a busy frame); off elsewhere until measured there.
        // Lighter passes (NFSMW_UNTILED_AREA, NFSMW_PASS_BARRIER): Apple only.
        // On the Frame the smaller area cost +0.3 ms (Turnip chooses tiled
        // or direct rendering by the area) and the barrier changed nothing.
        // Each variable overrides.
#if defined(__APPLE__) || (defined(__linux__) && defined(__aarch64__))
        constexpr bool kFewerPasses = true;
#else
        constexpr bool kFewerPasses = false;
#endif
#ifdef __APPLE__
        constexpr bool kLighterPasses = true;
#else
        constexpr bool kLighterPasses = false;
#endif

        // Where a frame's time went, for hitch reports (renderer::HitchReport).
        struct HitchTimes
        {
            double shaderMs = 0, pipelineMs = 0, textureMs = 0, shadowMs = 0, imageMs = 0, submitMs = 0;
            uint32_t shaders = 0, pipelines = 0, textures = 0, images = 0, scaledTextures = 0;
            uint64_t textureBytes = 0;
        };
        HitchTimes s_hitch;
        struct HitchTimer
        {
            double& total;
            std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
            explicit HitchTimer(double& t) : total(t) {}
            ~HitchTimer() { total += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count(); }
        };

        float RegF(uint32_t index)
        {
            float f;
            memcpy(&f, &s_regs[index], 4);
            return f;
        }

        template<typename T>
        T Reg()
        {
            T t;
            t.value = s_regs[T::register_index];
            return t;
        }

        template<typename T>
        T RegAt(uint32_t index)
        {
            T t;
            t.value = s_regs[index];
            return t;
        }

        void Check(VkResult r, const char* what)
        {
            if (r != VK_SUCCESS)
            {
                fprintf(stderr, "[renderer] %s failed: %d\n", what, int(r));
                abort();
            }
        }

        // Log a message the first time `key` is seen.
        void LogOnce(uint64_t key, const char* fmt, ...)
        {
            static std::unordered_map<uint64_t, bool> seen;
            if (seen[key])
                return;
            seen[key] = true;
            va_list args;
            va_start(args, fmt);
            fprintf(stderr, "[renderer] ");
            vfprintf(stderr, fmt, args);
            fprintf(stderr, "\n");
            va_end(args);
        }

        uint32_t GpuSwap(uint32_t v, uint32_t endian)
        {
            switch (endian & 3)
            {
            case 1: return ((v << 8) & 0xFF00FF00) | ((v >> 8) & 0x00FF00FF);
            case 2: return __builtin_bswap32(v);
            case 3: return (v >> 16) | (v << 16);
            default: return v;
            }
        }

        // ---------------------------------------------------------------
        // Memory

        VkPhysicalDeviceMemoryProperties s_memProps;
        bool s_memoryBudget = false;  // VK_EXT_memory_budget (the [perf] line's device memory)

        uint32_t FindMemoryType(uint32_t bits, VkMemoryPropertyFlags want)
        {
            for (uint32_t i = 0; i < s_memProps.memoryTypeCount; i++)
                if ((bits & (1u << i)) && (s_memProps.memoryTypes[i].propertyFlags & want) == want)
                    return i;
            for (uint32_t i = 0; i < s_memProps.memoryTypeCount; i++)
                if (bits & (1u << i))
                    return i;
            return 0;
        }

        // Guest physical memory (512 MB), imported without a copy.
        constexpr VkDeviceSize kSharedSize = 512ull << 20;
        VkBuffer s_shared = VK_NULL_HANDLE;
        VkDeviceMemory s_sharedMemory = VK_NULL_HANDLE;
        uint8_t* s_sharedHost = nullptr;
        // Devices that bind less of a storage buffer than guest memory
        // (maxStorageBufferRange: 128 MB on Turnip, the Steam Frame's Adreno,
        // where everything above the first 128 MB read as zeros: a black
        // picture) get it as four 128 MB parts: binding 0 and
        // kSharedPartBinding.. in every layout that has it, SPLIT_MEMORY in
        // the shaders (gpu/shared_memory_glsl.h). Elsewhere (RADV, MoltenVK:
        // 4 GB) one binding, and the shaders compile as before.
        // NFSMW_SPLIT_MEMORY=1|0 forces the choice.
        constexpr VkDeviceSize kSharedPart = 128ull << 20;
        constexpr uint32_t kSharedPartBinding = 13;
        bool s_splitMemory = false;
        VkDescriptorBufferInfo s_sharedInfo[4]{};  // the binding(s): [0] alone, or the four parts

        void SetSharedInfo()
        {
            for (uint32_t k = 0; k < 4; k++)
                s_sharedInfo[k] = s_splitMemory ? VkDescriptorBufferInfo{ s_shared, k * kSharedPart, kSharedPart }
                                                : VkDescriptorBufferInfo{ s_shared, 0, kSharedSize };
        }

        // A layout's bindings, plus (split) guest memory's other three
        // parts: copies of its binding `shared`.
        std::vector<VkDescriptorSetLayoutBinding> WithSharedParts(const VkDescriptorSetLayoutBinding* b, uint32_t count,
            uint32_t shared = 0)
        {
            std::vector<VkDescriptorSetLayoutBinding> out(b, b + count);
            for (uint32_t k = 0; s_splitMemory && k < 3; k++)
            {
                out.push_back(b[shared]);
                out.back().binding = kSharedPartBinding + k;
            }
            return out;
        }

        // Appends the writes for guest memory's other three parts (split
        // only) after `count` writes; returns the new count. `w` has room
        // for 3 more.
        uint32_t AddSharedParts(VkWriteDescriptorSet* w, uint32_t count, VkDescriptorSet set = VK_NULL_HANDLE)
        {
            for (uint32_t k = 1; s_splitMemory && k < 4; k++)
                w[count++] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, set, kSharedPartBinding + k - 1, 0, 1,
                    VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &s_sharedInfo[k], nullptr };
            return count;
        }

        // A stand-in storage buffer for a binding a shader won't read this
        // time (no scaled memory): guest memory, as much as one binding takes.
        VkDescriptorBufferInfo DummyStorage(VkBuffer buffer)
        {
            return { buffer ? buffer : s_shared, 0, buffer || !s_splitMemory ? VK_WHOLE_SIZE : kSharedPart };
        }
        // Discrete GPUs whose CPU-mapped VRAM holds it, and drivers that can't
        // import the guest's memory (amdgpu imports only anonymous memory, and
        // guest physical memory is a shared mapping, mirrored three times;
        // Intel's Linux driver loses the device when the write watch protects
        // imported pages): the GPU gets its own copy, kept current page by
        // page from the write watch (writewatch::SyncShadow). Where it goes,
        // and when guest memory is imported instead: gpu/memory_plan.h.
        // NFSMW_SHARED_MEMORY=import|shadow overrides the choice.
        bool s_shadowMode = false;
        bool s_shadowInVram = false;  // (the "ready" line)
        std::unordered_map<uint64_t, uint64_t> s_shadowSynced;  // range -> writewatch::Current() at its last sync

        uint32_t LoadPhysical(uint32_t address)
        {
            uint32_t v;
            memcpy(&v, s_sharedHost + (address & 0x1FFFFFFC), 4);
            return v;  // host-order load of the guest's bytes (as the GPU reads them)
        }

        // Upload ring: draw constants, converted indices and texture staging.
        // One slot per in-flight submission, so a slot is reused only after
        // the GPU finished with it.
        constexpr uint32_t kSlots = 12;
        constexpr VkDeviceSize kSlotSize = 16ull << 20;
        constexpr VkDeviceSize kRingSize = kSlots * kSlotSize;
        VkBuffer s_ring = VK_NULL_HANDLE;
        VkDeviceMemory s_ringMemory = VK_NULL_HANDLE;
        uint8_t* s_ringHost = nullptr;
        VkDeviceSize s_ringOffset = 0;

        // XenosConstants (shader_translator.cpp): c[512], fetch[48], bools[2],
        // loops[8], drawInfo[4], each a vec4/uvec4.
        constexpr uint32_t kConstantsSize = (512 + 48 + 2 + 8 + 4) * 16;

        // Image memory. Images are bound at offsets into blocks of device
        // memory (64 MB; gpu/block_ranges.h keeps each block's free ranges),
        // one list per memory type, instead of an allocation each. On the
        // Deck vkAllocateMemory was most of a new texture's cost (40-220 us
        // in scripted runs, 380 us while the kernel was moving memory out of
        // VRAM, about 1 ms after a long free roam, where frames that streamed
        // textures spent a median 9.5 ms making images); a range in a block
        // is a few us. A block is allocated ahead of need, off the command
        // processor: the nfsmw-memory thread keeps an empty spare of each
        // memory type in use (0.1-0.2 ms a block there, unless the kernel has
        // memory to move), so the command processor takes the spare when a
        // block fills and the thread makes the next. Without a spare ready (the first block, or a
        // burst that outran the thread) the command processor allocates the
        // block itself, counted in the [perf] line. An image bigger than half
        // a block, one the driver wants in an allocation of its own
        // (VkMemoryDedicatedRequirements) and any image when a block can't be
        // had keep their own allocation, as before. A block whose images all
        // went is a spare too, freed by the thread after a few seconds unused
        // beside another. Images are destroyed only with the GPU idle
        // (DestroyImage), so a range can be handed out again at once.
        // NFSMW_IMAGE_POOL=0: an allocation per image (the old path);
        // NFSMW_IMAGE_BLOCK_MB: the block size (16-1024).
        struct ImageBlock
        {
            VkDeviceMemory memory = VK_NULL_HANDLE;
            uint32_t type = 0;
            gpu::BlockRanges ranges;
        };

        struct Image
        {
            VkImage image = VK_NULL_HANDLE;
            VkDeviceMemory memory = VK_NULL_HANDLE;  // its own, or its block's
            VkImageView view = VK_NULL_HANDLE;
            VkDeviceSize bytes = 0;  // its allocation or range (the [perf] line's memory)
            ImageBlock* block = nullptr;  // bound at `offset` in this block, if any
            VkDeviceSize offset = 0;
        };

        const bool s_imagePool = [] { const char* v = std::getenv("NFSMW_IMAGE_POOL"); return !v || v[0] != '0'; }();
        const VkDeviceSize s_blockSize = [] {
            const char* v = std::getenv("NFSMW_IMAGE_BLOCK_MB");
            return VkDeviceSize(v ? std::clamp(std::atoi(v), 16, 1024) : 64) << 20;
        }();
        std::vector<std::unique_ptr<ImageBlock>> s_blocks;  // in use (command processor)
        // Empty blocks by memory type, the newest last, under s_spareMutex:
        // the one the nfsmw-memory thread keeps ready, and blocks whose images
        // all went. Those are kept kSpareKeep (a settings change drops
        // textures or targets and makes them again at once) and then freed
        // down to one.
        struct Spare
        {
            VkDeviceMemory memory;
            std::chrono::steady_clock::time_point since;
        };
        constexpr auto kSpareKeep = std::chrono::seconds(5);
        std::mutex s_spareMutex;
        std::condition_variable s_spareCv;
        std::vector<Spare> s_spares[VK_MAX_MEMORY_TYPES];
        uint32_t s_spareWanted = 0, s_spareFailed = 0;  // memory type bits
        // Totals for the [perf] line (command processor): images made and
        // the time it took, vkAllocateMemory calls for them (their own or a
        // block's) and the time those took, blocks taken from the spares,
        // blocks the spares didn't cover; under s_spareMutex, the memory
        // thread's: blocks it made and their time, blocks freed.
        struct ImageTotals
        {
            uint64_t images = 0, imageNs = 0, allocations = 0, allocationNs = 0, fromSpare = 0, blocksHere = 0;
            uint64_t spareBlocks = 0, spareNs = 0, blocksFreed = 0;
        };
        ImageTotals s_imageTotals;

        uint64_t SinceNs(std::chrono::steady_clock::time_point start)
        {
            return uint64_t(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - start).count());
        }

        VkDeviceMemory AllocateBlock(uint32_t type)
        {
            VkMemoryAllocateInfo mai{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
            mai.allocationSize = s_blockSize;
            mai.memoryTypeIndex = type;
            VkDeviceMemory memory = VK_NULL_HANDLE;
            return vkAllocateMemory(s_dev, &mai, nullptr, &memory) == VK_SUCCESS ? memory : VK_NULL_HANDLE;
        }

        // nfsmw-memory: makes a spare when one is wanted, and frees spares
        // beyond one once they have been empty kSpareKeep.
        void MemoryThread()
        {
            SetHostThreadName("nfsmw-memory");
            hostcpu::LeaveReservedCore();  // started by the command processor
            std::unique_lock lock(s_spareMutex);
            while (true)
            {
                auto now = std::chrono::steady_clock::now();
                auto next = now + std::chrono::hours(1);
                bool worked = false;
                for (uint32_t type = 0; type < VK_MAX_MEMORY_TYPES; type++)
                {
                    std::vector<Spare>& spares = s_spares[type];
                    if (spares.size() > 1 && now - spares.front().since >= kSpareKeep)
                    {
                        VkDeviceMemory memory = spares.front().memory;
                        spares.erase(spares.begin());
                        s_imageTotals.blocksFreed++;
                        lock.unlock();
                        vkFreeMemory(s_dev, memory, nullptr);
                        lock.lock();
                        worked = true;
                        break;
                    }
                    if (spares.size() > 1)
                        next = std::min(next, spares.front().since + kSpareKeep);
                    uint32_t bit = 1u << type;
                    if ((s_spareWanted & bit) && !(s_spareFailed & bit) && spares.empty())
                    {
                        lock.unlock();
                        auto start = std::chrono::steady_clock::now();
                        VkDeviceMemory memory = AllocateBlock(type);
                        uint64_t ns = SinceNs(start);
                        lock.lock();
                        s_spareWanted &= ~bit;
                        if (!memory)
                        {
                            // Left to the command processor, which tries again
                            // (and falls back to an allocation per image).
                            s_spareFailed |= bit;
                            fprintf(stderr, "[renderer] image memory: no spare block of memory type %u (%llu MB)\n", type,
                                (unsigned long long)(s_blockSize >> 20));
                        }
                        else
                        {
                            s_imageTotals.spareBlocks++;
                            s_imageTotals.spareNs += ns;
                            s_spares[type].push_back({ memory, std::chrono::steady_clock::now() });
                        }
                        worked = true;
                        break;
                    }
                    if (s_spareWanted & bit)
                        s_spareWanted &= ~bit;  // (a spare came back meanwhile, or the thread can't make one)
                }
                if (!worked)
                    s_spareCv.wait_until(lock, next);
            }
        }

        // A block of `type` for a new image: a spare (the newest; the thread
        // makes the next when it was the last), or one allocated here.
        ImageBlock* NewBlock(uint32_t type)
        {
            VkDeviceMemory memory = VK_NULL_HANDLE;
            {
                std::lock_guard lock(s_spareMutex);
                std::vector<Spare>& spares = s_spares[type];
                if (!spares.empty())
                {
                    memory = spares.back().memory;
                    spares.pop_back();
                }
                if (spares.empty())
                    s_spareWanted |= 1u << type;
            }
            s_spareCv.notify_one();
            if (memory)
                s_imageTotals.fromSpare++;
            else
            {
                auto start = std::chrono::steady_clock::now();
                memory = AllocateBlock(type);
                s_imageTotals.allocations++;
                s_imageTotals.allocationNs += SinceNs(start);
                if (!memory)
                    return nullptr;
                s_imageTotals.blocksHere++;
                std::lock_guard lock(s_spareMutex);
                s_spareFailed &= ~(1u << type);
            }
            s_blocks.push_back(std::make_unique<ImageBlock>(ImageBlock{ memory, type, gpu::BlockRanges(s_blockSize) }));
            return s_blocks.back().get();
        }

        // Binds `img` into a block of `type`; false when no block can be had.
        bool BindToBlock(Image& img, const VkMemoryRequirements& req, uint32_t type)
        {
            ImageBlock* block = nullptr;
            VkDeviceSize offset = gpu::BlockRanges::kNone;
            for (auto& b : s_blocks)
                if (b->type == type && (offset = b->ranges.Allocate(req.size, req.alignment)) != gpu::BlockRanges::kNone)
                {
                    block = b.get();
                    break;
                }
            if (!block)
            {
                block = NewBlock(type);
                if (!block || (offset = block->ranges.Allocate(req.size, req.alignment)) == gpu::BlockRanges::kNone)
                    return false;
            }
            img.block = block;
            img.offset = offset;
            img.memory = block->memory;
            return true;
        }

        void DestroyImage(Image& img);

        // `imageNext`, `viewNext`: chained to the image's and its view's create infos.
        // `fallible`: an image or view the device turns down returns an empty
        // Image (nothing kept) instead of aborting.
        Image CreateImage(VkImageType type, VkFormat format, VkExtent3D extent, uint32_t layers,
            VkImageUsageFlags usage, VkImageViewType viewType, VkImageAspectFlags aspect, VkImageCreateFlags flags = 0,
            uint32_t levels = 1, const void* imageNext = nullptr, const void* viewNext = nullptr, bool fallible = false)
        {
            HitchTimer hitchTimer(s_hitch.imageMs);
            s_hitch.images++;
            auto start = std::chrono::steady_clock::now();
            Image img;
            VkImageCreateInfo ici{ VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
            ici.pNext = imageNext;
            ici.flags = flags;
            ici.imageType = type;
            ici.format = format;
            ici.extent = extent;
            ici.mipLevels = levels;
            ici.arrayLayers = layers;
            ici.samples = VK_SAMPLE_COUNT_1_BIT;
            ici.tiling = VK_IMAGE_TILING_OPTIMAL;  // (image memory blocks hold nothing linear)
            ici.usage = usage;
            ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
            VkResult result = vkCreateImage(s_dev, &ici, nullptr, &img.image);
            if (fallible && result != VK_SUCCESS)
                return {};
            Check(result, "vkCreateImage");
            VkMemoryDedicatedRequirements dedicated{ VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS };
            VkMemoryRequirements2 req2{ VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedicated };
            if (s_imagePool)
            {
                VkImageMemoryRequirementsInfo2 info{ VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, nullptr, img.image };
                vkGetImageMemoryRequirements2(s_dev, &info, &req2);
            }
            else
                vkGetImageMemoryRequirements(s_dev, img.image, &req2.memoryRequirements);
            const VkMemoryRequirements& req = req2.memoryRequirements;
            uint32_t memoryType = FindMemoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
            img.bytes = req.size;
            if (!s_imagePool || dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation ||
                req.size > s_blockSize / 2 || !BindToBlock(img, req, memoryType))
            {
                VkMemoryDedicatedAllocateInfo dai{ VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, nullptr, img.image };
                VkMemoryAllocateInfo mai{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
                mai.pNext = dedicated.requiresDedicatedAllocation ? &dai : nullptr;
                mai.allocationSize = req.size;
                mai.memoryTypeIndex = memoryType;
                auto allocationStart = std::chrono::steady_clock::now();
                Check(vkAllocateMemory(s_dev, &mai, nullptr, &img.memory), "vkAllocateMemory(image)");
                s_imageTotals.allocations++;
                s_imageTotals.allocationNs += SinceNs(allocationStart);
            }
            Check(vkBindImageMemory(s_dev, img.image, img.memory, img.offset), "vkBindImageMemory");
            VkImageViewCreateInfo vci{ VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO };
            vci.pNext = viewNext;
            vci.image = img.image;
            vci.viewType = viewType;
            vci.format = format;
            vci.subresourceRange = { aspect, 0, levels, 0, layers };
            result = vkCreateImageView(s_dev, &vci, nullptr, &img.view);
            if (fallible && result != VK_SUCCESS)
            {
                img.view = VK_NULL_HANDLE;
                DestroyImage(img);
                return {};
            }
            Check(result, "vkCreateImageView");
            s_imageTotals.images++;
            s_imageTotals.imageNs += SinceNs(start);
            return img;
        }

        // Nothing the GPU may still run can use it (see ApplyLiveSettings):
        // its memory, or its range of a block, is free for reuse at once.
        void DestroyImage(Image& img)
        {
            if (img.view)
                vkDestroyImageView(s_dev, img.view, nullptr);
            if (img.image)
                vkDestroyImage(s_dev, img.image, nullptr);
            if (ImageBlock* block = img.block)
            {
                if (!block->ranges.Free(img.offset, img.bytes))
                {
                    fprintf(stderr, "[renderer] image memory: %llu bytes at %llu aren't in use in their block\n",
                        (unsigned long long)img.bytes, (unsigned long long)img.offset);
                    abort();
                }
                if (block->ranges.Empty())
                {
                    // A spare now: the memory thread frees it if it stays
                    // unused beside another.
                    Spare spare{ block->memory, std::chrono::steady_clock::now() };
                    uint32_t type = block->type;
                    std::erase_if(s_blocks, [block](const std::unique_ptr<ImageBlock>& b) { return b.get() == block; });
                    {
                        std::lock_guard lock(s_spareMutex);
                        s_spares[type].push_back(spare);
                    }
                    s_spareCv.notify_one();
                }
            }
            else if (img.memory)
                vkFreeMemory(s_dev, img.memory, nullptr);
            img = {};
        }

        // ---------------------------------------------------------------
        // Command recording

        // Submissions. Work is recorded into the current slot's command
        // buffer and submitted without waiting; CPU-visible effects of the
        // command stream (fences, the read pointer) are attached to the
        // submission they follow and applied by the completion thread once
        // the GPU is done, as the real GPU would write them.
        // NFSMW_OCCLUSION=1 (ZpassDone): a counting segment is one precise
        // occlusion query, begun before a draw while a query bracket is open
        // and ended at its pass's end, at any ZPD or when its weight changes;
        // an END report is stored once the submission holding it completes,
        // with the samples of segments [s0, s1).
        struct ZpdSegment { uint64_t id; uint32_t query, num, den; };
        struct ZpdEnd { uint32_t address; uint64_t s0, s1, frame; std::chrono::steady_clock::time_point queued; };
        struct Submission
        {
            VkCommandBuffer cmd = VK_NULL_HANDLE;
            VkFence fence = VK_NULL_HANDLE;
            bool inFlight = false;
            // A write whose `event` is set runs as soon as the GPU passes that
            // point of the command buffer (vkCmdSetEvent after the work it
            // follows); the rest run when the whole submission completes.
            struct Write { std::function<void()> fn; VkEvent event = VK_NULL_HANDLE; std::chrono::steady_clock::time_point queued = std::chrono::steady_clock::now(); };
            std::vector<Write> writes;
            std::vector<VkEvent> events;  // returned to the pool on completion
            uint64_t number = 0;           // s_submissionCounter when submitted (write-watch guard)
            size_t unsignaled = 0;         // first write not yet covered by an event
            std::chrono::steady_clock::time_point submitted;
            // NFSMW_OCCLUSION=1 (else always empty): the segments recorded
            // here, harvested at completion, then the END reports, in order.
            std::vector<ZpdSegment> zpdSegments;
            std::vector<ZpdEnd> zpdEnds;
        };
        // NFSMW_FENCE_EVENTS=0 disables mid-buffer fence publication.
        const bool s_fenceEvents = [] { const char* v = std::getenv("NFSMW_FENCE_EVENTS"); return !v || v[0] != '0'; }();
        std::vector<VkEvent> s_eventPool;  // under s_submitMutex
        // NFSMW_LOG_LATENCY: queue -> publish delay of deferred writes, by
        // how they were published (0 event, 1 completion). Under s_submitMutex.
        double s_writeDelayUs[2] = {};
        uint32_t s_writeCount[2] = {};
        void RunWrite(Submission::Write& write, int how)
        {
            s_writeDelayUs[how] += std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - write.queued).count();
            s_writeCount[how]++;
            if (how)
                timeline::Mark(timeline::WriteAtComplete);
            write.fn();
        }
        // GPU busy time: sum over submissions of the time from max(submit,
        // previous completion) to completion (completion-thread wake-up).
        std::atomic<uint64_t> s_gpuBusyUs{ 0 };
        // True GPU execution time: timestamps at each command buffer's start
        // and end (2 queries per slot).
        VkQueryPool s_timestamps = VK_NULL_HANDLE;  // NFSMW_GPU_TIMING=1 (each costs a blit encoder on MoltenVK)
        double s_timestampPeriod = 1.0;

        // NFSMW_PASS_PROFILE=<frame>: time every render pass of one frame and
        // print them (targets, area, draws, GPU us). 256 passes max.
        const int s_profileFrame = [] { const char* v = std::getenv("NFSMW_PASS_PROFILE"); return v ? std::atoi(v) : -1; }();
        VkQueryPool s_passQueries = VK_NULL_HANDLE;
        // Per profiled pass, also what was recorded in the gap before it:
        // resolves, texture uploads, submissions, guest memory synced to the
        // GPU copy (KB).
        struct PassInfo { std::string desc; uint32_t draws = 0; uint32_t slot, query; uint32_t resolves = 0, uploads = 0, submits = 0; uint64_t shadowKB = 0; };
        struct GapCounters { uint64_t resolves = 0, uploads = 0, submits = 0, shadowBytes = 0; } s_gapLast;
        std::vector<PassInfo> s_passInfos;
        bool s_profiling = false;
        uint32_t s_passQueryNext = 0;
        std::atomic<uint64_t> s_gpuExecNs{ 0 };
        std::chrono::steady_clock::time_point s_lastCompletion;
        VkCommandPool s_commandPool = VK_NULL_HANDLE;
        Submission s_submissions[kSlots];
        uint32_t s_slot = 0;
        uint64_t s_submissionCounter = 1;       // bumps on every submit
        VkDeviceSize s_lastConstantsOffset = 0;
        uint64_t s_lastConstantsSubmission = 0; // submission holding s_lastConstantsOffset
        const void* s_lastConstLayoutVs = nullptr;  // the layout of that upload (see Draw)
        const void* s_lastConstLayoutPs = nullptr;
        VkCommandBuffer s_cmd = VK_NULL_HANDLE;
        bool s_recording = false;
        std::chrono::steady_clock::time_point s_recordStart;
        std::mutex s_submitMutex;
        std::condition_variable s_submitCv;
        std::deque<uint32_t> s_inFlight;
        // NFSMW_LOG_LATENCY: time with nothing in flight (under s_submitMutex).
        std::chrono::steady_clock::time_point s_emptySince = std::chrono::steady_clock::now();
        double s_emptyUs = 0;

        // Occlusion queries counted on the GPU (NFSMW_OCCLUSION=1; the
        // presenter decides: VulkanContext::occlusionCounting). Command
        // processor thread, but for the results ring (completion thread) and
        // the store statistics (under s_submitMutex). See ZpassDone.
        bool s_zpdOn = false;                  // constant after Initialize
        constexpr uint32_t kZpdQueries = 256;  // per slot
        VkQueryPool s_zpdPool[kSlots] = {};
        uint32_t s_zpdUsed[kSlots] = {};       // used by the slot's recording (reset at its next Begin)
        int32_t s_zpdQuery = -1;               // the active segment's query in s_zpdPool[s_slot], or -1
        uint32_t s_zpdNum = 0, s_zpdDen = 0;   // its weight (guest samples per host sample, num / den)
        uint64_t s_zpdSegNext = 1;             // segment ids
        zpd::BracketBook s_zpdOpen;
        std::unique_ptr<zpd::ResultRing> s_zpdResults;
        // The [zpd] line's counts: the command processor's, and the stores'
        // (under s_submitMutex; the delays in ms, CP -> store).
        struct ZpdCounts { uint64_t zpds, begins, ends, segments, draws, unmatched, dropped, replaced, unknown, overflow, clamped, unclamped; };
        ZpdCounts s_zpdCounts{};
        struct ZpdStores { uint64_t stored, nonZero, lost; double delaySum, delayMax; };
        ZpdStores s_zpdStores{};
        std::atomic<uint64_t> s_zpdUnavailable{ 0 };
        // The latest END count per report address (count, frame), for
        // captures (LogOcclusionCounts, any thread).
        std::mutex s_zpdLatestMutex;
        std::map<uint32_t, std::pair<uint32_t, uint64_t>> s_zpdLatest;
        // NFSMW_LOG_ZPD=<lines>: each BEGIN, counted draw, segment and END,
        // that many lines at most (any thread).
        const int64_t s_zpdLogLines = [] { const char* v = std::getenv("NFSMW_LOG_ZPD"); return v ? int64_t(std::atoll(v)) : int64_t(0); }();
        std::atomic<int64_t> s_zpdLogged{ 0 };
        __attribute__((format(printf, 1, 2))) void ZpdLog(const char* fmt, ...)
        {
            if (s_zpdLogged.load(std::memory_order_relaxed) >= s_zpdLogLines || s_zpdLogged.fetch_add(1) >= s_zpdLogLines)
                return;
            char line[512];
            va_list args;
            va_start(args, fmt);
            vsnprintf(line, sizeof(line), fmt, args);
            va_end(args);
            fprintf(stderr, "[zpd] %s\n", line);
        }
        bool ZpdLogging() { return s_zpdLogged.load(std::memory_order_relaxed) < s_zpdLogLines; }

        // Ends the active segment. Inside its pass: one is begun only right
        // before a draw, and every pass end ends it first (EndPass).
        void ZpdEndSegment()
        {
            if (s_zpdQuery < 0)
                return;
            vkCmdEndQuery(s_cmd, s_zpdPool[s_slot], uint32_t(s_zpdQuery));
            s_zpdQuery = -1;
        }

        void EndPass();
        // Bumped when the command buffer's graphics state is unknown (a new
        // command buffer; compute work, which shares the push constants).
        uint64_t s_stateEpoch = 1;

        // Write-watch GPU guard for the submission being recorded, once per
        // range and submission (the same vertex buffer serves many draws).
        // The ranges guarded so far are a direct-mapped table stamped with
        // the submission: no clearing, no allocation (a range that a
        // collision evicted is guarded again, which changes nothing).
        // NFSMW_WATCH_RANGES=0 (write_watch.cpp): a hash set, as before.
        const bool s_guardTable = [] { const char* v = std::getenv("NFSMW_WATCH_RANGES"); return !v || v[0] != '0'; }();
        struct GuardedRange { uint64_t key, submission; };
        std::unique_ptr<GuardedRange[]> s_guardedRanges;  // 4096
        std::unordered_set<uint64_t> s_guarded;
        uint64_t s_guardedSubmission = 0;
        void GuardGpu(uint32_t base, uint32_t size, bool write, const char* tag, bool untile = false)
        {
            if ((!g_eagerWrites && !s_shadowMode) || size == 0)
                return;
            // (A texture's untile is keyed apart from a plain read of the same
            // range, or it would lose its flag: see writewatch::GpuRead.)
            uint64_t key = uint64_t(base) << 32 | (size ^ (write ? 0x80000000u : untile ? 0x40000000u : 0));
            if (s_guardTable)
            {
                if (!s_guardedRanges)
                    s_guardedRanges.reset(new GuardedRange[4096]());
                GuardedRange& slot = s_guardedRanges[(key * 0x9E3779B97F4A7C15ull) >> 52];
                if (slot.key == key && slot.submission == s_submissionCounter)
                    return;
                slot = { key, s_submissionCounter };
            }
            else
            {
                if (s_guardedSubmission != s_submissionCounter)
                {
                    s_guarded.clear();
                    s_guardedSubmission = s_submissionCounter;
                }
                if (!s_guarded.insert(key).second)
                    return;
            }
            if (write)
                writewatch::GpuWrite(base, size, s_submissionCounter, tag);
            else
                writewatch::GpuRead(base, size, s_submissionCounter, tag, untile);
        }

        // Outside a render pass: an event after everything recorded so far,
        // covering the current submission's writes that have none yet.
        void SignalWrites()
        {
            if (!s_fenceEvents || !s_recording)
                return;
            Submission& sub = s_submissions[s_slot];
            VkEvent event;
            {
                std::lock_guard lock(s_submitMutex);
                if (sub.unsignaled == sub.writes.size())
                    return;
                if (s_eventPool.empty())
                {
                    VkEventCreateInfo eci{ VK_STRUCTURE_TYPE_EVENT_CREATE_INFO };
                    Check(vkCreateEvent(s_dev, &eci, nullptr, &event), "vkCreateEvent");
                }
                else
                {
                    event = s_eventPool.back();
                    s_eventPool.pop_back();
                }
                for (size_t i = sub.unsignaled; i < sub.writes.size(); i++)
                    sub.writes[i].event = event;
                sub.unsignaled = sub.writes.size();
                sub.events.push_back(event);
            }
            vkCmdSetEvent(s_cmd, event, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
        }

        void Begin()
        {
            if (s_recording)
                return;
            Submission& sub = s_submissions[s_slot];
            {
                CpWaitTimer wait(CP_WAIT_SLOT);
                std::unique_lock lock(s_submitMutex);
                s_submitCv.wait(lock, [&] { return !sub.inFlight; });
            }
            vkResetFences(s_dev, 1, &sub.fence);
            vkResetCommandBuffer(sub.cmd, 0);
            s_cmd = sub.cmd;
            VkCommandBufferBeginInfo bi{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
            bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
            Check(vkBeginCommandBuffer(s_cmd, &bi), "vkBeginCommandBuffer");
            s_recording = true;
            s_stateEpoch++;
            s_recordStart = std::chrono::steady_clock::now();
            if (s_timestamps)
            {
                vkCmdResetQueryPool(s_cmd, s_timestamps, s_slot * 2, 2);
                vkCmdWriteTimestamp(s_cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, s_timestamps, s_slot * 2);
            }
            // Occlusion queries: the slot's last submission has retired, its
            // results harvested; reset the queries it used (a host reset is a
            // memset on MoltenVK; else a command, outside any pass here: for
            // devices before Vulkan 1.2, which requires host reset, so not
            // exercised on MoltenVK, RADV or Turnip).
            if (s_zpdOn && s_zpdUsed[s_slot])
            {
                if (s_vk->resetQueryPool)
                    s_vk->resetQueryPool(s_dev, s_zpdPool[s_slot], 0, s_zpdUsed[s_slot]);
                else
                    vkCmdResetQueryPool(s_cmd, s_zpdPool[s_slot], 0, s_zpdUsed[s_slot]);
                s_zpdUsed[s_slot] = 0;
            }
            s_ringOffset = 0;
        }

        // Submits per call site (NFSMW_LOG_LATENCY): 1 ring full, 2 Submit, 3 SubmitIfIdle,
        // 4 SubmitIfDue, 5 Flush, 6 QueueWrite on an idle GPU, 7 present, 8 after a draw
        // to a starving GPU, 9 after a resolve or upload to one (SubmitIfStarving).
        uint32_t s_submitWhy[10] = {};
        const uint32_t s_logTextureUse = [] { const char* v = std::getenv("NFSMW_LOG_TEXTURE_USE"); return v ? uint32_t(strtoul(v, nullptr, 16)) : 0u; }();
        bool s_drawUsesLoggedTexture = false;
        bool s_drawScaled = false;  // the current draw renders into scaled targets (internal resolution)
        bool s_logDrawsNow = false;
        std::string s_loggedTextureFetch;
        uint32_t s_drawsSinceCheck = 0;
        bool s_submitAfterDraw = false;  // mid-batch submit to a starving GPU (eager writes)
        double s_queueLockUs = 0, s_queueSubmitUs = 0;  // waiting for the queue lock, in vkQueueSubmit

        void SubmitRecorded(int why = 0)
        {
            if (!s_recording)
                return;
            s_submitWhy[why]++;
            timeline::Mark(timeline::SubmitBegin, why);
            EndPass();
            if (s_timestamps)
                vkCmdWriteTimestamp(s_cmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, s_timestamps, s_slot * 2 + 1);
            Check(vkEndCommandBuffer(s_cmd), "vkEndCommandBuffer");
            Submission& sub = s_submissions[s_slot];
            sub.number = s_submissionCounter;
            VkSubmitInfo si{ VK_STRUCTURE_TYPE_SUBMIT_INFO };
            si.commandBufferCount = 1;
            si.pCommandBuffers = &s_cmd;
            {
                auto t0 = std::chrono::steady_clock::now();
                std::unique_lock lock(*s_vk->queueMutex);
                auto tl = std::chrono::steady_clock::now();
                // The presenter's gate holds GPU work while the app is
                // inactive (iOS). It closes only after guesttime::Suspend()
                // and opens before Resume(), so a hold here is always the
                // suspension's time, not this submit's or the queue lock's.
                if (guesttime::Suspended())
                {
                    CpWaitTimer held(CP_WAIT_SUSPENDED);
                    video::WaitForeground(lock);
                }
                else
                    video::WaitForeground(lock);
                auto t1 = std::chrono::steady_clock::now();
                Check(vkQueueSubmit(s_vk->queue, 1, &si, sub.fence), "vkQueueSubmit");
                auto t2 = std::chrono::steady_clock::now();
                s_queueLockUs += std::chrono::duration<double, std::micro>(tl - t0).count();
                s_queueSubmitUs += std::chrono::duration<double, std::micro>(t2 - t1).count();
                s_hitch.submitMs += std::chrono::duration<double, std::milli>((t2 - t0) - (t1 - tl)).count();
            }
            {
                std::lock_guard lock(s_submitMutex);
                sub.inFlight = true;
                sub.submitted = std::chrono::steady_clock::now();
                if (s_inFlight.empty())
                    s_emptyUs += std::chrono::duration<double, std::micro>(sub.submitted - s_emptySince).count();
                s_inFlight.push_back(s_slot);
                timeline::Mark(timeline::Submit, s_slot | std::min<uint32_t>(uint32_t(sub.writes.size()), 0xFFFF) << 8 | uint32_t(why) << 24);
            }
            s_submitCv.notify_all();
            s_slot = (s_slot + 1) % kSlots;
            s_recording = false;
            s_submissionCounter++;
            s_stats.submits++;
            writewatch::Rearm();
        }

        // Completion thread, `sub` (in `slot`) done: its segments' counts into
        // the ring, in guest samples. Never waits: the fence has signalled
        // (MoltenVK makes queries available before that), and WAIT_BIT could
        // hang elsewhere on a query never ended; one not available counts 0.
        void ZpdHarvest(uint32_t slot, const Submission& sub)
        {
            static uint64_t results[kZpdQueries * 2];  // value, availability
            uint32_t n = 0;
            for (const ZpdSegment& s : sub.zpdSegments)
                n = std::max(n, s.query + 1);
            memset(results, 0, size_t(n) * 16);
            VkResult r = vkGetQueryPoolResults(s_dev, s_zpdPool[slot], 0, n, size_t(n) * 16, results, 16,
                VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
            const bool read = r == VK_SUCCESS || r == VK_NOT_READY;
            for (const ZpdSegment& s : sub.zpdSegments)
            {
                const bool available = read && results[s.query * 2 + 1] != 0;
                uint64_t raw = available ? results[s.query * 2] : 0;
                if (!available)
                    s_zpdUnavailable++;
                uint64_t guest = zpd::Normalize(raw, s.num, s.den);
                s_zpdResults->Put(s.id, guest, raw);
                ZpdLog("segment %llu: %s%llu host samples x %u/%u = %llu", (unsigned long long)s.id, available ? "" : "not available, ",
                    (unsigned long long)raw, s.num, s.den, (unsigned long long)guest);
            }
        }

        // An END report from its segments' counts, under s_submitMutex (the
        // completion thread; the command processor when nothing is in
        // flight). Stored as the GPU's stores are, around the write watch's
        // guard (opened once for its 8 dwords): only the CPU reads these
        // bytes (GetData), never the GPU, so imported and shadow memory
        // alike, and nothing guards them (GetData polls and takes "not
        // ready").
        void ZpdStore(const ZpdEnd& end)
        {
            uint32_t lost = 0;
            uint64_t raw = 0;
            // A runaway count stays one the game rejects (zpd::EndCount).
            const uint32_t count = zpd::EndCount(s_zpdResults->Sum(end.s0, end.s1, lost, &raw));
            uint8_t* host = s_sharedHost + end.address;
            writewatch::GpuThreadWrite(host, 32, [host, count] {
                zpd::StoreEnd(count, [host](uint32_t offset, uint32_t value) { memcpy(host + offset, &value, 4); });
            });
            double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - end.queued).count();
            s_zpdStores.stored++;
            s_zpdStores.nonZero += count != 0;
            s_zpdStores.lost += lost;
            s_zpdStores.delaySum += ms;
            s_zpdStores.delayMax = std::max(s_zpdStores.delayMax, ms);
            {
                std::lock_guard lock(s_zpdLatestMutex);
                if (s_zpdLatest.size() >= 64 && !s_zpdLatest.contains(end.address))
                    s_zpdLatest.clear();
                s_zpdLatest[end.address] = { count, end.frame };
            }
            ZpdLog("f%llu END %08X: segments %llu..%llu, %llu host samples: %u stored %.2f ms after its ZPD%s",
                (unsigned long long)end.frame, end.address, (unsigned long long)end.s0, (unsigned long long)end.s1,
                (unsigned long long)raw, count, ms, lost ? std::format(" ({} segments lost)", lost).c_str() : "");
        }

        void CompletionThread()
        {
#ifdef __APPLE__
            pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0);  // see the command processor's Worker
#endif
            writewatch::t_gpuThread = true;
            SetHostThreadName("nfsmw-gpu-done");
            hostcpu::LeaveReservedCore();  // started by the command processor
            while (true)
            {
                uint32_t slot;
                {
                    std::unique_lock lock(s_submitMutex);
                    s_submitCv.wait(lock, [] { return !s_inFlight.empty(); });
                    slot = s_inFlight.front();
                }
                Submission& sub = s_submissions[slot];
                // Publish writes as the GPU passes their events; block on the
                // fence once none is left to watch.
                size_t done = 0;
                while (true)
                {
                    bool finished = vkGetFenceStatus(s_dev, sub.fence) == VK_SUCCESS;
                    bool watching = false;
                    {
                        std::lock_guard lock(s_submitMutex);
                        while (!finished && done < sub.writes.size() && sub.writes[done].event &&
                            vkGetEventStatus(s_dev, sub.writes[done].event) == VK_EVENT_SET)
                            { timeline::Mark(timeline::WriteByEvent); RunWrite(sub.writes[done++], 0); }
                        watching = done < sub.writes.size() && sub.writes[done].event;
                    }
                    if (finished)
                        break;
                    if (watching)
                        std::this_thread::sleep_for(std::chrono::microseconds(30));
                    else
                    {
                        // Nothing mid-buffer to watch: wait for the fence, but
                        // wake now and then in case a write arrives with an event.
                        VkResult r = vkWaitForFences(s_dev, 1, &sub.fence, VK_TRUE, 1'000'000);
                        if (r == VK_SUCCESS)
                            break;
                    }
                }
                auto now = std::chrono::steady_clock::now();
                {
                    // NFSMW_LOG_LATENCY=1: submit -> completion-wake latency.
                    static const bool logLatency = std::getenv("NFSMW_LOG_LATENCY") != nullptr;
                    static double sumUs = 0, maxUs = 0;
                    static uint32_t n = 0;
                    static auto window = now;
                    if (logLatency)
                    {
                        double us = std::chrono::duration<double, std::micro>(now - sub.submitted).count();
                        sumUs += us;
                        maxUs = std::max(maxUs, us);
                        n++;
                        if (now - window > std::chrono::seconds(1))
                        {
                            double empty;
                            char writes[160];
                            {
                                std::lock_guard lock(s_submitMutex);
                                empty = s_emptyUs;
                                s_emptyUs = 0;
                                snprintf(writes, sizeof(writes), " | writes: %u by event (avg %.0f us), %u at completion (avg %.0f us)",
                                    s_writeCount[0], s_writeCount[0] ? s_writeDelayUs[0] / s_writeCount[0] : 0.0,
                                    s_writeCount[1], s_writeCount[1] ? s_writeDelayUs[1] / s_writeCount[1] : 0.0);
                                s_writeDelayUs[0] = s_writeDelayUs[1] = 0;
                                s_writeCount[0] = s_writeCount[1] = 0;
                            }
                            double windowUs = std::chrono::duration<double, std::micro>(now - window).count();
                            fprintf(stderr, "[latency] %u submissions/s, submit->done avg %.0f us, max %.0f us, queue empty %.0f%% | by site ring %u submit %u ifidle %u ifdue %u flush %u write %u present %u draw %u copies %u%s | queue lock %.1f ms/s, vkQueueSubmit %.1f ms/s\n",
                                n, sumUs / n, maxUs, 100.0 * empty / windowUs, s_submitWhy[1], s_submitWhy[2], s_submitWhy[3],
                                s_submitWhy[4], s_submitWhy[5], s_submitWhy[6], s_submitWhy[7], s_submitWhy[8], s_submitWhy[9], writes,
                                s_queueLockUs / 1000, s_queueSubmitUs / 1000);
                            s_queueLockUs = s_queueSubmitUs = 0;
                            for (auto& w : s_submitWhy)
                                w = 0;
                            sumUs = maxUs = 0;
                            n = 0;
                            window = now;
                        }
                    }
                }
                auto start = std::max(sub.submitted, s_lastCompletion);
                if (now > start)
                    s_gpuBusyUs += uint64_t(std::chrono::duration_cast<std::chrono::microseconds>(now - start).count());
                s_lastCompletion = now;
                if (s_timestamps)
                {
                    uint64_t ts[2];
                    if (vkGetQueryPoolResults(s_dev, s_timestamps, slot * 2, 2, sizeof(ts), ts, sizeof(uint64_t),
                            VK_QUERY_RESULT_64_BIT) == VK_SUCCESS && ts[1] > ts[0])
                        s_gpuExecNs += uint64_t(double(ts[1] - ts[0]) * s_timestampPeriod);
                }
                if (!sub.zpdSegments.empty())
                    ZpdHarvest(slot, sub);
                {
                    // Writes run under the lock so QueueWrite can't append to
                    // a submission that is being retired.
                    std::lock_guard lock(s_submitMutex);
                    for (; done < sub.writes.size(); done++)
                        RunWrite(sub.writes[done], 1);
                    sub.writes.clear();
                    // Occlusion query reports, in stream order: a query's
                    // last tile, which makes it ready, is stored last.
                    for (const ZpdEnd& end : sub.zpdEnds)
                        ZpdStore(end);
                    sub.zpdEnds.clear();
                    sub.zpdSegments.clear();
                    sub.unsignaled = 0;
                    for (VkEvent event : sub.events)
                    {
                        vkResetEvent(s_dev, event);
                        s_eventPool.push_back(event);
                    }
                    sub.events.clear();
                    writewatch::SetCompleted(sub.number);
                    s_inFlight.pop_front();
                    timeline::Mark(timeline::Complete, slot);
                    if (s_inFlight.empty())
                        s_emptySince = std::chrono::steady_clock::now();
                    sub.inFlight = false;
                }
                s_submitCv.notify_all();
            }
        }

        // Until submission `number` completed (submitting it if it is still
        // being recorded).
        void WaitForSubmission(uint64_t number)
        {
            s_stats.shadowWaits++;
            if (number >= s_submissionCounter)
                SubmitRecorded(9);
            CpWaitTimer wait(CP_WAIT_SHADOW);
            std::unique_lock lock(s_submitMutex);
            s_submitCv.wait(lock, [&] { return s_inFlight.empty() || s_submissions[s_inFlight.front()].number > number; });
        }

        // Shadow mode: bring the GPU's copy of [base, base + size) up to date
        // before recording work that reads it.
        void EnsureShadow(uint32_t base, uint32_t size)
        {
            if (!s_shadowMode || size == 0)
                return;
            base &= 0x1FFFFFFF;
            size = std::min<uint32_t>(size, uint32_t(kSharedSize) - base);
            uint64_t key = uint64_t(base) << 32 | size;
            auto it = s_shadowSynced.find(key);
            if (it != s_shadowSynced.end() && !writewatch::RangeWrittenSince(base, size, it->second))
                return;
            uint64_t sequence = writewatch::Current();
            uint64_t since = it != s_shadowSynced.end() ? it->second : 0;
            HitchTimer hitchTimer(s_hitch.shadowMs);
            while (uint64_t busy = writewatch::SyncShadow(base, size, since))
                WaitForSubmission(busy);
            if (s_shadowSynced.size() > 65536)
                s_shadowSynced.clear();
            s_shadowSynced[key] = sequence;
        }

        // NFSMW_PASS_BARRIER=0|1|2|3 (default 2 with kLighterPasses, else 0): narrower
        // barriers between passes. On MoltenVK a barrier outside a pass is a
        // Metal fence wait per stage (vertex, fragment, compute, copy) of the
        // encoders after it on those before it; a full one makes the next
        // pass's vertex stage wait for everything before, so no pass's vertex
        // work overlaps the last pass's fragments or the resolve between them.
        // 1: a pass ends with its attachment writes (and its shaders' reads)
        // before fragment, compute and transfer work only: no vertex shader
        // reads an attachment (targets reach draws only through resolves).
        // Every pass is followed by a resolve now, so alone it overlaps
        // nothing (a control). 2: a resolve's writes, too, go before those
        // stages only, and a draw whose vertex shader may read what one wrote
        // (vertex or index data in its range, the cached texture it stored
        // into) gets a full barrier first (VertexBarrierIfNeeded). Vertex
        // stages then run ahead through a chain of passes and resolves (the
        // cube faces'). 3: 2 with full pass ends: a pass's vertex stage
        // overlaps only the resolve before it. Texture loads keep full
        // barriers (rare since the write watch's subpages; the ring they
        // stage through is a Metal-tracked index buffer anyway).
        // An M4 Max (race, alternating windows): at 2 and 3 all but one
        // post-resolve vertex stage start early, and the renderer's GPU time
        // drops 0.22-0.34 ms (12-16%); 1 changes nothing. For the iPad to
        // decide.
        // NFSMW_PASS_BARRIER_ALT=1 (A/B): that level in every other 120-frame
        // [perf] window, off in the rest: one run, the same content in both
        // arms. Each window's [barrier] line names the level it ran at.
        // Default 2 with kLighterPasses (Apple: iPad Pro M2 GPU-bound at 2x,
        // same-run windows: executing -0.5 ms), else 0.
        const uint32_t s_passBarrierLevel = [] { const char* v = std::getenv("NFSMW_PASS_BARRIER"); return v ? uint32_t(std::clamp(std::atoi(v), 0, 3)) : kLighterPasses ? 2u : 0u; }();
        const bool s_passBarrierAlt = [] { const char* v = std::getenv("NFSMW_PASS_BARRIER_ALT"); return v && v[0] == '1'; }() && s_passBarrierLevel;
        uint32_t s_passBarrier = s_passBarrierAlt ? 0u : s_passBarrierLevel;
        // Levels 2-3: guest memory resolves wrote since the last full barrier,
        // sorted and merged [base, end) (no vertex stage waits for them yet),
        // their hull for the common miss, and the cached textures (by guest
        // base) they stored into.
        std::vector<std::pair<uint32_t, uint32_t>> s_vertexPending;
        uint32_t s_vertexPendingLo = UINT32_MAX, s_vertexPendingHi = 0;
        std::vector<uint32_t> s_vertexPendingTextures;
        // Pass ends narrowed since the last full barrier. MoltenVK names each
        // stage's fences from a ring of 63 (MVKCommandEncoder::setBarrier):
        // a pass's vertex wait left on an old fragment fence must never come
        // round to the one that pass updates itself, so every 48th is full.
        uint32_t s_narrowPassEnds = 0;
        // The [barrier] line (OnSwap): narrowed pass ends and resolves, full
        // ones forced by the cap, vertex barriers (by reason: vertex data,
        // indices, a texture) and those that ended an open pass.
        struct BarrierCounts { uint64_t passEnds, resolves, capped, vertex[3], splits; };
        BarrierCounts s_barrierCounts{};

        // Everything before -> everything after. Outside rendering only.
        void FullBarrier()
        {
            VkMemoryBarrier mb{ VK_STRUCTURE_TYPE_MEMORY_BARRIER };
            mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
            mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
            vkCmdPipelineBarrier(s_cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
                0, 1, &mb, 0, nullptr, 0, nullptr);
            s_vertexPending.clear();
            s_vertexPendingLo = UINT32_MAX;
            s_vertexPendingHi = 0;
            s_vertexPendingTextures.clear();
            s_narrowPassEnds = 0;
        }

        // `src` stages' work (and, in Vulkan, everything logically earlier)
        // -> every stage after but the vertex ones: on MoltenVK nothing waits
        // before a vertex stage. Outside rendering only.
        void NarrowBarrier(VkPipelineStageFlags src, VkAccessFlags srcAccess)
        {
            VkMemoryBarrier mb{ VK_STRUCTURE_TYPE_MEMORY_BARRIER };
            mb.srcAccessMask = srcAccess;
            mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
            vkCmdPipelineBarrier(s_cmd, src,
                VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
                    VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT |
                    VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT,
                0, 1, &mb, 0, nullptr, 0, nullptr);
        }

        // At a pass's end. Its attachment writes go before what reads or
        // writes those images next (passes' loads, clears and stores;
        // resolves), its shaders' reads before compute and transfer writes
        // (resolves, texture loads). The vertex stage is named in the source
        // although Vulkan implies it: MoltenVK maps stages one to one, and
        // without it a resolve overwriting what this pass's vertex shader
        // read would wait only for its fragment stage.
        void PassEndBarrier()
        {
            if (s_passBarrier < 1 || s_passBarrier == 3)
                return FullBarrier();
            if (s_narrowPassEnds >= 48)
            {
                s_barrierCounts.capped++;
                return FullBarrier();
            }
            s_narrowPassEnds++;
            s_barrierCounts.passEnds++;
            NarrowBarrier(VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT |
                    VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
                VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT);
        }

        // After a 1x resolve's dispatch, which wrote guest memory [base, base
        // + size) and, unless `texture` is 0, the texels of the cached texture
        // at that guest base. Levels 2-3 leave the vertex stages out and keep
        // both for VertexBarrierIfNeeded until a full barrier (the front
        // buffer's, once a frame, at least).
        void ResolveBarrier(uint32_t base, uint32_t size, uint32_t texture)
        {
            if (s_passBarrier < 2)
                return FullBarrier();
            s_barrierCounts.resolves++;
            NarrowBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT);
            if (texture && std::find(s_vertexPendingTextures.begin(), s_vertexPendingTextures.end(), texture) == s_vertexPendingTextures.end())
                s_vertexPendingTextures.push_back(texture);
            if (!size)
                return;
            uint32_t end = base + size;
            s_vertexPendingLo = std::min(s_vertexPendingLo, base);
            s_vertexPendingHi = std::max(s_vertexPendingHi, end);
            // Into the sorted list, merged with what it overlaps or touches.
            auto it = std::lower_bound(s_vertexPending.begin(), s_vertexPending.end(), base,
                [](const std::pair<uint32_t, uint32_t>& r, uint32_t b) { return r.second < b; });
            auto last = it;
            while (last != s_vertexPending.end() && last->first <= end)
            {
                base = std::min(base, last->first);
                end = std::max(end, last->second);
                ++last;
            }
            it = s_vertexPending.erase(it, last);
            s_vertexPending.insert(it, { base, end });
        }

        // As FullBarrier, when the only work since the last one is a compute
        // dispatch: its shader writes before everything after.
        void ComputeBarrier()
        {
            VkMemoryBarrier mb{ VK_STRUCTURE_TYPE_MEMORY_BARRIER };
            mb.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
            mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
            vkCmdPipelineBarrier(s_cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
                0, 1, &mb, 0, nullptr, 0, nullptr);
        }

        void ToGeneral(VkImage image, VkImageAspectFlags aspect, uint32_t layers)
        {
            VkImageMemoryBarrier b{ VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER };
            b.srcAccessMask = 0;
            b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
            b.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
            b.newLayout = VK_IMAGE_LAYOUT_GENERAL;
            b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
            b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
            b.image = image;
            b.subresourceRange = { aspect, 0, VK_REMAINING_MIP_LEVELS, 0, layers };
            vkCmdPipelineBarrier(s_cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
                0, 0, nullptr, 0, nullptr, 1, &b);
        }

        // After a resolve or a texture upload, as every 128 draws: submit to a
        // starving GPU (fewer than two submissions in flight) a recording at
        // least NFSMW_STARVE_SUBMIT_US old. Around the shadow maps and the
        // mirror there are few draws but long GPU work, and on the Steam Frame
        // the GPU sat idle beside them. NFSMW_STARVE_AT_COPIES=0 turns it off.
        void SubmitIfStarving()
        {
            static const bool on = [] { const char* v = std::getenv("NFSMW_STARVE_AT_COPIES"); return !v || v[0] != '0'; }();
            static const int64_t starveUs = [] { const char* v = std::getenv("NFSMW_STARVE_SUBMIT_US"); return v ? int64_t(std::atoll(v)) : 1000; }();
            if (!on || !g_eagerWrites || !s_recording)  // (called between passes)
                return;
            if (std::chrono::steady_clock::now() - s_recordStart < std::chrono::microseconds(starveUs))
                return;
            {
                std::lock_guard lock(s_submitMutex);
                if (s_inFlight.size() >= 2)
                    return;
            }
            SubmitRecorded(9);
        }

        // Space in the current submission's ring slot (may submit and move
        // on to the next slot). Returns ~0 if `size` can never fit.
        VkDeviceSize RingAllocate(VkDeviceSize size)
        {
            if (size > kSlotSize)
                return ~VkDeviceSize(0);
            Begin();
            VkDeviceSize offset = (s_ringOffset + 255) & ~VkDeviceSize(255);
            if (offset + size > kSlotSize)
            {
                SubmitRecorded(1);
                Begin();
                offset = 0;
            }
            s_ringOffset = offset + size;
            return VkDeviceSize(s_slot) * kSlotSize + offset;
        }

        // ---------------------------------------------------------------
        // Shaders

        // A translated shader. Its SPIR-V is compiled when a pipeline first
        // needs it, on a pipeline worker (compileMutex guards the modules).
        struct Shader
        {
            ShaderStage stage;
            uint64_t hash;
            TranslatedShader info;         // immutable once loaded
            std::mutex compileMutex;
            bool vertexCompiled = false;
            VkShaderModule module = VK_NULL_HANDLE;  // VS
            // PS variants by (colour outputs, interpolator inputs, alpha test).
            std::unordered_map<uint32_t, VkShaderModule> variants;
        };
        std::unordered_map<uint64_t, std::unique_ptr<Shader>> s_shaders;
        Shader* s_vs = nullptr;
        Shader* s_ps = nullptr;

        // Depth-only draws run no pixel shader on Xenos (depth_only.h): they
        // get this empty one instead of the guest's bound one, left over from
        // earlier draws. NFSMW_DEPTH_ONLY_PS=1 runs the bound one, as up to
        // v0.1.2 (A/B only).
        constexpr uint64_t kDepthOnlyPsHash = 0xDE97400000000001ull;  // (a guest shader hashing to it: a 2^-64 chance)
        const bool s_depthOnlyGuestPs = [] { const char* v = std::getenv("NFSMW_DEPTH_ONLY_PS"); return v && v[0] == '1'; }();

        Shader* DepthOnlyPixelShader()
        {
            static Shader* const ps = [] {
                auto& slot = s_shaders[kDepthOnlyPsHash];
                slot = std::make_unique<Shader>();
                slot->stage = ShaderStage::Pixel;
                slot->hash = kDepthOnlyPsHash;
                slot->info.ok = true;
                slot->info.constPacked = true;  // (reads no constants: none uploaded for it)
                slot->info.glsl = gpu::kEmptyPixelShaderGlsl;
                return slot.get();
            }();
            return ps;
        }

        // IM_LOAD shaders by where they are (LoadShader). A dense frame loads
        // thousands, the same few hundred from the same addresses, and
        // finding each by a hash of all its dwords (FNV-1a, a latency-bound
        // multiply chain) was 2-2.7% of the command processor (Deck
        // profile). An entry keeps the words it was made from and is used
        // only while memory holds exactly those words: a compare, many times
        // cheaper than the hash, and the shader the hash would find.
        // NFSMW_SHADER_CACHE=0 turns it off; NFSMW_CP_CACHE_VERIFY=1 also
        // hashes on every hit and logs any difference.
        struct ShaderCacheEntry
        {
            const uint32_t* words = nullptr;  // where they were loaded from (the key)
            uint32_t dwords = 0, type = 0;
            Shader* shader = nullptr;
            std::vector<uint32_t> copy;  // the words, as loaded (guest order)
        };
        struct ShaderCache
        {
            ShaderCacheEntry entries[1024];
            bool on = false, verify = false;
            uint64_t loads = 0, hits = 0, comparedBytes = 0, mismatches = 0;
        };
        ShaderCache* const s_shaderCache = [] {
            auto* c = new ShaderCache();
            const char* v = std::getenv("NFSMW_SHADER_CACHE");
            c->on = (g_cpOpt & CP_OPT_SHADER_CACHE) && !(v && v[0] == '0');
            c->verify = std::getenv("NFSMW_CP_CACHE_VERIFY") != nullptr;
            return c;
        }();

        // Compiled shaders are cached on disk (GetCachePath()/spirv), keyed by
        // their exact source: new shaders compile once (10-20 ms each on the
        // Steam Machine, a hitch whenever new content appears), then load.
        // NFSMW_SHADER_CACHE=0 disables it.
        const bool s_shaderCacheEnabled = [] { const char* v = std::getenv("NFSMW_SHADER_CACHE"); return !v || v[0] != '0'; }();

        std::filesystem::path ShaderCacheFile(const std::string& source, glslang_stage_t stage, const std::string& preamble)
        {
            uint64_t h = 0xCBF29CE484222325ull;
            auto mix = [&](const void* data, size_t size) {
                for (size_t i = 0; i < size; i++)
                    h = (h ^ static_cast<const uint8_t*>(data)[i]) * 0x100000001B3ull;
            };
            const uint32_t format = 1;  // bump when the compile options below change
            mix(&format, sizeof(format));
            mix(&stage, sizeof(stage));
            mix(preamble.data(), preamble.size());
            mix("\0", 1);
            mix(source.data(), source.size());
            return GetCachePath() / "spirv" / std::format("{:016x}.spv", h);
        }

        std::vector<uint32_t> CompileGlslUncached(const std::string& source, glslang_stage_t stage, std::string& log,
            const std::string& preamble);
        std::atomic<uint64_t> s_shaderCacheHits{ 0 };

        std::vector<uint32_t> CompileGlsl(const std::string& source, glslang_stage_t stage, std::string& log,
            const std::string& shaderPreamble = {})
        {
            const std::string preamble = s_splitMemory ? "#define SPLIT_MEMORY\n" + shaderPreamble : shaderPreamble;
            if (!s_shaderCacheEnabled)
                return CompileGlslUncached(source, stage, log, preamble);
            std::filesystem::path file = ShaderCacheFile(source, stage, preamble);
            if (FILE* f = fopen(file.c_str(), "rb"))
            {
                std::vector<uint32_t> spirv;
                fseek(f, 0, SEEK_END);
                long size = ftell(f);
                fseek(f, 0, SEEK_SET);
                if (size >= 20 && size % 4 == 0)
                {
                    spirv.resize(size_t(size) / 4);
                    if (fread(spirv.data(), 4, spirv.size(), f) != spirv.size() || spirv[0] != 0x07230203u)
                        spirv.clear();
                }
                fclose(f);
                if (!spirv.empty())
                {
                    s_shaderCacheHits.fetch_add(1, std::memory_order_relaxed);
                    return spirv;
                }
            }
            std::vector<uint32_t> spirv = CompileGlslUncached(source, stage, log, preamble);
            if (!spirv.empty())
            {
                std::error_code ec;
                std::filesystem::create_directories(file.parent_path(), ec);
                // Unique per thread: two workers may compile the same source.
                std::filesystem::path temp = file;
                temp += std::format(".{}.tmp", std::hash<std::thread::id>{}(std::this_thread::get_id()));
                if (FILE* f = fopen(temp.c_str(), "wb"))
                {
                    bool ok = fwrite(spirv.data(), 4, spirv.size(), f) == spirv.size();
                    ok = fclose(f) == 0 && ok;
                    if (ok)
                        std::filesystem::rename(temp, file, ec);
                    else
                        std::filesystem::remove(temp, ec);
                }
            }
            return spirv;
        }

        std::vector<uint32_t> CompileGlslUncached(const std::string& source, glslang_stage_t stage, std::string& log,
            const std::string& preamble)
        {
            glslang_input_t input{};
            input.language = GLSLANG_SOURCE_GLSL;
            input.stage = stage;
            input.client = GLSLANG_CLIENT_VULKAN;
            input.client_version = GLSLANG_TARGET_VULKAN_1_2;
            input.target_language = GLSLANG_TARGET_SPV;
            input.target_language_version = GLSLANG_TARGET_SPV_1_5;
            input.code = source.c_str();
            input.default_version = 460;
            input.default_profile = GLSLANG_NO_PROFILE;
            input.messages = glslang_messages_t(GLSLANG_MSG_SPV_RULES_BIT | GLSLANG_MSG_VULKAN_RULES_BIT);
            input.resource = glslang_default_resource();

            std::vector<uint32_t> spirv;
            glslang_shader_t* shader = glslang_shader_create(&input);
            if (!preamble.empty())
                glslang_shader_set_preamble(shader, preamble.c_str());
            if (!glslang_shader_preprocess(shader, &input) || !glslang_shader_parse(shader, &input))
            {
                log = glslang_shader_get_info_log(shader);
                glslang_shader_delete(shader);
                return spirv;
            }
            glslang_program_t* program = glslang_program_create();
            glslang_program_add_shader(program, shader);
            if (!glslang_program_link(program, int(GLSLANG_MSG_SPV_RULES_BIT | GLSLANG_MSG_VULKAN_RULES_BIT)))
            {
                log = glslang_program_get_info_log(program);
            }
            else
            {
                glslang_program_SPIRV_generate(program, stage);
                spirv.resize(glslang_program_SPIRV_get_size(program));
                glslang_program_SPIRV_get(program, spirv.data());
            }
            glslang_program_delete(program);
            glslang_shader_delete(shader);
            return spirv;
        }

        // Vulkan pipeline cache, persisted as GetCachePath()/pipelines.bin.
        // Saved every 600 frames when new pipelines appeared, and at exit.
        VkPipelineCache s_pipelineCache = VK_NULL_HANDLE;
        uint32_t s_pipelinesAtSave = 0;

        void CreatePipelineCache()
        {
            std::vector<uint8_t> data;
            if (s_shaderCacheEnabled)
                if (FILE* f = fopen((GetCachePath() / "pipelines.bin").c_str(), "rb"))
                {
                    fseek(f, 0, SEEK_END);
                    long size = ftell(f);
                    fseek(f, 0, SEEK_SET);
                    if (size > 0)
                    {
                        data.resize(size_t(size));
                        if (fread(data.data(), 1, data.size(), f) != data.size())
                            data.clear();
                    }
                    fclose(f);
                }
            // A cache another GPU, driver or OS version made is set aside by
            // the driver without a word (MoltenVK's device ID carries the
            // macOS version, so every macOS update does it), and the first
            // drive afterwards stutters while the pipelines compile again.
            // The header says so in advance (gpu/pipeline_cache_header.h): one
            // line for the log, and the stale data isn't handed over. The
            // next save writes this device's header, so it's said once. No
            // toast: it isn't something to act on, it would come after every
            // SteamOS and macOS update, and the stutter is a frame or two at
            // the first world entry (a first run with no cache at all has
            // more, and says nothing either).
            {
                VkPhysicalDeviceProperties props;
                vkGetPhysicalDeviceProperties(s_vk->physical, &props);
                gpu::pipeline_cache::Maker device{ props.vendorID, props.deviceID, {} };
                std::memcpy(device.uuid.data(), props.pipelineCacheUUID, device.uuid.size());
                const auto check = gpu::pipeline_cache::CheckHeader(data.data(), data.size(), device);
                if (const std::string line = gpu::pipeline_cache::LogLine(check, device); !line.empty())
                {
                    fprintf(stderr, "%s\n", line.c_str());
                    data.clear();
                }
            }
            VkPipelineCacheCreateInfo pcci{ VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO };
            pcci.initialDataSize = data.size();
            pcci.pInitialData = data.empty() ? nullptr : data.data();
            if (vkCreatePipelineCache(s_dev, &pcci, nullptr, &s_pipelineCache) != VK_SUCCESS)
            {
                pcci.initialDataSize = 0;
                pcci.pInitialData = nullptr;
                Check(vkCreatePipelineCache(s_dev, &pcci, nullptr, &s_pipelineCache), "vkCreatePipelineCache");
            }
            fprintf(stderr, "[renderer] pipeline cache: %zu KB loaded\n", data.size() / 1024);
        }

        // Saves are serialized (the background saver and the one at exit).
        std::mutex s_pipelineSaveMutex;

        void SavePipelineCache()
        {
            if (!s_pipelineCache || !s_shaderCacheEnabled)
                return;
            std::lock_guard saveLock(s_pipelineSaveMutex);
            // Workers may add pipelines between the two calls (VK_INCOMPLETE).
            size_t size = 0;
            std::vector<uint8_t> data;
            for (int attempt = 0;; attempt++)
            {
                if (vkGetPipelineCacheData(s_dev, s_pipelineCache, &size, nullptr) != VK_SUCCESS || size == 0)
                    return;
                data.resize(size);
                VkResult r = vkGetPipelineCacheData(s_dev, s_pipelineCache, &size, data.data());
                if (r == VK_SUCCESS)
                    break;
                if (r != VK_INCOMPLETE || attempt == 3)
                    return;
            }
            std::error_code ec;
            std::filesystem::create_directories(GetCachePath(), ec);
            std::filesystem::path file = GetCachePath() / "pipelines.bin", temp = file;
            temp += ".tmp";
            if (FILE* f = fopen(temp.c_str(), "wb"))
            {
                bool ok = fwrite(data.data(), 1, size, f) == size;
                ok = fclose(f) == 0 && ok;
                if (ok)
                    std::filesystem::rename(temp, file, ec);
                else
                    std::filesystem::remove(temp, ec);
            }
        }

        VkShaderModule CreateModule(const std::vector<uint32_t>& spirv)
        {
            VkShaderModuleCreateInfo smci{ VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO };
            smci.codeSize = spirv.size() * 4;
            smci.pCode = spirv.data();
            VkShaderModule module;
            Check(vkCreateShaderModule(s_dev, &smci, nullptr, &module), "vkCreateShaderModule");
            return module;
        }

        // A pixel shader variant: colour outputs only for bound attachments
        // (Metal rejects others), inputs only for interpolators the vertex
        // shader writes, and the alpha test compiled in only when enabled.
        VkShaderModule VertexModule(Shader& vs)
        {
            std::lock_guard lock(vs.compileMutex);
            if (!vs.vertexCompiled)
            {
                vs.vertexCompiled = true;
                std::string log;
                auto spirv = CompileGlsl(vs.info.glsl, GLSLANG_STAGE_VERTEX, log);
                if (spirv.empty())
                    fprintf(stderr, "[renderer] vs_%016llx: GLSL compile failed:\n%s\n", (unsigned long long)vs.hash, log.c_str());
                else
                    vs.module = CreateModule(spirv);
            }
            return vs.module;
        }

        VkShaderModule PixelVariant(Shader& ps, uint32_t outputs, uint32_t inputs, bool alphaTest)
        {
            uint32_t key = (outputs & 0xF) | ((inputs & 0xFFFF) << 4) | (alphaTest ? 1u << 20 : 0);
            std::lock_guard lock(ps.compileMutex);
            auto it = ps.variants.find(key);
            if (it != ps.variants.end())
                return it->second;
            std::string preamble;
            for (int i = 0; i < 4; i++)
                if (outputs & (1u << i))
                    preamble += std::format("#define OUT{}\n", i);
            for (int i = 0; i < 16; i++)
                if (inputs & (1u << i))
                    preamble += std::format("#define IN{}\n", i);
            if (alphaTest)
                preamble += "#define ALPHA_TEST\n";
            std::string log;
            auto spirv = CompileGlsl(ps.info.glsl, GLSLANG_STAGE_FRAGMENT, log, preamble);
            VkShaderModule module = VK_NULL_HANDLE;
            if (spirv.empty())
                fprintf(stderr, "[renderer] ps_%016llx (variant %x): GLSL compile failed:\n%s\n",
                    (unsigned long long)ps.hash, key, log.c_str());
            else
                module = CreateModule(spirv);
            ps.variants[key] = module;
            return module;
        }

        // ---------------------------------------------------------------
        // Layouts: set 0 = shared memory + constants (one static set),
        // set 1 = the textures a shader pair uses (push descriptors).

        VkDescriptorSetLayout s_set0Layout = VK_NULL_HANDLE;
        VkDescriptorPool s_descriptorPool = VK_NULL_HANDLE;
        VkDescriptorSet s_set0 = VK_NULL_HANDLE;
        VkSampler s_sampler = VK_NULL_HANDLE;
        Image s_dummy[3];  // 2D, 3D, cube: until the texture cache (M4)
        Image s_dummyStorage;  // R32_UINT 1x1: a resolve's texture binding when it stores into none

        struct DrawConstants
        {
            uint32_t vsConstBase, psConstBase, indexAddress, indexInfo, indexOffset, alphaTest;
            float alphaRef;
            uint32_t flags;
            float posScale[4];
            uint32_t vfRedirect[4][4];  // guest start, size, ring offset, 0 (vertex data snapshots)
        };
        static_assert(sizeof(DrawConstants) == 112);

        struct TextureLayout
        {
            VkDescriptorSetLayout set = VK_NULL_HANDLE;
            VkPipelineLayout pipeline = VK_NULL_HANDLE;
            std::array<uint32_t, 3> vsMasks{}, psMasks{};
        };
        std::map<std::array<uint32_t, 6>, std::unique_ptr<TextureLayout>> s_textureLayouts;

        const TextureLayout& GetTextureLayoutUncached(const Shader& vs, const Shader& ps);

        const TextureLayout& GetTextureLayout(const Shader& vs, const Shader& ps)
        {
            static const Shader *lastVs = nullptr, *lastPs = nullptr;
            static const TextureLayout* last = nullptr;
            if (&vs == lastVs && &ps == lastPs)
                return *last;
            lastVs = &vs;
            lastPs = &ps;
            last = &GetTextureLayoutUncached(vs, ps);
            return *last;
        }

        const TextureLayout& GetTextureLayoutUncached(const Shader& vs, const Shader& ps)
        {
            std::array<uint32_t, 3> v = { vs.info.texture2DMask, vs.info.texture3DMask, vs.info.textureCubeMask };
            std::array<uint32_t, 3> p = { ps.info.texture2DMask, ps.info.texture3DMask, ps.info.textureCubeMask };
            std::array<uint32_t, 6> key = { v[0], v[1], v[2], p[0], p[1], p[2] };
            auto& slot = s_textureLayouts[key];
            if (slot)
                return *slot;
            slot = std::make_unique<TextureLayout>();
            slot->vsMasks = v;
            slot->psMasks = p;
            std::vector<VkDescriptorSetLayoutBinding> bindings;
            for (uint32_t d = 0; d < 3; d++)
                for (uint32_t n = 0; n < 32; n++)
                {
                    VkShaderStageFlags stages = ((v[d] >> n) & 1 ? VK_SHADER_STAGE_VERTEX_BIT : 0) |
                        ((p[d] >> n) & 1 ? VK_SHADER_STAGE_FRAGMENT_BIT : 0);
                    if (stages)
                        bindings.push_back({ d * 32 + n, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, stages, nullptr });
                }
            VkDescriptorSetLayoutCreateInfo dslci{ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO };
            dslci.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
            dslci.bindingCount = uint32_t(bindings.size());
            dslci.pBindings = bindings.data();
            Check(vkCreateDescriptorSetLayout(s_dev, &dslci, nullptr, &slot->set), "vkCreateDescriptorSetLayout(textures)");
            VkDescriptorSetLayout sets[] = { s_set0Layout, slot->set };
            VkPushConstantRange range{ VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(DrawConstants) };
            VkPipelineLayoutCreateInfo plci{ VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO };
            plci.setLayoutCount = 2;
            plci.pSetLayouts = sets;
            plci.pushConstantRangeCount = 1;
            plci.pPushConstantRanges = &range;
            Check(vkCreatePipelineLayout(s_dev, &plci, nullptr, &slot->pipeline), "vkCreatePipelineLayout");
            return *slot;
        }

        // ---------------------------------------------------------------
        // Render targets (EDRAM)

        struct RenderTarget
        {
            uint32_t base, format, pitch;
            bool depth;
            bool wide;                         // side-by-side tiles drawn untiled (see GetTarget)
            uint32_t width, height;            // host pixels (guest x scale, rounded up)
            uint32_t guestWidth, guestHeight;  // guest pixels
            gpu::RenderScale scale;            // host pixels per guest pixel (gpu/render_scale.h)
            VkFormat vkFormat;
            bool blendable;
            Image image;
            VkImageView sampleView = VK_NULL_HANDLE;  // depth aspect only, for resolves
            // NFSMW_UNTILED_AREA: in the last frame untiled draws drew it (until
            // the next one's first), whether one was cut short of the
            // attachments' common size, what those cut draws drew (their host
            // scissors' union; one cut to nothing adds nothing) and that
            // common size; and whether a resolve has read past them (its
            // draws then keep the whole target).
            uint64_t areaFrame = ~0ull;
            VkRect2D cutDrawn{};
            VkExtent2D cutReach{};
            bool cut = false;
            bool wholeArea = false;
        };
        std::unordered_map<uint64_t, std::unique_ptr<RenderTarget>> s_targets;

        VkFormat ColorFormat(uint32_t format)
        {
            switch (xenos::ColorRenderTargetFormat(format))
            {
            case xenos::ColorRenderTargetFormat::k_8_8_8_8:
            case xenos::ColorRenderTargetFormat::k_8_8_8_8_GAMMA: return VK_FORMAT_R8G8B8A8_UNORM;
            case xenos::ColorRenderTargetFormat::k_2_10_10_10:
            case xenos::ColorRenderTargetFormat::k_2_10_10_10_AS_10_10_10_10: return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
            case xenos::ColorRenderTargetFormat::k_2_10_10_10_FLOAT:
            case xenos::ColorRenderTargetFormat::k_2_10_10_10_FLOAT_AS_16_16_16_16:
            case xenos::ColorRenderTargetFormat::k_16_16_16_16_FLOAT: return VK_FORMAT_R16G16B16A16_SFLOAT;
            case xenos::ColorRenderTargetFormat::k_16_16: return VK_FORMAT_R16G16_SNORM;
            case xenos::ColorRenderTargetFormat::k_16_16_16_16: return VK_FORMAT_R16G16B16A16_SNORM;
            case xenos::ColorRenderTargetFormat::k_16_16_FLOAT: return VK_FORMAT_R16G16_SFLOAT;
            case xenos::ColorRenderTargetFormat::k_32_FLOAT: return VK_FORMAT_R32_SFLOAT;
            case xenos::ColorRenderTargetFormat::k_32_32_FLOAT: return VK_FORMAT_R32G32_SFLOAT;
            default: return VK_FORMAT_R8G8B8A8_UNORM;
            }
        }

        bool Is64bpp(uint32_t format)
        {
            switch (xenos::ColorRenderTargetFormat(format))
            {
            case xenos::ColorRenderTargetFormat::k_16_16_16_16:
            case xenos::ColorRenderTargetFormat::k_16_16_16_16_FLOAT:
            case xenos::ColorRenderTargetFormat::k_2_10_10_10_FLOAT_AS_16_16_16_16:
            case xenos::ColorRenderTargetFormat::k_32_32_FLOAT: return true;
            default: return false;
            }
        }

        // Untiled rendering (see TileShift); NFSMW_TILING=1 renders per tile.
        const bool s_untiled = [] { const char* v = std::getenv("NFSMW_TILING"); return !(v && v[0] == '1'); }();
        // NFSMW_UNTILED_AREA=1 (default with kLighterPasses): an untiled draw with clipping on
        // is scissored to its guest viewport, not the whole EDRAM-sized
        // target, so its pass's area (what a tile GPU loads and stores:
        // MoltenVK's render target runs from (0,0) to the area's far corner)
        // is the frame: the scene's 1280x720, not 1280x1024, and a cube
        // face's 256x256, not 1280x896. ~130 MB a race frame less (modelled:
        // the area's far corner x the attachments' bytes, loaded and stored);
        // an M4 Max measured -0.09 ms, a phone is unmeasured. The console
        // never draws past the viewport there (its window scissor is the
        // tile inside it); the host clipped only to the target, so the road
        // below the frame and the neighbouring faces' geometry were shaded
        // too. Clipping off, or x and y not both scaled and offset: the whole
        // target, as before. Where cutting would change more than the area,
        // the target falls back to the whole target from then on
        // (RenderTarget::wholeArea), logged: a resolve reading past what its
        // cut draws drew (rows the host used to overdraw; see Resolve), a
        // draw that would restart its pass for its area (no untiled draw
        // did), two cuts whose union isn't exact.
        // =alt (measurement only): on in every other 120-frame [perf] window,
        // a paired A/B within one run.
        // NFSMW_LOG_UNTILED_AREA=1: counts with each [perf] line (draws cut,
        // pass restarts, the passes' area and bytes), and each distinct cut
        // and each shader pair keeping the whole target, once.
        // On by default with kLighterPasses (Apple: iPad Pro M2 GPU-bound at
        // 2x, same-run windows: executing -1.0 ms), else off; =0 turns it off.
        const uint32_t s_untiledArea = [] {
            const char* v = std::getenv("NFSMW_UNTILED_AREA");
            return !v ? (kLighterPasses ? 1u : 0u) : v[0] == '1' ? 1u : strcmp(v, "alt") == 0 ? 2u : 0u;
        }();
        const bool s_logUntiledArea = [] { const char* v = std::getenv("NFSMW_LOG_UNTILED_AREA"); return v && v[0] == '1'; }();
        struct UntiledAreaCounts
        {
            uint64_t cut = 0, whole = 0, clipOff = 0, noViewport = 0, wholeArea = 0, empty = 0, restarts = 0;
            uint64_t passes = 0, passPixels = 0, passBytes = 0;                // every pass: its area, its attachments' bytes there
            uint64_t untiledPasses = 0, untiledPixels = 0, untiledBytes = 0;  // passes on wide or MSAA-tiled targets
            uint64_t resolvesPast = 0;
        } s_ua;

        // Internal resolution. Render targets of the scene's surface pitches
        // (NFSMW_SCALE_PITCH, default 640-1280: the scene, its side-by-side
        // tiles, the road reflection, the final composite) are created at
        // guest size x (sx, sy) host pixels; the rest (the 1600x1600 shadow
        // map, the 256x256 cube faces, the bloom and luminance chain from
        // 320x180 down) stay 1x. The bloom blurs step in texels: at scale
        // they cover a fraction of their area and the game's haze thins (the
        // colour grade visibly changed with 320 scaled), and they are blurry
        // by design anyway. Draws keep the guest's viewport transform in guest pixels
        // and scale only the host viewport and scissor; resolves average each
        // guest pixel's host pixels down into guest memory (depth takes one
        // sample), so the game and the texture cache see 1x data as before.
        // The scale is a ratio on each axis (gpu/render_scale.h): 1, 1.5
        // (3 host pixels for every 2 guest ones: 1920x1080), 2, 3, 4.
        // NFSMW_RENDER_SCALE=<n> or <sx>x<sy> (1, 1.5, 2, 3 or 4; default 1).
        // The setting applies live (ApplyLiveSettings): every scaled target,
        // the scaled resolve memory and the scaled textures are made again at
        // the new scale between two frames, and Auto follows the picture's
        // size (the window, fullscreen, the aspect ratio setting).
        gpu::RenderScale s_renderScale;
        uint32_t s_scalePitchMin = 640, s_scalePitchMax = 1280;
        uint32_t s_scaleBudget = 6;  // Auto's most host pixels per guest pixel
        bool s_scaleFixed = false;   // NFSMW_RENDER_SCALE: no live changes

        // What the Internal Resolution setting asks for with the picture
        // covering `regionW` x `regionH` of the swapchain.
        // The setting's value is the scale in half steps (settings.h: 2 is
        // 1x, 3 is 1.5x, 4 is 2x, 6 is 3x). Auto (whole scales only, for
        // now: video::AutoRenderScale) could pick 1.5x by returning one.
        gpu::RenderScale WantedRenderScale(int32_t setting, uint32_t regionW, uint32_t regionH)
        {
            if (setting > 0)
            {
                gpu::AxisScale a = gpu::AxisFromHalves(uint32_t(std::clamp(setting, 2, 6)));
                return { a, a };
            }
            uint32_t sx = 1, sy = 1;
            // Auto: enough host pixels to cover the screen area the frame
            // fills (video/picture_fit: 3440x1440 filled: 3x2; a 1440p 16:9
            // box: 2x2; a Steam Deck's 1280x800 filled with Vert+: 1x1),
            // within a budget of sx x sy host pixels per guest pixel: 6 on a
            // discrete GPU, 4 on an integrated one (the M1 Pro holds a race
            // at 2x2), 9 on a Mac with an M3 or later Max or Ultra, and on an
            // iPhone or iPad the chip's (video/scale_budget: 1 on every chip
            // measured so far).
            video::AutoRenderScale(regionW, regionH, s_scaleBudget, sx, sy);
            return gpu::RenderScale::Whole(sx, sy);
        }

        std::string ScaleReason(int32_t setting, uint32_t regionW, uint32_t regionH)
        {
            if (s_scaleFixed)
                return "NFSMW_RENDER_SCALE";
            if (setting > 0)
                return "setting";
            return std::format("auto for a {}x{} picture", regionW, regionH);
        }

        void ReadRenderScale(const VkPhysicalDeviceProperties& props)
        {
            // Settings > Video > Internal Resolution; NFSMW_RENDER_SCALE
            // (<n> or <sx>x<sy>, developer override) wins.
            s_scaleBudget = props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 6 : 4;
#if defined(__APPLE__) && TARGET_OS_IOS
            // iPhones and iPads: chip by chip (video/scale_budget), more than
            // 1x1 only where a chip was measured to hold a race at 60 fps with
            // it and stay out of serious thermal throttling (none yet: the A20
            // Pro gets hot at 1x1 already), and 1x1 on a chip not measured.
            // Scripted drive, GPU time a frame: iPhone 15 Pro Max (A17 Pro)
            // 15 ms at 1x, 36-40 at 2x; iPad Pro M2 20 ms at 2x (43-50 fps).
            {
                const video::ScaleBudget mobile = video::MobileScaleBudget(props.deviceName);
                s_scaleBudget = mobile.pixels;
                fprintf(stderr, "[renderer] Auto internal resolution: at most %u host pixel%s per guest pixel on the %s (%s)\n",
                    s_scaleBudget, s_scaleBudget == 1 ? "" : "s", props.deviceName, mobile.reason);
            }
#elif defined(__APPLE__)
            // Apple GPUs are all integrated, but a recent Max or Ultra has
            // the headroom for 3x3 (4K). Scripted race, GPU time a frame, M4
            // Max: 5-6.6 ms at 2x2, 9-11 ms at 3x3, 60 fps locked.
            // An M1 or M2 Max has about half that GPU (its 3x3 would be ~20
            // ms): from the M3 on only. MoltenVK names the device after the
            // Metal one ("Apple M4 Max").
            std::string_view name = props.deviceName;
            if (name.starts_with("Apple M") && (name.ends_with(" Max") || name.ends_with(" Ultra")))
            {
                unsigned generation = 0;
                for (char c : name.substr(7))
                {
                    if (c < '0' || c > '9')
                        break;
                    generation = generation * 10 + unsigned(c - '0');
                }
                if (generation >= 3)
                    s_scaleBudget = 9;
            }
#endif
            int32_t setting = settings::GetInt(settings::Id::InternalResolution);
            uint32_t regionW = 0, regionH = 0;
            const char* fixed = std::getenv("NFSMW_RENDER_SCALE");
            if (fixed && gpu::ParseRenderScale(fixed, s_renderScale))
                s_scaleFixed = true;
            else
            {
                if (fixed)
                    fprintf(stderr, "[renderer] NFSMW_RENDER_SCALE=%s ignored: each axis takes 1, 1.5, 2, 3 or 4 (\"1.5\", \"3x2\")\n", fixed);
                video::FrameRegion(regionW, regionH);
                s_renderScale = WantedRenderScale(setting, regionW, regionH);
            }
            if (const char* v = std::getenv("NFSMW_SCALE_PITCH"))
                sscanf(v, "%u-%u", &s_scalePitchMin, &s_scalePitchMax);
            fprintf(stderr, "[renderer] internal resolution %s (%ux%u, %s; surface pitches %u-%u)\n", gpu::ScaleName(s_renderScale).c_str(),
                s_renderScale.x.Size(1280), s_renderScale.y.Size(720), ScaleReason(setting, regionW, regionH).c_str(), s_scalePitchMin,
                s_scalePitchMax);
        }

        bool ScaledPitch(uint32_t pitch)
        {
            return pitch >= s_scalePitchMin && pitch <= s_scalePitchMax;
        }

        // Untiled rendering draws tile 0 over the whole frame (TileShift).
        // With tiles side by side (NFSMW switches to two 640-wide tiles in
        // some scenes) the frame is wider than the surface pitch: those
        // targets get the full 720p width.
        constexpr uint32_t kUntiledWidth = 1280;

        // ~5 lookups a draw: a small direct-mapped cache before the map
        // (targets are freed only when the internal resolution changes,
        // which clears it).
        struct RecentTarget { uint64_t key = ~0ull; RenderTarget* target = nullptr; };
        RecentTarget s_recentTargets[16];

        // A new target, at the current scale if its pitch is a scaled one,
        // cleared to zero (new EDRAM contents).
        std::unique_ptr<RenderTarget> CreateTarget(uint32_t base, uint32_t format, bool depth, uint32_t pitch, bool wide)
        {
            // Height: whatever fits in EDRAM (2048 tiles of 80x16 32-bit
            // samples) from this base, capped at the 2048-line maximum.
            uint32_t tilesPerRow = std::max(1u, (pitch + 79) / 80) * (!depth && Is64bpp(format) ? 2 : 1);
            uint32_t rows = std::clamp((2048u - std::min(base, 2047u)) / tilesPerRow * 16, 16u, 2048u);

            auto rt = std::make_unique<RenderTarget>();
            rt->base = base;
            rt->format = format;
            rt->pitch = pitch;
            rt->depth = depth;
            rt->wide = wide;
            rt->guestWidth = wide ? kUntiledWidth : pitch;
            rt->guestHeight = rows;
            if (ScaledPitch(pitch))
                rt->scale = s_renderScale;
            // (Rounded up: a pitch that isn't whole blocks at 1.5x still
            // has room for its last host column.)
            rt->width = rt->scale.x.Size(rt->guestWidth);
            rt->height = rt->scale.y.Size(rt->guestHeight);
            rt->vkFormat = depth ? VK_FORMAT_D32_SFLOAT_S8_UINT : ColorFormat(format);
            VkFormatProperties fp;
            vkGetPhysicalDeviceFormatProperties(s_vk->physical, rt->vkFormat, &fp);
            rt->blendable = (fp.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT) != 0;
            VkImageAspectFlags aspect = depth ? VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_COLOR_BIT;
            VkImageUsageFlags usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
                (depth ? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT : VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
            rt->image = CreateImage(VK_IMAGE_TYPE_2D, rt->vkFormat, { rt->width, rt->height, 1 }, 1, usage,
                VK_IMAGE_VIEW_TYPE_2D, aspect);
            if (depth)
            {
                VkImageViewCreateInfo vci{ VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO };
                vci.image = rt->image.image;
                vci.viewType = VK_IMAGE_VIEW_TYPE_2D;
                vci.format = rt->vkFormat;
                vci.subresourceRange = { VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1 };
                Check(vkCreateImageView(s_dev, &vci, nullptr, &rt->sampleView), "vkCreateImageView(depth)");
            }
            else
            {
                rt->sampleView = rt->image.view;
            }

            // New EDRAM contents: zero.
            EndPass();
            Begin();
            ToGeneral(rt->image.image, aspect, 1);
            VkImageSubresourceRange range{ aspect, 0, 1, 0, 1 };
            if (depth)
            {
                VkClearDepthStencilValue v{ 0.0f, 0 };
                vkCmdClearDepthStencilImage(s_cmd, rt->image.image, VK_IMAGE_LAYOUT_GENERAL, &v, 1, &range);
            }
            else
            {
                VkClearColorValue v{};
                vkCmdClearColorImage(s_cmd, rt->image.image, VK_IMAGE_LAYOUT_GENERAL, &v, 1, &range);
            }
            FullBarrier();

            fprintf(stderr, "[renderer] EDRAM target: base %u %s format %u pitch %u -> %ux%u%s\n",
                base, depth ? "depth" : "color", format, pitch, rt->guestWidth, rt->guestHeight,
                !rt->scale.IsOne() ? std::format(" (host {}x{})", rt->width, rt->height).c_str() : "");
            return rt;
        }

        void DestroyTarget(RenderTarget& rt)
        {
            if (rt.sampleView && rt.sampleView != rt.image.view)
                vkDestroyImageView(s_dev, rt.sampleView, nullptr);
            rt.sampleView = VK_NULL_HANDLE;
            DestroyImage(rt.image);
        }

        RenderTarget* GetTarget(uint32_t base, uint32_t format, bool depth, uint32_t pitch)
        {
            bool wide = s_untiled && Reg<reg::RB_SURFACE_INFO>().msaa_samples != xenos::MsaaSamples::k1X &&
                Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable && pitch < kUntiledWidth;
            uint64_t key = uint64_t(base) | (uint64_t(format) << 12) | (uint64_t(depth) << 16) | (uint64_t(pitch) << 17) |
                (uint64_t(wide) << 31);
            RecentTarget& hit = s_recentTargets[(key ^ (key >> 12) ^ (key >> 17)) & 15];
            if (hit.key == key)
                return hit.target;
            auto& slot = s_targets[key];
            if (!slot)
                slot = CreateTarget(base, format, depth, pitch, wide);
            hit = { key, slot.get() };
            return slot.get();
        }

        // Current dynamic rendering pass.
        RenderTarget* s_passColor[4] = {};
        RenderTarget* s_passDepth = nullptr;
        bool s_passActive = false;
        VkExtent2D s_passExtent{};  // the attachments' common size
        VkRect2D s_passArea{};      // what this pass loads and stores (tile GPUs move only this)
        // NFSMW_PASS_MERGE bitmask (Draw; default 3 with kFewerPasses, else
        // 0), fewer passes around the sun: 1 skip draws that write nothing (its
        // occlusion quad), 2 draw into the open pass a draw whose targets are
        // some of its (its glow). In every view with the sun in it (the
        // mirror, up to three cube faces, the scene) the pair split the
        // view's pass in three: 10 of the 46 passes of an iPad race frame's
        // profile. Expected ~0.4-0.5 ms of its GPU time (bit 1 ~0.2, bit 2
        // ~0.15-0.25: the glow and flare draws still run).
        const uint32_t s_passMerge = [] { const char* v = std::getenv("NFSMW_PASS_MERGE"); return v ? uint32_t(std::atoi(v)) : kFewerPasses ? 3u : 0u; }();

        void EndPass()
        {
            if (!s_passActive)
                return;
            ZpdEndSegment();  // a segment never outlives its pass
            s_vk->cmdEndRendering(s_cmd);
            s_passActive = false;
            if (s_profiling && !s_passInfos.empty() && s_passInfos.back().query + 1 < s_passQueryNext)
                vkCmdWriteTimestamp(s_cmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, s_passQueries, s_passInfos.back().query + 1);
            PassEndBarrier();
            SignalWrites();
        }

        // Guest rectangle [x0, x1) x [y0, y1) in host pixels at `scale`: its
        // edges' (gpu/render_scale.h; a negative edge is 0), so rectangles
        // that share a guest edge share the host one.
        VkRect2D HostRect(const gpu::RenderScale& scale, int32_t x0, int32_t y0, int32_t x1, int32_t y1)
        {
            const int32_t hx0 = scale.x.EdgeAt(x0), hy0 = scale.y.EdgeAt(y0), hx1 = scale.x.EdgeAt(x1), hy1 = scale.y.EdgeAt(y1);
            return { { hx0, hy0 }, { uint32_t(std::max(hx1 - hx0, 0)), uint32_t(std::max(hy1 - hy0, 0)) } };
        }

        bool Contains(const VkRect2D& outer, const VkRect2D& inner)
        {
            return inner.offset.x >= outer.offset.x && inner.offset.y >= outer.offset.y &&
                inner.offset.x + int32_t(inner.extent.width) <= outer.offset.x + int32_t(outer.extent.width) &&
                inner.offset.y + int32_t(inner.extent.height) <= outer.offset.y + int32_t(outer.extent.height);
        }

        VkRect2D Union(const VkRect2D& a, const VkRect2D& b)
        {
            int32_t x0 = std::min(a.offset.x, b.offset.x), y0 = std::min(a.offset.y, b.offset.y);
            int32_t x1 = std::max(a.offset.x + int32_t(a.extent.width), b.offset.x + int32_t(b.extent.width));
            int32_t y1 = std::max(a.offset.y + int32_t(a.extent.height), b.offset.y + int32_t(b.extent.height));
            return { { x0, y0 }, { uint32_t(x1 - x0), uint32_t(y1 - y0) } };
        }

        bool Overlaps(const VkRect2D& a, const VkRect2D& b)
        {
            return a.offset.x < b.offset.x + int32_t(b.extent.width) && b.offset.x < a.offset.x + int32_t(a.extent.width) &&
                a.offset.y < b.offset.y + int32_t(b.extent.height) && b.offset.y < a.offset.y + int32_t(a.extent.height);
        }

        // Stacked or side by side with a whole edge in common: their union is
        // exactly the two.
        bool Adjoins(const VkRect2D& a, const VkRect2D& b)
        {
            int32_t ax1 = a.offset.x + int32_t(a.extent.width), ay1 = a.offset.y + int32_t(a.extent.height);
            int32_t bx1 = b.offset.x + int32_t(b.extent.width), by1 = b.offset.y + int32_t(b.extent.height);
            if (a.offset.x == b.offset.x && ax1 == bx1)
                return ay1 == b.offset.y || by1 == a.offset.y;
            if (a.offset.y == b.offset.y && ay1 == by1)
                return ax1 == b.offset.x || bx1 == a.offset.x;
            return false;
        }

        // `area` (in target pixels) is what the next draw or clear touches; a
        // pass on the same targets is kept while it covers that, else it is
        // restarted over the union.
        // Resolve clears are deferred into the next pass on their target (a
        // pass of their own would load and store the attachments on a tile
        // GPU): a load-op clear when they cover the render area, else an
        // in-pass clear.
        // Untiled rendering of predicated tiling. At 720p with 4x MSAA the
        // game renders the scene once per EDRAM tile, 3 tiles of 256 rows
        // (window offsets 0, -256, -512), and resolves each tile. The host
        // has no EDRAM limit (and renders at 1x), so tile 0's draws cover the
        // whole target, later tiles' draws are skipped, and each tile's
        // resolve and clear read the target at screen coordinates (EDRAM
        // rectangle minus the window offset). NFSMW_TILING=1 keeps the
        // per-tile rendering.

        // The window offset to undo for the current MSAA surface, or 0,0.
        std::pair<int32_t, int32_t> TileShift()
        {
            if (!s_untiled || Reg<reg::RB_SURFACE_INFO>().msaa_samples == xenos::MsaaSamples::k1X ||
                !Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable)
                return { 0, 0 };
            auto wo = Reg<reg::PA_SC_WINDOW_OFFSET>();
            return { -wo.window_x_offset, -wo.window_y_offset };
        }

        struct PendingClear
        {
            RenderTarget* rt;
            VkRect2D rect;
            VkClearValue value;
        };
        std::vector<PendingClear> s_pendingClears;
        // NFSMW_CLEAR_FLUSH_RECT=1 (default kFewerPasses; 0: a resolve runs
        // every pending clear of its source): a resolve runs only the pending clears of
        // what it reads (FlushPendingClears), and a clear adjoining its
        // target's newest pending one of the same value extends it
        // (ClearRect). The 4 empty passes a race frame spent on the scene's
        // tile bands, ~0.2-0.3 ms of the iPad's GPU time. The clears kept
        // wait for the next pass on their target, which must hold them
        // (BeginPass logs one that reaches past its pass).
        const bool s_clearFlushRect = [] { const char* v = std::getenv("NFSMW_CLEAR_FLUSH_RECT"); return v ? v[0] == '1' : kFewerPasses; }();

        void BeginPass(RenderTarget* const color[4], RenderTarget* depth, VkRect2D area)
        {
            bool sameTargets = s_passActive && depth == s_passDepth && std::equal(color, color + 4, s_passColor);
            if (sameTargets && Contains(s_passArea, area))
                return;
            if (sameTargets)
            {
                area = Union(area, s_passArea);
                if (s_logUntiledArea)
                    s_ua.restarts++;
            }
            auto attached = [&](RenderTarget* rt) { return rt == depth || std::find(color, color + 4, rt) != color + 4; };
            for (const PendingClear& pc : s_pendingClears)
                if (attached(pc.rt))
                    area = Union(area, pc.rect);
            EndPass();
            Begin();
            uint32_t w = UINT32_MAX, h = UINT32_MAX;
            VkRenderingAttachmentInfoKHR colors[4]{};
            for (int i = 0; i < 4; i++)
            {
                s_passColor[i] = color[i];
                colors[i].sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR;
                colors[i].imageView = color[i] ? color[i]->image.view : VK_NULL_HANDLE;
                colors[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
                colors[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
                colors[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
                if (color[i])
                {
                    w = std::min(w, color[i]->width);
                    h = std::min(h, color[i]->height);
                }
            }
            s_passDepth = depth;
            VkRenderingAttachmentInfoKHR ds{ VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR };
            if (depth)
            {
                ds.imageView = depth->image.view;
                ds.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
                ds.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
                ds.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
                w = std::min(w, depth->width);
                h = std::min(h, depth->height);
            }
            s_passExtent = { w, h };
            // Clamp to the attachments.
            int32_t x0 = std::clamp(area.offset.x, 0, int32_t(w)), y0 = std::clamp(area.offset.y, 0, int32_t(h));
            int32_t x1 = std::clamp(area.offset.x + int32_t(area.extent.width), x0, int32_t(w));
            int32_t y1 = std::clamp(area.offset.y + int32_t(area.extent.height), y0, int32_t(h));
            s_passArea = { { x0, y0 }, { uint32_t(x1 - x0), uint32_t(y1 - y0) } };
            if (s_logUntiledArea)
            {
                // What Metal loads and stores: MoltenVK's render target runs
                // from (0,0) to the area's far corner. D32S8: 4 + 1 bytes.
                uint64_t px = uint64_t(x1) * uint64_t(y1), bpp = depth ? 5 : 0;
                bool untiled = depth && depth->wide;
                for (int i = 0; i < 4; i++)
                    if (color[i])
                    {
                        bpp += Is64bpp(color[i]->format) ? 8 : 4;
                        untiled = untiled || color[i]->wide;
                    }
                untiled = untiled || (s_untiled && Reg<reg::RB_SURFACE_INFO>().msaa_samples != xenos::MsaaSamples::k1X &&
                    Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable);
                s_ua.passes++;
                s_ua.passPixels += px;
                s_ua.passBytes += 2 * px * bpp;
                if (untiled)
                {
                    s_ua.untiledPasses++;
                    s_ua.untiledPixels += px;
                    s_ua.untiledBytes += 2 * px * bpp;
                }
            }

            // Pending clears of these targets: whole-area ones become load ops.
            std::vector<PendingClear> inPass;
            for (auto it = s_pendingClears.begin(); it != s_pendingClears.end();)
            {
                if (!attached(it->rt))
                {
                    ++it;
                    continue;
                }
                bool whole = Contains(it->rect, s_passArea);
                // One reaching past the attachments' common size (a target
                // larger than another attached) is consumed here, but the
                // part beyond this pass is never cleared; one not covering
                // the pass is also a vkCmdClearAttachments outside the render
                // area (invalid usage). A deferred clear relies on its
                // target's next pass holding it, and NFSMW_CLEAR_FLUSH_RECT
                // defers more of them: with it, logged once per target.
                if (s_clearFlushRect && !Contains(s_passArea, it->rect))
                    LogOnce((0x27ull << 56) | (uint64_t(it->rt->base) << 24) | (uint64_t(it->rt->pitch) << 1) | uint64_t(it->rt->depth),
                        "pending clear %ux%u@%d,%d of target b%u p%u%s reaches past its pass (area %ux%u@%d,%d): the rest is not cleared%s",
                        it->rect.extent.width, it->rect.extent.height, it->rect.offset.x, it->rect.offset.y, it->rt->base, it->rt->pitch,
                        it->rt->depth ? " (depth)" : "", s_passArea.extent.width, s_passArea.extent.height, s_passArea.offset.x,
                        s_passArea.offset.y, whole ? "" : " (cleared in the pass: a vkCmdClearAttachments rect outside its render area)");
                VkRenderingAttachmentInfoKHR* att = nullptr;
                if (it->rt == depth)
                    att = &ds;
                else
                    for (int i = 0; i < 4; i++)
                        if (color[i] == it->rt)
                            att = &colors[i];
                if (whole && att)
                {
                    // Supersedes any older partial clear of this target.
                    std::erase_if(inPass, [&](const PendingClear& pc) { return pc.rt == it->rt; });
                    att->loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
                    att->clearValue = it->value;
                }
                else
                {
                    inPass.push_back(*it);
                }
                it = s_pendingClears.erase(it);
            }
            VkRenderingInfoKHR ri{ VK_STRUCTURE_TYPE_RENDERING_INFO_KHR };
            ri.renderArea = s_passArea;
            ri.layerCount = 1;
            ri.colorAttachmentCount = 4;
            ri.pColorAttachments = colors;
            ri.pDepthAttachment = depth ? &ds : nullptr;
            ri.pStencilAttachment = depth ? &ds : nullptr;
            if (s_profiling && s_passQueryNext + 2 <= 512)
            {
                char desc[256];
                int n = snprintf(desc, sizeof(desc), "area %ux%u@%d,%d", s_passArea.extent.width, s_passArea.extent.height,
                    s_passArea.offset.x, s_passArea.offset.y);
                for (int i = 0; i < 4; i++)
                    if (color[i])
                        n += snprintf(desc + n, sizeof(desc) - n, " C%d[b%u f%u p%u]", i, color[i]->base, color[i]->format, color[i]->pitch);
                if (depth)
                    n += snprintf(desc + n, sizeof(desc) - n, " D[b%u p%u]", depth->base, depth->pitch);
                vkCmdResetQueryPool(s_cmd, s_passQueries, s_passQueryNext, 2);
                vkCmdWriteTimestamp(s_cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, s_passQueries, s_passQueryNext);
                GapCounters now{ s_stats.resolves, s_stats.uploads, s_stats.submits, writewatch::GetGuardStats().uploadBytes };
                s_passInfos.push_back({ desc, 0, s_slot, s_passQueryNext, uint32_t(now.resolves - s_gapLast.resolves),
                    uint32_t(now.uploads - s_gapLast.uploads), uint32_t(now.submits - s_gapLast.submits), (now.shadowBytes - s_gapLast.shadowBytes) / 1024 });
                s_gapLast = now;
                s_passQueryNext += 2;
            }
            s_vk->cmdBeginRendering(s_cmd, &ri);
            s_passActive = true;
            s_stats.passes++;
            for (const PendingClear& pc : inPass)
            {
                VkClearAttachment ca{};
                ca.clearValue = pc.value;
                if (pc.rt == depth)
                    ca.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
                else
                {
                    ca.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
                    for (uint32_t i = 0; i < 4; i++)
                        if (color[i] == pc.rt)
                            ca.colorAttachment = i;
                }
                VkClearRect cr{ pc.rect, 0, 1 };
                vkCmdClearAttachments(s_cmd, 1, &ca, 1, &cr);
            }
        }

        // Before something reads `rt` outside a pass (a resolve): run its
        // pending clears. Given what it reads (`read`, target pixels; see
        // s_clearFlushRect), only those overlapping that, and older ones
        // overlapping those (they must land first); the rest stay pending for
        // the next pass on `rt`. Each tile resolve of the untiled scene reads
        // the band after the last tile's clear: four passes of their own a
        // frame, each a Metal render encoder and a FullBarrier.
        void FlushPendingClears(RenderTarget* rt, const VkRect2D* read = nullptr)
        {
            std::vector<PendingClear> kept;
            if (read)
            {
                // Newest first, so `run` (the read, then each clear that runs
                // now) holds every newer clear an older one must land under.
                std::vector<VkRect2D> run{ *read };
                for (size_t i = s_pendingClears.size(); i-- > 0;)
                {
                    if (s_pendingClears[i].rt != rt)
                        continue;
                    const VkRect2D rect = s_pendingClears[i].rect;
                    if (std::any_of(run.begin(), run.end(), [&](const VkRect2D& o) { return Overlaps(o, rect); }))
                        run.push_back(rect);
                    else
                    {
                        kept.insert(kept.begin(), s_pendingClears[i]);
                        s_pendingClears.erase(s_pendingClears.begin() + i);
                    }
                }
            }
            for (const PendingClear& pc : s_pendingClears)
                if (pc.rt == rt)
                {
                    RenderTarget* none[4] = {};
                    RenderTarget* color[4] = { rt };
                    BeginPass(rt->depth ? none : color, rt->depth ? rt : nullptr, pc.rect);
                    EndPass();
                    break;
                }
            // (In their order: the pass took every other clear of `rt`.)
            s_pendingClears.insert(s_pendingClears.end(), kept.begin(), kept.end());
        }

        // Levels 2-3 (see s_passBarrier): before a draw whose vertex shader may
        // read what a resolve wrote since the last full barrier (its vertex
        // data, the indices it fetches itself: `indexBytes` from `indexBase`,
        // a texture of `vsMasks` a resolve stored into), that barrier.
        // Mid-pass it ends the pass. Guest memory is no Metal-tracked binding
        // of a draw, so this is its only ordering.
        void VertexBarrierIfNeeded(const std::array<uint32_t, 3>& vsMasks, uint32_t indexBase, uint32_t indexBytes)
        {
            if (s_vertexPendingLo >= s_vertexPendingHi && s_vertexPendingTextures.empty())
                return;
            auto pending = [](uint32_t base, uint32_t size) {
                base &= 0x1FFFFFFF;
                uint32_t end = base + size;
                if (size == 0 || base >= s_vertexPendingHi || end <= s_vertexPendingLo)
                    return false;
                auto it = std::upper_bound(s_vertexPending.begin(), s_vertexPending.end(), base,
                    [](uint32_t b, const std::pair<uint32_t, uint32_t>& r) { return b < r.second; });
                return it != s_vertexPending.end() && it->first < end;
            };
            int why = -1;
            // (A texture's cache entry and image are keyed by its guest base.)
            for (uint32_t d = 0; d < 3 && why < 0 && !s_vertexPendingTextures.empty(); d++)
                for (uint32_t bits = vsMasks[d]; bits && why < 0; bits &= bits - 1)
                {
                    xenos::xe_gpu_texture_fetch_t f;
                    memcpy(&f, &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + uint32_t(__builtin_ctz(bits)) * 6], sizeof(f));
                    if (std::find(s_vertexPendingTextures.begin(), s_vertexPendingTextures.end(), (f.base_address << 12) & 0x1FFFFFFF) !=
                        s_vertexPendingTextures.end())
                        why = 2;
                }
            for (uint32_t word = 0; word < 3 && why < 0; word++)
                for (uint32_t bits = s_vs->info.vertexFetchConstants[word]; bits && why < 0; bits &= bits - 1)
                {
                    const uint32_t* vf = &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + (word * 32 + uint32_t(__builtin_ctz(bits))) * 2];
                    if (pending(vf[0] & 0x1FFFFFFC, ((vf[1] >> 2) & 0xFFFFFF) * 4))
                        why = 0;
                }
            if (why < 0 && pending(indexBase, indexBytes))
                why = 1;
            if (why < 0)
                return;
            static const char* const kWhy[3] = { "vertex data", "indices", "a texture" };
            LogOnce((0x2Full << 56) | (s_vs->hash & 0xFFFFFFFFFFFFFCull) | uint64_t(why),
                "vertex barrier (NFSMW_PASS_BARRIER): vs_%016llx reads %s a resolve wrote since the last full barrier%s",
                (unsigned long long)s_vs->hash, kWhy[why], s_passActive ? " (its pass is split)" : "");
            s_barrierCounts.vertex[why]++;
            s_barrierCounts.splits += s_passActive;
            EndPass();
            Begin();
            FullBarrier();
        }

        // ---------------------------------------------------------------
        // Pipelines

        struct PipelineKey
        {
            uint64_t vs, ps;
            uint32_t topology, restart;
            uint32_t colorFormats[4];
            uint32_t depthFormat;
            uint32_t blend[4];       // RB_BLENDCONTROL, or ~0u when blending is off
            uint32_t colorMask;      // 4 bits per target
            uint32_t depthControl;   // RB_DEPTHCONTROL (0 without a depth target, or one attached for the pass only)
            uint32_t raster;         // cull | front CW << 2 | depth clamp << 3 | depth bias << 4
            uint32_t pad;

            bool operator==(const PipelineKey& o) const { return memcmp(this, &o, sizeof(*this)) == 0; }
        };
        static_assert(sizeof(PipelineKey) == 80);

        struct PipelineKeyHash
        {
            size_t operator()(const PipelineKey& k) const
            {
                uint64_t words[sizeof(k) / 8];
                memcpy(words, &k, sizeof(k));
                uint64_t h = 0xCBF29CE484222325ull;
                for (uint64_t w : words)
                    h = (h ^ w) * 0x100000001B3ull ^ (h >> 29);
                return size_t(h);
            }
        };
        std::unordered_map<PipelineKey, VkPipeline, PipelineKeyHash> s_pipelines;

        VkBlendFactor BlendFactor(uint32_t f)
        {
            switch (f)
            {
            case 0: return VK_BLEND_FACTOR_ZERO;
            case 1: return VK_BLEND_FACTOR_ONE;
            case 4: return VK_BLEND_FACTOR_SRC_COLOR;
            case 5: return VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR;
            case 6: return VK_BLEND_FACTOR_SRC_ALPHA;
            case 7: return VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
            case 8: return VK_BLEND_FACTOR_DST_COLOR;
            case 9: return VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR;
            case 10: return VK_BLEND_FACTOR_DST_ALPHA;
            case 11: return VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA;
            case 12: return VK_BLEND_FACTOR_CONSTANT_COLOR;
            case 13: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR;
            case 14: return VK_BLEND_FACTOR_CONSTANT_ALPHA;
            case 15: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA;
            case 16: return VK_BLEND_FACTOR_SRC_ALPHA_SATURATE;
            default: return VK_BLEND_FACTOR_ONE;
            }
        }

        VkBlendOp BlendOp(uint32_t op)
        {
            switch (op)
            {
            case 1: return VK_BLEND_OP_SUBTRACT;
            case 2: return VK_BLEND_OP_MIN;
            case 3: return VK_BLEND_OP_MAX;
            case 4: return VK_BLEND_OP_REVERSE_SUBTRACT;
            default: return VK_BLEND_OP_ADD;
            }
        }

        // Compiles the pipeline for `key` (and the SPIR-V it needs). Runs on a
        // pipeline worker: touches only its arguments, the shaders' own
        // locked modules and the (internally synchronized) pipeline cache.
        VkPipeline BuildPipeline(const PipelineKey& key, VkPipelineLayout layout, Shader& vs, Shader& ps)
        {
            VkShaderModule vsModule = VertexModule(vs);
            if (!vsModule)
                return VK_NULL_HANDLE;
            VkPipelineShaderStageCreateInfo stages[2]{};
            stages[0] = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_VERTEX_BIT,
                vsModule, "main", nullptr };
            uint32_t outputs = 0;
            for (int i = 0; i < 4; i++)
                if (key.colorFormats[i] != 0)
                    outputs |= 1u << i;
            uint32_t psRegs = std::min(16u, ps.info.registerCount);
            uint32_t inputs = vs.info.interpolatorMask & ((1u << psRegs) - 1);
            VkShaderModule psModule = PixelVariant(ps, outputs, inputs, (key.raster >> 5) & 1);
            if (!psModule)
                return VK_NULL_HANDLE;
            stages[1] = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_FRAGMENT_BIT,
                psModule, "main", nullptr };

            VkPipelineVertexInputStateCreateInfo vi{ VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO };
            VkPipelineInputAssemblyStateCreateInfo ia{ VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO };
            ia.topology = VkPrimitiveTopology(key.topology);
            ia.primitiveRestartEnable = key.restart;
            VkPipelineViewportStateCreateInfo vp{ VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO };
            vp.viewportCount = 1;
            vp.scissorCount = 1;

            VkPipelineRasterizationStateCreateInfo rs{ VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO };
            rs.depthClampEnable = (key.raster >> 3) & 1;
            rs.polygonMode = VK_POLYGON_MODE_FILL;
            rs.cullMode = key.raster & 3;
            rs.frontFace = (key.raster >> 2) & 1 ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE;
            rs.depthBiasEnable = (key.raster >> 4) & 1;
            rs.lineWidth = 1.0f;

            VkPipelineMultisampleStateCreateInfo ms{ VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO };
            ms.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;

            VkPipelineDepthStencilStateCreateInfo dss{ VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO };
            if (key.depthFormat != 0)
            {
                reg::RB_DEPTHCONTROL dc{};
                dc.value = key.depthControl;
                dss.depthTestEnable = dc.z_enable;
                dss.depthWriteEnable = dc.z_enable && dc.z_write_enable;
                dss.depthCompareOp = VkCompareOp(dc.zfunc);
                dss.stencilTestEnable = dc.stencil_enable;
                dss.front = { VkStencilOp(dc.stencilfail), VkStencilOp(dc.stencilzpass), VkStencilOp(dc.stencilzfail),
                    VkCompareOp(dc.stencilfunc), 0xFF, 0xFF, 0 };
                if (dc.backface_enable)
                    dss.back = { VkStencilOp(dc.stencilfail_bf), VkStencilOp(dc.stencilzpass_bf), VkStencilOp(dc.stencilzfail_bf),
                        VkCompareOp(dc.stencilfunc_bf), 0xFF, 0xFF, 0 };
                else
                    dss.back = dss.front;
            }

            VkPipelineColorBlendAttachmentState att[4]{};
            VkFormat formats[4];
            for (int i = 0; i < 4; i++)
            {
                formats[i] = VkFormat(key.colorFormats[i]);
                if (key.colorFormats[i] == 0)
                    continue;
                att[i].colorWriteMask = (key.colorMask >> (4 * i)) & 0xF;
                if (key.blend[i] != ~0u)
                {
                    reg::RB_BLENDCONTROL bc{};
                    bc.value = key.blend[i];
                    att[i].blendEnable = VK_TRUE;
                    att[i].srcColorBlendFactor = BlendFactor(uint32_t(bc.color_srcblend));
                    att[i].dstColorBlendFactor = BlendFactor(uint32_t(bc.color_destblend));
                    att[i].colorBlendOp = BlendOp(uint32_t(bc.color_comb_fcn));
                    att[i].srcAlphaBlendFactor = BlendFactor(uint32_t(bc.alpha_srcblend));
                    att[i].dstAlphaBlendFactor = BlendFactor(uint32_t(bc.alpha_destblend));
                    att[i].alphaBlendOp = BlendOp(uint32_t(bc.alpha_comb_fcn));
                }
            }
            VkPipelineColorBlendStateCreateInfo cb{ VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO };
            cb.attachmentCount = 4;
            cb.pAttachments = att;

            VkDynamicState dyn[] = { VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR, VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
                VK_DYNAMIC_STATE_STENCIL_WRITE_MASK, VK_DYNAMIC_STATE_STENCIL_REFERENCE, VK_DYNAMIC_STATE_BLEND_CONSTANTS,
                VK_DYNAMIC_STATE_DEPTH_BIAS };
            VkPipelineDynamicStateCreateInfo ds{ VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO };
            ds.dynamicStateCount = uint32_t(std::size(dyn));
            ds.pDynamicStates = dyn;

            VkPipelineRenderingCreateInfoKHR rci{ VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR };
            rci.colorAttachmentCount = 4;
            rci.pColorAttachmentFormats = formats;
            rci.depthAttachmentFormat = VkFormat(key.depthFormat);
            rci.stencilAttachmentFormat = VkFormat(key.depthFormat);
            // NFSMW_SHADING_RATE (an experiment, video/presenter.h): every draw
            // shades once per w x h pixels.
            VkPipelineFragmentShadingRateStateCreateInfoKHR rate{ VK_STRUCTURE_TYPE_PIPELINE_FRAGMENT_SHADING_RATE_STATE_CREATE_INFO_KHR };
            if (s_vk->shadingRateW)
            {
                rate.fragmentSize = { s_vk->shadingRateW, s_vk->shadingRateH };
                rate.combinerOps[0] = rate.combinerOps[1] = VK_FRAGMENT_SHADING_RATE_COMBINER_OP_KEEP_KHR;
                rci.pNext = &rate;
            }

            VkGraphicsPipelineCreateInfo gpci{ VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO };
            gpci.pNext = &rci;
            gpci.stageCount = 2;
            gpci.pStages = stages;
            gpci.pVertexInputState = &vi;
            gpci.pInputAssemblyState = &ia;
            gpci.pViewportState = &vp;
            gpci.pRasterizationState = &rs;
            gpci.pMultisampleState = &ms;
            gpci.pDepthStencilState = &dss;
            gpci.pColorBlendState = &cb;
            gpci.pDynamicState = &ds;
            gpci.layout = layout;
            VkPipeline pipeline = VK_NULL_HANDLE;
            VkResult r = vkCreateGraphicsPipelines(s_dev, s_pipelineCache, 1, &gpci, nullptr, &pipeline);
            if (r != VK_SUCCESS)
            {
                fprintf(stderr, "[renderer] pipeline vs %016llx ps %016llx failed: %d\n",
                    (unsigned long long)key.vs, (unsigned long long)key.ps, int(r));
                pipeline = VK_NULL_HANDLE;
            }
            return pipeline;
        }

        // Pipelines are compiled on worker threads. A new one costs 20-80 ms
        // on RADV (it was 100-460 ms before the vertex shader's single call
        // site), and a command processor waiting on it stalls the game: a
        // traffic car streaming in mid-race froze it for 2 s. The command
        // processor waits briefly (NFSMW_PIPELINE_WAIT_US a frame, default
        // 3000: disk cache hits make it), then skips the draw until it is
        // ready, as Xenia does. For 2 s after a loading screen (a frame of
        // under 64 draws) it waits in full, so scene setup, including
        // anything the game renders once, is drawn exactly.
        // NFSMW_PIPELINE_THREADS=<n> (default: a third of the cores, at most
        // 4; 0 compiles on the command processor). Settings > Advanced >
        // Asynchronous Shaders off (NFSMW_ASYNC_PIPELINES=0): always wait.
        // NFSMW_PIPELINE_LOG=1 logs each pipeline's timing.
        struct PendingPipeline
        {
            std::mutex mutex;
            std::condition_variable cv;
            bool done = false;
            VkPipeline pipeline = VK_NULL_HANDLE;
            std::chrono::steady_clock::time_point queued = std::chrono::steady_clock::now();
            double createMs = 0;
            uint64_t queuedFrame = 0;
            bool outlasted = false;  // outlasted a frame's wait budget: a real compile, not worth waiting for again
        };
        std::unordered_map<PipelineKey, std::shared_ptr<PendingPipeline>, PipelineKeyHash> s_pendingPipelines;
        const uint32_t s_pipelineThreads = [] {
            if (const char* v = std::getenv("NFSMW_PIPELINE_THREADS"))
                return uint32_t(std::atoi(v));
            return std::clamp(std::thread::hardware_concurrency() / 3, 1u, 4u);
        }();
        const int64_t s_pipelineWaitUs = [] { const char* v = std::getenv("NFSMW_PIPELINE_WAIT_US"); return v ? int64_t(std::atoll(v)) : 3000; }();
        const bool s_logPipelines = std::getenv("NFSMW_PIPELINE_LOG") != nullptr;
        uint64_t s_frame = 0, s_drawsAtSwap = 0, s_waitPipelinesUntil = 120;  // boot: wait in full
        uint32_t s_pipelineSkips = 0;         // draws skipped this frame, their pipeline not ready
        int64_t s_pipelineWaitLeftUs = 0;     // this frame's wait budget left

        std::mutex s_compileMutex;
        std::condition_variable s_compileCv;
        std::deque<std::function<void()>> s_compileQueue;
        uint32_t s_compileActive = 0;         // jobs running (under s_compileMutex)
        bool s_compileStopping = false;

        void CompileWorker()
        {
            SetHostThreadName("nfsmw-pipelines");
            hostcpu::LeaveReservedCore();  // started by the command processor
            while (true)
            {
                std::function<void()> job;
                {
                    std::unique_lock lock(s_compileMutex);
                    s_compileCv.wait(lock, [] { return s_compileStopping || !s_compileQueue.empty(); });
                    if (s_compileStopping)
                        return;
                    job = std::move(s_compileQueue.front());
                    s_compileQueue.pop_front();
                    s_compileActive++;
                }
                job();
                {
                    std::lock_guard lock(s_compileMutex);
                    s_compileActive--;
                }
                s_compileCv.notify_all();
            }
        }

        // The pipeline for `key`; VK_NULL_HANDLE with `pending` set while a
        // worker still compiles it (the caller skips the draw).
        VkPipeline GetPipeline(const PipelineKey& key, VkPipelineLayout layout, bool& pending)
        {
            pending = false;
            static PipelineKey lastKey{};
            static VkPipeline lastPipeline = VK_NULL_HANDLE;
            if (lastPipeline != VK_NULL_HANDLE && key == lastKey)
                return lastPipeline;
            auto it = s_pipelines.find(key);
            if (it != s_pipelines.end())
            {
                lastKey = key;
                lastPipeline = it->second;
                return it->second;
            }
            std::shared_ptr<PendingPipeline> job;
            if (auto p = s_pendingPipelines.find(key); p != s_pendingPipelines.end())
                job = p->second;
            else
            {
                s_hitch.pipelines++;
                job = std::make_shared<PendingPipeline>();
                job->queuedFrame = s_frame;
                s_pendingPipelines[key] = job;
                Shader* vs = s_shaders[key.vs].get();
                Shader* ps = s_shaders[key.ps].get();
                auto build = [job, key, layout, vs, ps] {
                    auto start = std::chrono::steady_clock::now();
                    VkPipeline pipeline = BuildPipeline(key, layout, *vs, *ps);
                    {
                        std::lock_guard lock(job->mutex);
                        job->pipeline = pipeline;
                        job->createMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
                        job->done = true;
                    }
                    job->cv.notify_all();
                };
                if (s_pipelineThreads == 0)
                {
                    HitchTimer hitchTimer(s_hitch.pipelineMs);
                    build();
                }
                else
                {
                    static std::once_flag started;
                    std::call_once(started, [] {
                        for (uint32_t i = 0; i < s_pipelineThreads; i++)
                            std::thread(CompileWorker).detach();
                    });
                    {
                        std::lock_guard lock(s_compileMutex);
                        s_compileQueue.push_back(std::move(build));
                    }
                    s_compileCv.notify_all();
                }
            }
            {
                HitchTimer hitchTimer(s_hitch.pipelineMs);
                CpWaitTimer wait(CP_WAIT_PIPELINE);
                std::unique_lock lock(job->mutex);
                // Waits share one budget a frame (cache hits make it); once it
                // is spent, draws just check. A per-draw wait made a pipeline
                // used by 13 draws cost 47 ms a frame while it compiled.
                if (!settings::GetBool(settings::Id::AsyncShaders) || s_frame < s_waitPipelinesUntil)
                    job->cv.wait(lock, [&] { return job->done; });
                else
                {
                    // The budget catches compiles that finish within a few
                    // milliseconds (cache hits); one that outlasted it once is
                    // only checked afterwards, not waited for every frame
                    // (that cost 3 ms a frame for as long as it compiled).
                    auto start = std::chrono::steady_clock::now();
                    int64_t budget = job->outlasted ? 0 : std::max<int64_t>(s_pipelineWaitLeftUs, 0);
                    bool ready = job->cv.wait_for(lock, std::chrono::microseconds(budget), [&] { return job->done; });
                    s_pipelineWaitLeftUs -= std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start).count();
                    if (!ready && budget > 0)
                        job->outlasted = true;
                    if (!ready)
                    {
                        pending = true;
                        s_pipelineSkips++;
                        return VK_NULL_HANDLE;
                    }
                }
            }
            if (s_logPipelines)
                fprintf(stderr, "[pipeline] vs %016llx ps %016llx: %.1f ms to compile, ready %.1f ms and %llu frames after the first draw\n",
                    (unsigned long long)key.vs, (unsigned long long)key.ps, job->createMs,
                    std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - job->queued).count(),
                    (unsigned long long)(s_frame - job->queuedFrame));
            s_stats.pipelines++;
            s_pipelines[key] = job->pipeline;
            s_pendingPipelines.erase(key);
            return job->pipeline;
        }

        // ---------------------------------------------------------------
        // Scissor (Xenia's draw_util GetScissor)

        struct Rect { int32_t x0, y0, x1, y1; };

        Rect GuestScissor()
        {
            auto tl = Reg<reg::PA_SC_WINDOW_SCISSOR_TL>();
            auto br = Reg<reg::PA_SC_WINDOW_SCISSOR_BR>();
            Rect r{ int32_t(tl.tl_x), int32_t(tl.tl_y), int32_t(br.br_x), int32_t(br.br_y) };
            if (!tl.window_offset_disable)
            {
                auto wo = Reg<reg::PA_SC_WINDOW_OFFSET>();
                r.x0 = std::max(r.x0 + wo.window_x_offset, 0);
                r.y0 = std::max(r.y0 + wo.window_y_offset, 0);
                r.x1 = std::max(r.x1 + wo.window_x_offset, 0);
                r.y1 = std::max(r.y1 + wo.window_y_offset, 0);
            }
            auto stl = Reg<reg::PA_SC_SCREEN_SCISSOR_TL>();
            auto sbr = Reg<reg::PA_SC_SCREEN_SCISSOR_BR>();
            r.x0 = std::max(r.x0, int32_t(stl.tl_x));
            r.y0 = std::max(r.y0, int32_t(stl.tl_y));
            r.x1 = std::min(r.x1, int32_t(sbr.br_x));
            r.y1 = std::min(r.y1, int32_t(sbr.br_y));
            r.x1 = std::max(r.x1, r.x0);
            r.y1 = std::max(r.y1, r.y0);
            return r;
        }

        // NFSMW_UNTILED_AREA on for this frame (=alt: in odd 120-frame windows).
        bool UntiledAreaOn()
        {
            return s_untiledArea == 1 || (s_untiledArea == 2 && s_frame / 120 % 2 == 1);
        }

        // Where an untiled draw into `targets` (4 colour, then depth; any
        // null) may draw, in guest pixels of their common w x h (the caller
        // clamps to it): tile 0 draws the whole frame, so the whole target.
        // With NFSMW_UNTILED_AREA, one clipped to its viewport (x and y
        // scaled and offset, clipping on) only the pixels whose centres lie
        // inside the viewport (a counted draw's rule, in Draw). No scissor:
        // they are tile 0's, and the rows and columns past tile 0 are drawn
        // by tile 0 alone (the window scissor is the tile: for side-by-side
        // tiles only the left one's columns, and the right half went black
        // when they were drawn to it; D3D leaves the screen scissor
        // 8192x8192). `why`: 0 cut to the viewport, 1 clipping off, 2 no
        // viewport transform (or garbage in it), 3 a target that fell back
        // to the whole target (wholeArea), 4 off.
        Rect UntiledScissor(RenderTarget* const targets[5], int32_t w, int32_t h, int* why = nullptr)
        {
            Rect r{ 0, 0, w, h };
            int reason = 4;
            if (UntiledAreaOn())
            {
                float ax = std::fabs(RegF(XE_GPU_REG_PA_CL_VPORT_XSCALE)), ox = RegF(XE_GPU_REG_PA_CL_VPORT_XOFFSET);
                float ay = std::fabs(RegF(XE_GPU_REG_PA_CL_VPORT_YSCALE)), oy = RegF(XE_GPU_REG_PA_CL_VPORT_YOFFSET);
                if (Reg<reg::PA_CL_CLIP_CNTL>().clip_disable)
                    reason = 1;
                else if ((Reg<reg::PA_CL_VTE_CNTL>().value & 0xF) != 0xF || !std::isfinite(ax + ox + ay + oy))
                    reason = 2;
                else if (std::any_of(targets, targets + 5, [](RenderTarget* rt) { return rt && rt->wholeArea; }))
                    reason = 3;
                else
                    reason = 0;
                if (reason == 0)
                {
                    // As the host viewport: the window offset (0 for tile 0)
                    // and D3D's integer pixel centres. |scale|: y is flipped.
                    if (Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable)
                    {
                        auto wo = Reg<reg::PA_SC_WINDOW_OFFSET>();
                        ox += float(wo.window_x_offset);
                        oy += float(wo.window_y_offset);
                    }
                    if (Reg<reg::PA_SU_VTX_CNTL>().pix_center == xenos::PixelCenter::kD3DZero)
                    {
                        ox += 0.5f;
                        oy += 0.5f;
                    }
                    auto edge = [](float v, int32_t hi) { return int32_t(std::fmax(0.0f, std::fmin(std::ceil(v - 0.5f), float(hi)))); };
                    r = { edge(ox - ax, w), edge(oy - ay, h), edge(ox + ax, w), edge(oy + ay, h) };
                }
            }
            if (why)
                *why = reason;
            return r;
        }

        // ---------------------------------------------------------------
        // Indices

        // Index `i` of the current draw, as the vertex shader's fetchIndex.
        struct IndexSource
        {
            bool indexed;
            uint32_t base;       // physical
            bool is32;
            uint32_t endian;

            uint32_t Get(uint32_t i) const
            {
                if (!indexed)
                    return i;
                if (is32)
                    return GpuSwap(LoadPhysical(base + i * 4), endian) & 0xFFFFFF;
                uint32_t addr = base + i * 2;
                uint32_t w = GpuSwap(LoadPhysical(addr & ~3u), endian);
                bool firstInHigh = endian == 2 || endian == 3;
                bool high = ((addr >> 1) & 1) != firstInHigh;
                return high ? (w >> 16) : (w & 0xFFFF);
            }
        };

        // Get() for a whole draw, written straight into the ring: `out16`
        // keeps 16-bit indices (the pipeline's restart index is then 0xFFFF,
        // so only when restart is off or the guest's reset index is 0xFFFF);
        // otherwise 32-bit, with the reset index mapped to ~0 when `restart`.
        // For 16-bit indices, 8in16 and 8in32 swap each index's bytes; none
        // and 16in32 keep them (Get() picks the other half of the swapped word).
        void ConvertIndices(uint8_t* dst, const IndexSource& src, uint32_t count, bool out16, bool restart, uint32_t resetIndex)
        {
            if (!src.is32)
            {
                const uint8_t* p = s_sharedHost + (src.base & 0x1FFFFFFE);
                bool swap = src.endian == 1 || src.endian == 2;
                if (out16)
                {
                    if (!swap)
                    {
                        memcpy(dst, p, size_t(count) * 2);
                        return;
                    }
                    for (uint32_t i = 0; i < count; i++)
                    {
                        uint16_t v;
                        memcpy(&v, p + size_t(i) * 2, 2);
                        v = __builtin_bswap16(v);
                        memcpy(dst + size_t(i) * 2, &v, 2);
                    }
                    return;
                }
                for (uint32_t i = 0; i < count; i++)
                {
                    uint16_t v;
                    memcpy(&v, p + size_t(i) * 2, 2);
                    uint32_t x = swap ? __builtin_bswap16(v) : v;
                    x = restart && x == resetIndex ? 0xFFFFFFFFu : x;
                    memcpy(dst + size_t(i) * 4, &x, 4);
                }
                return;
            }
            const uint8_t* p = s_sharedHost + (src.base & 0x1FFFFFFC);
            for (uint32_t i = 0; i < count; i++)
            {
                uint32_t w;
                memcpy(&w, p + size_t(i) * 4, 4);
                uint32_t v = GpuSwap(w, src.endian) & 0xFFFFFF;
                v = restart && v == resetIndex ? 0xFFFFFFFFu : v;
                memcpy(dst + size_t(i) * 4, &v, 4);
            }
        }


        // ---------------------------------------------------------------
        // Textures: 2D, cube and 3D, with the guest's mip levels (2D, cube).
        // Validity comes from write watches (kernel/write_watch.h): a
        // texture reloads when any page it covers was written (a CPU store,
        // a file read or a resolve) since it was last loaded.

        struct TextureFormatInfo
        {
            VkFormat vk;
            uint32_t bpbLog2;   // guest bytes per block
            uint32_t block;     // block width/height in texels
            uint32_t convert;   // 0 copy; 1 565, 2 1555, 3 4444 -> RGBA8; 4 24_8 -> R32F
        };

        bool GetTextureFormat(uint32_t f, TextureFormatInfo& out)
        {
            switch (f)
            {
            case 2: out = { VK_FORMAT_R8_UNORM, 0, 1, 0 }; return true;
            case 3: out = { VK_FORMAT_R8G8B8A8_UNORM, 1, 1, 2 }; return true;
            case 4: out = { VK_FORMAT_R8G8B8A8_UNORM, 1, 1, 1 }; return true;
            case 6: case 14: out = { VK_FORMAT_R8G8B8A8_UNORM, 2, 1, 0 }; return true;
            case 7: out = { VK_FORMAT_A2B10G10R10_UNORM_PACK32, 2, 1, 0 }; return true;
            case 10: out = { VK_FORMAT_R8G8_UNORM, 1, 1, 0 }; return true;
            case 15: out = { VK_FORMAT_R8G8B8A8_UNORM, 1, 1, 3 }; return true;
            case 18: out = { VK_FORMAT_BC1_RGBA_UNORM_BLOCK, 3, 4, 0 }; return true;
            case 19: out = { VK_FORMAT_BC2_UNORM_BLOCK, 4, 4, 0 }; return true;
            case 20: out = { VK_FORMAT_BC3_UNORM_BLOCK, 4, 4, 0 }; return true;
            case 22: case 23: out = { VK_FORMAT_R32_SFLOAT, 2, 1, 4 }; return true;
            case 24: out = { VK_FORMAT_R16_UNORM, 1, 1, 0 }; return true;
            case 25: out = { VK_FORMAT_R16G16_UNORM, 2, 1, 0 }; return true;
            case 26: out = { VK_FORMAT_R16G16B16A16_UNORM, 3, 1, 0 }; return true;
            case 30: out = { VK_FORMAT_R16_SFLOAT, 1, 1, 0 }; return true;
            case 31: out = { VK_FORMAT_R16G16_SFLOAT, 2, 1, 0 }; return true;
            case 32: out = { VK_FORMAT_R16G16B16A16_SFLOAT, 3, 1, 0 }; return true;
            case 36: out = { VK_FORMAT_R32_SFLOAT, 2, 1, 0 }; return true;
            case 37: out = { VK_FORMAT_R32G32_SFLOAT, 3, 1, 0 }; return true;
            case 38: out = { VK_FORMAT_R32G32B32A32_SFLOAT, 4, 1, 0 }; return true;
            case 49: out = { VK_FORMAT_BC5_UNORM_BLOCK, 4, 4, 0 }; return true;
            default: return false;
            }
        }

        // Stored as resolves write it: one 32-bit word a texel, unconverted
        // (the formats a scaled texture can have).
        bool ScaledFormat(const TextureFormatInfo& fi)
        {
            return fi.bpbLog2 == 2 && fi.convert == 0 && fi.block == 1;
        }

        uint32_t Tiled2D(uint32_t x, uint32_t y, uint32_t pitchAligned, uint32_t bppLog2)
        {
            uint32_t outer = (((y >> 5) * (pitchAligned >> 5)) + (x >> 5)) << 6;
            uint32_t inner = (((y >> 1) & 7) << 3) | (x & 7);
            uint32_t oi = (outer | inner) << bppLog2;
            uint32_t bank = (y >> 4) & 1;
            uint32_t pipe = ((x >> 3) & 3) ^ (((y >> 3) & 1) << 1);
            return ((y & 1) << 4) | (pipe << 6) | (bank << 11) | (oi & 0xF) | (((oi >> 4) & 1) << 5) |
                (((oi >> 5) & 7) << 8) | ((oi >> 8) << 12);
        }

        uint32_t Log2Ceil(uint32_t v)
        {
            return v <= 1 ? 0 : 32 - __builtin_clz(v - 1);
        }

        // Mip levels (Xenia's texture_util GetGuestTextureLayout and
        // GetPackedMipOffset, for 2D and cube textures). Levels 1.. are
        // stored one after another from mip_address, each (per face) with a
        // pitch and height from the base size rounded up to a power of two,
        // shifted to the level and aligned to 32x32 blocks (linear rows to
        // 256 bytes), faces 4 KB-aligned. With packed mips, the levels from
        // the one whose short side is 16 texels (packedLevel) share that
        // level's storage, each at an offset in its 32x32 tile; when the
        // base itself is that small (packedLevel 0), the mips' shared tile
        // is at mip_address.
        struct MipLayout
        {
            uint32_t width, height, block, bpbLog2, layers;
            bool tiled, packed;
            uint32_t packedLevel;

            uint32_t StorageLevel(uint32_t level) const { return packed ? std::min(level, packedLevel) : level; }
            uint32_t PitchBlocks(uint32_t storage) const  // 32-aligned
            {
                uint32_t texels = std::max(std::bit_ceil(width) >> storage, 1u);
                return (((texels + block - 1) / block) + 31) & ~31u;
            }
            uint32_t RowsBlocks(uint32_t storage) const  // 32-aligned
            {
                uint32_t texels = std::max(std::bit_ceil(height) >> storage, 1u);
                return (((texels + block - 1) / block) + 31) & ~31u;
            }
            uint32_t RowBytes(uint32_t storage) const  // linear
            {
                return ((PitchBlocks(storage) << bpbLog2) + 255) & ~255u;
            }
            uint32_t SliceStride(uint32_t storage) const
            {
                uint32_t bytes = tiled ? (PitchBlocks(storage) * RowsBlocks(storage)) << bpbLog2 : RowBytes(storage) * RowsBlocks(storage);
                return (bytes + 4095) & ~4095u;
            }
            uint32_t Offset(uint32_t storage) const  // from mip_address
            {
                uint32_t offset = 0;
                for (uint32_t s = 1; s < storage; s++)
                    offset += SliceStride(s) * layers;
                return offset;
            }
            // The level's position in its storage, in blocks.
            void PackedOffset(uint32_t level, uint32_t& x, uint32_t& y) const
            {
                x = y = 0;
                if (!packed || level < packedLevel)
                    return;
                uint32_t log2w = Log2Ceil(width), log2h = Log2Ceil(height);
                uint32_t log2Size = std::min(log2w, log2h);
                uint32_t base = log2Size > 4 ? log2Size - 4 : 0;
                uint32_t sub = level - base;
                if (sub < 3)
                {
                    if (log2w > log2h)
                        y = 16 >> sub;
                    else
                        x = 16 >> sub;
                }
                else if (log2w > log2h)
                    x = (1u << (log2w - base)) >> (sub - 2);
                else
                    y = (1u << (log2h - base)) >> (sub - 2);
                x /= block;
                y /= block;
            }
        };

        // Texture loading runs on the GPU: a compute shader untiles and
        // converts from guest memory into the ring, then a buffer->image copy.
        // Ordering against resolves is then just a barrier (no CPU wait).
        // ---------------------------------------------------------------
        // Scaled resolve memory (internal resolution above 1x)
        //
        // A resolve from a scaled render target writes each guest pixel's
        // host samples here, besides the 1x average into guest memory
        // (which the CPU, 1x draws and the auto-exposure readback keep
        // using). Storage is per 4 KB guest page: a slot of 4096 x s bytes
        // (s = sx x sy host pixels per guest pixel: 9/4 at 1.5x) in one pool
        // buffer, found through a page table (a GPU buffer and its CPU copy),
        // as MoltenVK has no sparse buffers. At a whole scale the samples of
        // the 32-bit texel at guest byte address A are at
        //   slot(A >> 12) x 4096 x s + (A & 4095) x s + (j x sx + i) x 4;
        // at 1.5x a 2x2 block of texels keeps its 3x3 samples together
        // (gpu/render_scale.h, sbSampleWord, has the layout).
        // A page's scaled copy is current while the last write to the page
        // was a scaled resolve: its write sequence is still the one recorded
        // right after that resolve (a CPU write, a 1x resolve or any other
        // GPU write moves it on). Draws into scaled targets then sample
        // scaled textures made from it, and the front buffer is presented at
        // host resolution. NFSMW_SCALED_RESOLVE=0 keeps everything 1x (the
        // targets are still scaled: supersampling). NFSMW_SCALED_MB sizes the
        // pool (default 512). Scaled textures are stored into straight from
        // the pool through an R32_UINT view; NFSMW_SCALED_DIRECT=0 (or a
        // device that can't) loads them through a scratch buffer and a copy.
        // Scaled resolves write the pool coalesced (see the resolve shader;
        // not on Apple GPUs); NFSMW_SCALED_COALESCE=0 has each thread store
        // its own samples, =1 coalesces on any GPU.
        constexpr uint32_t kGuestPages = uint32_t(kSharedSize >> 12);
        struct ScaledMemory
        {
            bool tried = false, enabled = false, direct = false, coalesce = false;
            gpu::RenderScale scale;  // the scale its samples are at
            uint32_t slotWords = 0, slots = 0;
            VkBuffer pool = VK_NULL_HANDLE, table = VK_NULL_HANDLE, scratch = VK_NULL_HANDLE;
            VkDeviceMemory poolMemory = VK_NULL_HANDLE, tableMemory = VK_NULL_HANDLE, scratchMemory = VK_NULL_HANDLE;
            VkDeviceSize scratchSize = 0;  // also the largest scaled texture (no buffer when direct)
            VkDeviceSize poolBytes = 0, tableBytes = 0, scratchBytes = 0;  // allocated (the [perf] line's memory)
            std::vector<uint32_t> pageSlot;  // slot + 1; 0: none
            std::vector<uint64_t> pageSeq;   // write sequence after the page's last scaled resolve; 0: none or stale
            std::vector<uint32_t> slotPage;  // page + 1; 0: free
            std::vector<uint8_t> slotRef;    // clock bit (used since the hand last passed)
            uint32_t clock = 0;
            std::vector<uint32_t> dirty;     // pages whose table entry changed
            uint64_t allocations = 0, evictions = 0, textures = 0, fallbacks = 0;
            uint64_t loads = 0, untiledBytes = 0, resolvedBytes = 0;  // for the [perf] line
            uint32_t hostPageGuestPages = 1;  // guest pages per host page (write sequences are per host page)
        } s_scaled;

        bool EnsureScaledMemory();
        void FillFreshSlots(const std::vector<std::pair<uint32_t, uint32_t>>& fresh);

        // The page's slot, allocated (clock replacement) if it has none.
        uint32_t ScaledSlot(uint32_t page, bool& fresh)
        {
            fresh = false;
            uint32_t& entry = s_scaled.pageSlot[page];
            if (entry)
            {
                s_scaled.slotRef[entry - 1] = 1;
                return entry - 1;
            }
            uint32_t slot;
            for (;;)
            {
                slot = s_scaled.clock;
                s_scaled.clock = (slot + 1) % s_scaled.slots;
                if (!s_scaled.slotPage[slot])
                    break;
                if (s_scaled.slotRef[slot])
                {
                    s_scaled.slotRef[slot] = 0;
                    continue;
                }
                // Evict: that page falls back to its 1x copy. (Command
                // buffers are ordered by full barriers, so earlier work is
                // done with the slot before the table changes.)
                uint32_t old = s_scaled.slotPage[slot] - 1;
                s_scaled.pageSlot[old] = 0;
                s_scaled.pageSeq[old] = 0;
                s_scaled.dirty.push_back(old);
                s_scaled.evictions++;
                break;
            }
            s_scaled.slotPage[slot] = page + 1;
            s_scaled.slotRef[slot] = 1;
            entry = slot + 1;
            s_scaled.dirty.push_back(page);
            s_scaled.allocations++;
            fresh = true;
            return slot;
        }

        // Upload changed page-table entries (in the command stream, so the
        // GPU sees them in order).
        void FlushScaledTable()
        {
            auto& dirty = s_scaled.dirty;
            if (dirty.empty())
                return;
            std::sort(dirty.begin(), dirty.end());
            dirty.erase(std::unique(dirty.begin(), dirty.end()), dirty.end());
            EndPass();
            Begin();
            for (size_t i = 0; i < dirty.size();)
            {
                size_t j = i + 1;
                while (j < dirty.size() && dirty[j] == dirty[j - 1] + 1 && j - i < 16384)
                    j++;
                vkCmdUpdateBuffer(s_cmd, s_scaled.table, VkDeviceSize(dirty[i]) * 4, VkDeviceSize(j - i) * 4,
                    &s_scaled.pageSlot[dirty[i]]);
                i = j;
            }
            dirty.clear();
            FullBarrier();
        }

        // The page's scaled copy is current (see above).
        bool ScaledPageValid(uint32_t page)
        {
            uint64_t seq = s_scaled.pageSeq[page];
            return seq != 0 && s_scaled.pageSlot[page] != 0 && writewatch::WriteSequence(page << 12, 4096) == seq;
        }

        // Every page of [base, base + size) has a current scaled copy.
        bool ScaledRangeValid(uint32_t base, uint32_t size)
        {
            if (!s_scaled.enabled || size == 0)
                return false;
            uint32_t first = base >> 12, last = std::min<uint32_t>((base + size - 1) >> 12, kGuestPages - 1);
            for (uint32_t page = first; page <= last; page++)
                if (!ScaledPageValid(page))
                    return false;
            for (uint32_t page = first; page <= last; page++)
                s_scaled.slotRef[s_scaled.pageSlot[page] - 1] = 1;
            return true;
        }

        const char* kUntileGlsl = R"(#version 460
layout(local_size_x = 8, local_size_y = 8) in;
layout(std430, set = 0, binding = 0) readonly buffer SharedMemory { uint g_mem[]; };)" GLSL_SHARED_MEMORY_READ R"(
#ifdef TO_IMAGE
// 1x textures of 32-bit texels, stored into the image through an R32_UINT view
// (no staging copy): x is the texel, as the staging row's word index.
layout(set = 0, binding = 1, r32ui) writeonly uniform uimage2D o_image;
#define STORE(x, v) imageStore(o_image, ivec2(int(x), int(gy)), uvec4((v), 0u, 0u, 0u))
#else
layout(std430, set = 0, binding = 1) writeonly buffer Staging { uint o_mem[]; };
#define STORE(x, v) o_mem[row + (x)] = (v)
#endif
layout(push_constant) uniform P
{
    uint srcBase, tiled, pitch, bpbLog2;   // pitch: tiled = blocks (32-aligned), linear = bytes
    uint offX, offY, width, height;        // blocks
    uint endian, convert, dstWord, dstRowWords;
    uint z, heightAligned;                 // tiled == 2: 3D slice and aligned height (blocks)
} p;

uint tiledCombine(uint oi, uint bank, uint pipe, uint yLsb)
{
    return (yLsb << 4) | (pipe << 6) | (bank << 11) | (oi & 0xFu) | (((oi >> 4) & 1u) << 5) |
        (((oi >> 5) & 7u) << 8) | ((oi >> 8) << 12);
}

uint tiled3D(uint x, uint y, uint z, uint pitch, uint height, uint bppLog2)
{
    uint outer = ((((z >> 2) * (height >> 4) + (y >> 4)) * (pitch >> 5)) + (x >> 5)) << 7;
    uint inner = ((z & 3u) << 5) | (((y >> 1) & 3u) << 3) | (x & 7u);
    uint bank = ((y >> 3) ^ (z >> 2)) & 1u;
    uint pipe = ((x >> 3) & 3u) ^ (bank << 1);
    return tiledCombine((outer | inner) << bppLog2, bank, pipe, y & 1u);
}

uint tiled2D(uint x, uint y, uint pitch, uint bppLog2)
{
    uint outer = (((y >> 5) * (pitch >> 5)) + (x >> 5)) << 6;
    uint inner = (((y >> 1) & 7u) << 3) | (x & 7u);
    uint oi = (outer | inner) << bppLog2;
    uint bank = (y >> 4) & 1u;
    uint pipe = ((x >> 3) & 3u) ^ (((y >> 3) & 1u) << 1);
    return ((y & 1u) << 4) | (pipe << 6) | (bank << 11) | (oi & 0xFu) | (((oi >> 4) & 1u) << 5) |
        (((oi >> 5) & 7u) << 8) | ((oi >> 8) << 12);
}

uint swapWord(uint v, uint e)
{
    if (e == 1u) return ((v << 8) & 0xFF00FF00u) | ((v >> 8) & 0x00FF00FFu);
    if (e == 2u) return (v << 24) | ((v << 8) & 0x00FF0000u) | ((v >> 8) & 0x0000FF00u) | (v >> 24);
    if (e == 3u) return (v >> 16) | (v << 16);
    return v;
}

uint addr(uint bx, uint by)
{
    uint x = bx + p.offX, y = by + p.offY;
    if (p.tiled == 2u)
        return p.srcBase + tiled3D(x, y, p.z, p.pitch, p.heightAligned, p.bpbLog2);
    return p.srcBase + (p.tiled != 0u ? tiled2D(x, y, p.pitch, p.bpbLog2) : y * p.pitch + (x << p.bpbLog2));
}

uint word(uint a) { return swapWord(memLoad((a & 0x1FFFFFFFu) >> 2), p.endian); }

uint element(uint bx, uint by)
{
    if (bx >= p.width) return 0u;
    uint a = addr(bx, by);
    uint w = word(a & ~3u);
    return p.bpbLog2 == 1u ? (w >> ((a & 2u) * 8u)) & 0xFFFFu : (w >> ((a & 3u) * 8u)) & 0xFFu;
}

uint expand(uint v, uint bits) { uint m = (1u << bits) - 1u; return (v * 255u + m / 2u) / m; }

void main()
{
    uint gx = gl_GlobalInvocationID.x, gy = gl_GlobalInvocationID.y;
    if (gy >= p.height) return;
    uint row = p.dstWord + gy * p.dstRowWords;
    if (p.convert != 0u)
    {
        if (gx >= p.width) return;
        uint o;
        if (p.convert == 4u)
        {
            o = floatBitsToUint(float(word(addr(gx, gy)) >> 8) / 16777215.0);
        }
        else
        {
            uint v = element(gx, gy);
            uint r, g, b, a;
            if (p.convert == 1u) { r = expand(v & 31u, 5u); g = expand((v >> 5) & 63u, 6u); b = expand((v >> 11) & 31u, 5u); a = 255u; }
            else if (p.convert == 2u) { r = expand(v & 31u, 5u); g = expand((v >> 5) & 31u, 5u); b = expand((v >> 10) & 31u, 5u); a = (v >> 15) != 0u ? 255u : 0u; }
            else { r = expand(v & 15u, 4u); g = expand((v >> 4) & 15u, 4u); b = expand((v >> 8) & 15u, 4u); a = expand(v >> 12, 4u); }
            o = r | (g << 8) | (b << 16) | (a << 24);
        }
        STORE(gx, o);
        return;
    }
    if (p.bpbLog2 >= 2u)
    {
        if (gx >= p.width) return;
        uint words = 1u << (p.bpbLog2 - 2u);
        uint a = addr(gx, gy);
        for (uint k = 0u; k < words; k++)
            STORE(gx * words + k, word(a + k * 4u));
        return;
    }
    uint perWord = 4u >> p.bpbLog2;
    uint bx = gx * perWord;
    if (bx >= p.width) return;
    uint o = 0u;
    for (uint k = 0u; k < perWord; k++)
        o |= element(bx + k, gy) << (k * (8u << p.bpbLog2));
    STORE(gx, o);
}
)";

        VkPipeline s_untilePipeline = VK_NULL_HANDLE;
        VkPipelineLayout s_untileLayout = VK_NULL_HANDLE;
        VkDescriptorSetLayout s_untileSetLayout = VK_NULL_HANDLE;
        // 1x textures stored directly (TO_IMAGE): 2D (or, NFSMW_DIRECT_CUBE, a
        // cube, a face at a time), one level, uncompressed, 32-bit texels; the
        // untile writes the image, with no staging slice and no copy after it
        // (on the Steam Frame a 1600x1600 shadow map's upload was ~0.7 ms,
        // about half of it the copy). NFSMW_UNTILE_DIRECT=0 keeps the staging
        // path.
        VkPipeline s_untileStorePipeline = VK_NULL_HANDLE;
        VkPipelineLayout s_untileStoreLayout = VK_NULL_HANDLE;
        VkDescriptorSetLayout s_untileStoreSetLayout = VK_NULL_HANDLE;
        const bool s_untileDirect = [] { const char* v = std::getenv("NFSMW_UNTILE_DIRECT"); return !v || v[0] != '0'; }();
        // NFSMW_DIRECT_CUBE=1 (default kFewerPasses; 0: cubes take the staging path): an
        // un-mipped cube is a direct texture too, each face stored through an
        // R32_UINT 2D view of its layer (Texture::storeViews), so a resolve
        // into a face stores into it as well (Resolve). The race's
        // environment cube (256x256, faces resolved every frame) was staged
        // again at its first use in the scene, on every platform: six
        // untiles, six copies and a split render pass a frame (two in heavy
        // scenes). A device that turns the image or a view down keeps the
        // staging path for that format.
        const bool s_directCube = [] { const char* v = std::getenv("NFSMW_DIRECT_CUBE"); return v ? v[0] == '1' : kFewerPasses; }();
        // fi.vk -> a storable R32_UINT view is allowed (2D; cube compatible, with six layers)
        std::unordered_map<VkFormat, bool> s_directFormats, s_directCubeFormats;

        bool DirectFormat(VkFormat format, bool cube = false)
        {
            auto [it, fresh] = (cube ? s_directCubeFormats : s_directFormats).try_emplace(format, false);
            if (fresh)
            {
                VkImageFormatProperties props;
                it->second = vkGetPhysicalDeviceImageFormatProperties(s_vk->physical, format, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL,
                    VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
                    VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT | (cube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0),
                    &props) == VK_SUCCESS && (!cube || props.maxArrayLayers >= 6);
            }
            return it->second;
        }

        struct UntileConstants
        {
            uint32_t srcBase, tiled, pitch, bpbLog2;
            uint32_t offX, offY, width, height;
            uint32_t endian, convert, dstWord, dstRowWords;
            uint32_t z, heightAligned;
        };

        struct Texture
        {
            Image image;
            VkFormat format;
            uint32_t width, height;
            uint32_t guestStart, guestSize;
            uint64_t loadedSequence = 0;  // writewatch::Current() when last loaded
            uint64_t checkedEpoch = ~0ull;  // writewatch::WriteEpoch() at the last validity check
            bool uploaded = false;
            bool cube = false;
            bool volume = false;
            uint32_t layers = 1, depth = 1;
            uint32_t levels = 1;
            uint32_t mipStart = 0, mipSize = 0;  // the mip levels' guest memory (size 0: none)
            bool scaled = false;  // host resolution, from the scaled resolve memory
            VkImageView storeView = VK_NULL_HANDLE;  // scaled: R32_UINT, stored into by the untile (if the device can)
            // The same, a 2D view of each layer: [0] is storeView, and a
            // direct cube (NFSMW_DIRECT_CUBE) has one per face.
            VkImageView storeViews[6] = {};
            std::unordered_map<uint32_t, VkImageView> views;  // by swizzle
            // How the untile reads a direct 1x texture (a resolve can store
            // into it the same way, see Resolve); sliceStride: a cube's guest
            // bytes from one face to the next.
            bool tiled = false;
            uint32_t pitch = 0, bpbLog2 = 0, convert = 0, endian = 0, offX = 0, offY = 0, sliceStride = 0;
            // NFSMW_TEXTURE_VERIFY: a hash of each 4 KB guest page of its
            // base, then its mips, as loaded (empty: not checked), and the
            // frame it was last checked in.
            std::vector<uint64_t> verifyHashes;
            uint64_t verifyFrame = 0;
        };
        void LoadScaledTexture(Texture& t, uint32_t base, uint32_t pitchBlocks, uint32_t endian);
        bool ScaledTextureValid(uint32_t base, uint32_t size);
        std::unordered_map<uint64_t, std::unique_ptr<Texture>> s_textures;
        // Direct 1x 2D textures by guest address (the latest made there), for
        // resolves that store into them too (DestroyTexture drops its entry).
        std::unordered_map<uint32_t, Texture*> s_directByBase;
        // Resolves store into such a texture too when it is exactly their
        // surface (see Resolve); NFSMW_RESOLVE_TO_TEXTURE=0 turns it off.
        const bool s_resolveToTexture = [] { const char* v = std::getenv("NFSMW_RESOLVE_TO_TEXTURE"); return !v || v[0] != '0'; }();
        // NFSMW_FUSE_PARTIAL=1 (default kFewerPasses; 0: only a resolve
        // covering the whole texture stores into it): a resolve into whole 32x32 tiles
        // inside a direct texture (or a cube's face) stores there too, at
        // those tiles' texels (Resolve). The scene's colour and depth
        // (1280x720) are each resolved in three strips, a predicated tile
        // each, and were untiled again between passes every frame (3.7 MB
        // each), on every platform.
        const bool s_fusePartial = [] { const char* v = std::getenv("NFSMW_FUSE_PARTIAL"); return v ? v[0] == '1' : kFewerPasses; }();
        // Direct textures by their first guest byte (the latest made there),
        // for resolves into part of one (Resolve): every one with
        // NFSMW_FUSE_PARTIAL, the cubes with NFSMW_DIRECT_CUBE alone; and the
        // largest one's guest size, how far before an address one holding
        // it can start.
        std::map<uint32_t, Texture*> s_directByStart;
        uint32_t s_maxDirectSize = 0;

        // NFSMW_LOG_UPLOADS=<first>[,<last>]: in those frames (s_frame, as
        // NFSMW_PASS_PROFILE numbers them), a line per texture (re)load: its
        // path, whether it ended a render pass, and for a reload the guest
        // pages written since the last one with what last wrote each
        // (write_watch keeps that while this is set); a line per resolve
        // (stored into its cached texture or not, and why); a summary at the
        // end of each frame. Null when off.
        struct UploadLog
        {
            uint64_t first = 0, last = 0;
            bool now = false;  // s_frame is logged
            struct Resolve { uint64_t frame; const char* unfused; };  // unfused: null when it stored into the texture
            std::unordered_map<uint32_t, Resolve> resolves;  // by destination, the latest (every frame)
            uint32_t uploads = 0, endedPasses = 0, resolvesNow = 0, fusedNow = 0;
            uint64_t bytes = 0;
            std::map<std::string, uint32_t> byPath, byWriters;
        };
        UploadLog* const s_uploadLog = []() -> UploadLog* {
            const char* v = std::getenv("NFSMW_LOG_UPLOADS");
            if (!v)
                return nullptr;
            auto* log = new UploadLog();
            log->first = log->last = std::strtoull(v, nullptr, 10);
            if (const char* comma = std::strchr(v, ','))
                log->last = std::strtoull(comma + 1, nullptr, 10);
            return log;
        }();

        // Whether the latest resolve to `dest` stored into its cached texture.
        std::string ResolveNote(uint32_t dest)
        {
            auto it = s_uploadLog->resolves.find(dest);
            if (it == s_uploadLog->resolves.end())
                return "?";
            return it->second.unfused ? std::string("unfused: ") + it->second.unfused : "fused";
        }

        // The guest pages of `t` written since its load, and what last wrote
        // them (the first 4, then counts by kind, also added to `kinds`).
        std::string DescribeWrites(const Texture& t, std::set<std::string>& kinds)
        {
            auto pageCount = [](uint32_t b, uint32_t n) { return n ? ((b + n - 1) >> 12) - (b >> 12) + 1 : 0u; };
            auto inTexture = [&](uint32_t a) { return a - t.guestStart < t.guestSize || (t.mipSize && a - t.mipStart < t.mipSize); };
            std::vector<uint32_t> pages(pageCount(t.guestStart, t.guestSize) + pageCount(t.mipStart, t.mipSize));
            uint32_t n = writewatch::WrittenPages(t.guestStart, t.guestSize, t.loadedSequence, pages.data(), pageCount(t.guestStart, t.guestSize));
            uint32_t mipN = t.mipSize ? writewatch::WrittenPages(t.mipStart, t.mipSize, t.loadedSequence, pages.data() + n,
                pageCount(t.mipStart, t.mipSize)) : 0;
            std::string text = std::format("dirty {}/{} pages, mips {}/{}:", n, pageCount(t.guestStart, t.guestSize), mipN,
                pageCount(t.mipStart, t.mipSize));
            std::map<std::string, uint32_t> counts;
            for (uint32_t i = 0; i < n + mipN; i++)
            {
                writewatch::LastWrite w = writewatch::LastWriter(pages[i]);
                // (NFSMW_WATCH_SUBPAGES: compared, a store its host page's
                // snapshot found there; unverified, marked unseen as the
                // snapshot closed without a compare, at the fault that opened it.)
                std::string kind = w.kind == writewatch::Writer::kCpuFault ? (inTexture(w.tag) ? "cpu-fault" : "cpu-fault-nbr")
                    : w.kind == writewatch::Writer::kResolve ? "resolve" : w.kind == writewatch::Writer::kHostWrite ? "host-write"
                    : w.kind == writewatch::Writer::kGpuStore ? "gpu-store" : w.kind == writewatch::Writer::kRange ? "range"
                    : w.kind == writewatch::Writer::kCompared ? "compared" : w.kind == writewatch::Writer::kUnverified ? "unverified" : "none";
                counts[kind]++;
                kinds.insert(kind);
                if (i < 4)
                    text += std::format(" [{:08X} {}{} -{}f]", pages[i], kind,
                        w.kind == writewatch::Writer::kResolve ? std::format(" dest {:08X} {}", w.tag, ResolveNote(w.tag))
                        : w.kind != writewatch::Writer::kNone ? std::format(" at {:08X}{}", w.tag, w.pc ? std::format(" pc +{:X}", w.pc) : "")
                        : std::string(), (uint32_t(s_frame) - w.frame) & 0x0FFFFFFF);
            }
            text += " |";
            for (auto& [kind, count] : counts)
                text += std::format(" {} {}", kind, count);
            return text;
        }

        // Before (re)loading `t` (its loadedSequence is still the last load's).
        void LogUpload(const Texture& t, uint32_t format, bool tiled)
        {
            UploadLog& log = *s_uploadLog;
            const char* path = t.scaled ? "scaled" : t.storeView ? "direct" : "staging";
            std::set<std::string> writers;
            std::string line = std::format("[upload] frame {} base {:08X} size {} mip {:08X}/{} {}x{} {}/{} levels {} fmt {}/vk{} tiled {} {} path {} {} reason {}",
                s_frame, t.guestStart, t.guestSize, t.mipStart, t.mipSize, t.width, t.height, t.depth, t.layers, t.levels, format,
                int(t.format), int(tiled), t.cube ? "cube" : t.volume ? "volume" : "2d", path, s_passActive ? "ends-pass" : "between-passes",
                t.uploaded ? "written " + DescribeWrites(t, writers) : "first");
            if (!t.uploaded)
                writers.insert("first");
            auto it = log.resolves.find(t.guestStart);
            line += it == log.resolves.end() ? std::string(" | resolve-dest no")
                : std::format(" | resolve-dest -{}f {}", s_frame - it->second.frame, ResolveNote(t.guestStart));
            fprintf(stderr, "%s\n", line.c_str());
            std::string key;
            for (const std::string& w : writers)
                key += (key.empty() ? "" : "+") + w;
            log.byWriters[key]++;
            log.byPath[path]++;
            log.uploads++;
            log.endedPasses += s_passActive;
            log.bytes += t.guestSize + t.mipSize;
        }

        // NFSMW_TEXTURE_VERIFY=1 (debug; as NFSMW_CP_CACHE_VERIFY): that the
        // write watch sees every change to a texture's memory, which
        // NFSMW_WATCH_SUBPAGES partly finds by comparing instead of faulting.
        // A 1x texture's guest pages are hashed at its load (after Watch
        // protects them), and again at its first use each frame that finds
        // it still valid; a change with no write recorded since the load is
        // logged ([texverify], the first 100). Not checked: scaled textures,
        // pages a GPU write leaves unreadable (resolve destinations), and a
        // texture a resolve stored into (the GPU changes its memory later).
        const bool s_textureVerify = [] { const char* v = std::getenv("NFSMW_TEXTURE_VERIFY"); return v && v[0] == '1'; }();
        uint64_t s_textureMisses = 0;

        // A hash of each 4 KB guest page of `t`'s base, then its mips (the
        // part in the range), as they are now.
        void HashTexturePages(const Texture& t, std::vector<uint64_t>& hashes)
        {
            hashes.clear();
            for (auto [start, size] : { std::pair{ t.guestStart, t.guestSize }, std::pair{ t.mipStart, t.mipSize } })
                for (uint64_t page = start & ~0xFFFull; page < uint64_t(start) + size; page += 0x1000)
                {
                    uint64_t from = std::max<uint64_t>(page, start), to = std::min<uint64_t>(page + 0x1000, uint64_t(start) + size);
                    const uint8_t* p = s_sharedHost + from;
                    size_t n = size_t(to - from), i = 0;
                    // Four lanes: one multiply chain would be latency-bound.
                    uint64_t h[4] = { n, 1, 2, 3 };
                    for (; i + 32 <= n; i += 32)
                        for (int lane = 0; lane < 4; lane++)
                        {
                            uint64_t w;
                            memcpy(&w, p + i + lane * 8, 8);
                            h[lane] = (h[lane] ^ w) * 0x9E3779B97F4A7C15ull;
                            h[lane] ^= h[lane] >> 32;
                        }
                    for (; i < n; i++)
                        h[0] = (h[0] ^ p[i]) * 0x100000001B3ull;
                    hashes.push_back(h[0] ^ std::rotl(h[1], 16) ^ std::rotl(h[2], 32) ^ std::rotl(h[3], 48));
                }
        }

        // NFSMW_TEXTURE_VERIFY: after a load's Watch, what its pages hold (a
        // store from now on faults, or is compared).
        void HashLoadedTexture(Texture& t)
        {
            // Only the GPU guard (NFSMW_EAGER_WRITES, the default, or shadow
            // mode) leaves a resolve's destination unreadable until the CPU
            // next touches it: without it, one would be hashed before the
            // resolve lands, and each later check would report it.
            bool guarded = g_eagerWrites || s_shadowMode;
            LogOnce(0x2900, "%s", guarded ? "NFSMW_TEXTURE_VERIFY: texture pages hashed at each load, checked at the first use each frame"
                : "NFSMW_TEXTURE_VERIFY needs the GPU guard (NFSMW_EAGER_WRITES=0 turns it off): nothing checked");
            t.verifyFrame = s_frame;
            if (!guarded || t.scaled || !writewatch::CpuReadable(t.guestStart, t.guestSize) || !writewatch::CpuReadable(t.mipStart, t.mipSize))
                t.verifyHashes.clear();
            else
                HashTexturePages(t, t.verifyHashes);
        }

        // NFSMW_TEXTURE_VERIFY: a use found `t` valid, deciding at WriteEpoch()
        // `epoch`. Its pages must hold what they did at its load.
        void VerifyTexture(Texture& t, uint64_t epoch)
        {
            if (t.verifyHashes.empty() || t.verifyFrame == s_frame)
                return;
            t.verifyFrame = s_frame;
            if (!writewatch::CpuReadable(t.guestStart, t.guestSize) || !writewatch::CpuReadable(t.mipStart, t.mipSize))
                return;
            std::vector<uint64_t> now;
            HashTexturePages(t, now);
            if (now == t.verifyHashes)
                return;
            // A write recorded since that use decided (another thread, just
            // now) moved the epoch: the next use loads it again.
            if (writewatch::WriteEpoch() != epoch && (writewatch::WrittenSince(t.guestStart, t.guestSize, t.loadedSequence) ||
                    (t.mipSize && writewatch::WrittenSince(t.mipStart, t.mipSize, t.loadedSequence))))
                return;
            if (s_textureMisses++ < 100)
            {
                // The changed pages' addresses, in HashTexturePages' order.
                uint32_t basePages = t.guestSize ? ((t.guestStart + t.guestSize - 1) >> 12) - (t.guestStart >> 12) + 1 : 0;
                std::string pages;
                uint32_t changed = 0;
                for (size_t i = 0; i < now.size(); i++)
                    if (now[i] != t.verifyHashes[i] && changed++ < 4)
                        pages += std::format(" {:08X}", i < basePages ? ((t.guestStart >> 12) + uint32_t(i)) << 12
                            : ((t.mipStart >> 12) + uint32_t(i - basePages)) << 12);
                fprintf(stderr, "[texverify] frame %llu texture %08X+%u mips %08X+%u %ux%u%s vk%d: %u of %zu pages changed since its load "
                    "with no write recorded:%s%s\n", (unsigned long long)s_frame, t.guestStart, t.guestSize, t.mipStart, t.mipSize, t.width,
                    t.height, t.cube ? " cube" : t.volume ? " volume" : "", int(t.format), changed, now.size(), pages.c_str(),
                    changed > 4 ? " ..." : "");
            }
            t.verifyHashes = std::move(now);  // (each change reported once)
        }

        std::unordered_map<uint64_t, VkSampler> s_samplers;
        bool s_loadMips = true;  // Settings > Video > Mipmaps, as applied (ApplyLiveSettings)

        // Anisotropic filtering on linearly filtered textures: 16x on a
        // discrete GPU or an M-series iPad, 4x on other integrated ones (16x
        // cost the M1 Pro ~4 ms a frame in races at 2x2). Settings > Video (NFSMW_ANISOTROPY=<1..16>)
        // changes it live; "game" uses the game's own setting.
        float s_maxAnisotropy = 1.0f;  // device limit (1: no anisotropic filtering)
        int s_autoAnisotropy = 16;      // "Auto": set in CreateSharedMemory

        // `maxLevel`: the texture's last mip level (0: base only).
        VkSampler GetSampler(const xenos::xe_gpu_texture_fetch_t& f, uint32_t maxLevel)
        {
            // The setting is part of the key: a change makes new samplers,
            // while command buffers in flight keep theirs (never destroyed).
            int anisotropy = settings::GetInt(settings::Id::Anisotropy);
            if (anisotropy == settings::kAnisotropyAuto)
                anisotropy = s_autoAnisotropy;
            uint64_t key = uint64_t(f.clamp_x) | (uint64_t(f.clamp_y) << 3) | (uint64_t(f.clamp_z) << 6) |
                (uint64_t(f.mag_filter) << 9) | (uint64_t(f.min_filter) << 11) | (uint64_t(f.border_color) << 13) |
                (uint64_t(f.mip_filter) << 15) | (uint64_t(f.aniso_filter) << 17) | (uint64_t(f.mip_min_level) << 20) |
                (uint64_t(maxLevel) << 24) | (uint64_t(uint8_t(anisotropy)) << 32);
            auto& slot = s_samplers[key];
            if (slot)
                return slot;
            auto address = [](xenos::ClampMode m) {
                switch (m)
                {
                case xenos::ClampMode::kRepeat: return VK_SAMPLER_ADDRESS_MODE_REPEAT;
                case xenos::ClampMode::kMirroredRepeat: return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
                case xenos::ClampMode::kClampToBorder:
                case xenos::ClampMode::kMirrorClampToBorder: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
                default: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
                }
            };
            VkSamplerCreateInfo sci{ VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO };
            sci.magFilter = f.mag_filter == xenos::TextureFilter::kPoint ? VK_FILTER_NEAREST : VK_FILTER_LINEAR;
            sci.minFilter = f.min_filter == xenos::TextureFilter::kPoint ? VK_FILTER_NEAREST : VK_FILTER_LINEAR;
            sci.addressModeU = address(f.clamp_x);
            sci.addressModeV = address(f.clamp_y);
            sci.addressModeW = address(f.clamp_z);
            // Mips as the fetch constant says: point or linear between
            // levels, or the base level only; its level range and bias.
            sci.mipmapMode = f.mip_filter == xenos::TextureFilter::kLinear ? VK_SAMPLER_MIPMAP_MODE_LINEAR : VK_SAMPLER_MIPMAP_MODE_NEAREST;
            bool baseOnly = maxLevel == 0 || f.mip_filter == xenos::TextureFilter::kBaseMap;
            sci.minLod = baseOnly ? 0.0f : float(std::min<uint32_t>(f.mip_min_level, maxLevel));
            sci.maxLod = baseOnly ? 0.0f : float(maxLevel);
            // (The fetch constant's LOD bias is applied in the shader, as
            // Xenia does: MoltenVK has no sampler LOD bias.)
            // Anisotropy (the game's: 1, 2, 4, 8 or 16:1, unless overridden)
            // for linear minification.
            static const float gameAniso[8] = { 1, 1, 2, 4, 8, 16, 1, 1 };
            float aniso = anisotropy > 0 ? float(anisotropy) : gameAniso[uint32_t(f.aniso_filter) & 7];
            aniso = std::min(aniso, s_maxAnisotropy);
            if (aniso > 1.0f && sci.minFilter == VK_FILTER_LINEAR)
            {
                sci.anisotropyEnable = VK_TRUE;
                sci.maxAnisotropy = aniso;
            }
            sci.borderColor = f.border_color == xenos::BorderColor::k_ABGR_White
                ? VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
            Check(vkCreateSampler(s_dev, &sci, nullptr, &slot), "vkCreateSampler");
            return slot;
        }

        VkImageView TextureView(Texture& t, uint32_t swizzle)
        {
            auto& view = t.views[swizzle];
            if (view)
                return view;
            auto comp = [](uint32_t s) {
                switch (s & 7)
                {
                case 0: return VK_COMPONENT_SWIZZLE_R;
                case 1: return VK_COMPONENT_SWIZZLE_G;
                case 2: return VK_COMPONENT_SWIZZLE_B;
                case 3: return VK_COMPONENT_SWIZZLE_A;
                case 4: return VK_COMPONENT_SWIZZLE_ZERO;
                case 5: return VK_COMPONENT_SWIZZLE_ONE;
                default: return VK_COMPONENT_SWIZZLE_IDENTITY;
                }
            };
            // Sampled only (a storable scaled texture's format need not support storage).
            VkImageViewUsageCreateInfo usage{ VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO, nullptr, VK_IMAGE_USAGE_SAMPLED_BIT };
            VkImageViewCreateInfo vci{ VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO };
            vci.pNext = t.storeView ? &usage : nullptr;
            vci.image = t.image.image;
            vci.viewType = t.cube ? VK_IMAGE_VIEW_TYPE_CUBE : t.volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D;
            vci.format = t.format;
            vci.components = { comp(swizzle), comp(swizzle >> 3), comp(swizzle >> 6), comp(swizzle >> 9) };
            vci.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, t.cube ? 6u : 1u };
            Check(vkCreateImageView(s_dev, &vci, nullptr, &view), "vkCreateImageView(texture)");
            return view;
        }

        // The view for texture fetch constant `slot` sampled as 2D or as a
        // cube, or the placeholder. Base level only; each cube face is a
        // tiled 2D slice, 4 KB-aligned (Xenia's GetGuestTextureLayout).
        enum class TexDim { k2D, k3D, kCube };

        // What a lookup's result depends on beyond the fetch constant (for
        // the lookup cache below). `texture`: null when it isn't kept (a
        // placeholder, a scaled texture).
        struct TextureLookup
        {
            Texture* texture = nullptr;
            uint32_t base = 0, size = 0, mipBase = 0, mipSize = 0;
            bool scaledCandidate = false;  // a draw into scaled targets may take the scaled path
        };

        // The full lookup (GetTexture below keeps its results).
        VkImageView LookupTexture(uint32_t slot, TexDim dim, VkSampler& sampler, TextureLookup* out)
        {
            const bool cube = dim == TexDim::kCube, volume = dim == TexDim::k3D;
            xenos::xe_gpu_texture_fetch_t f;
            memcpy(&f, &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6], sizeof(f));
            sampler = s_sampler;
            const VkImageView dummy = s_dummy[cube ? 2 : volume ? 1 : 0].view;
            if (f.type != xenos::FetchConstantType::kTexture)
                return dummy;
            bool dimensionOk = cube ? f.dimension == xenos::DataDimension::kCube
                : volume ? f.dimension == xenos::DataDimension::k3D
                : (f.dimension == xenos::DataDimension::k2DOrStacked || f.dimension == xenos::DataDimension::k1D);
            if (!dimensionOk)
            {
                LogOnce(0x2000 + uint32_t(f.dimension) + uint32_t(dim) * 8, "texture dimension %u sampled as dimension %u",
                    uint32_t(f.dimension), uint32_t(dim));
                return dummy;
            }
            uint32_t layers = cube ? 6 : 1;
            TextureFormatInfo fi;
            uint32_t format = uint32_t(f.format);
            if (!GetTextureFormat(format, fi))
            {
                LogOnce(0x3000 + format, "texture format %u not implemented", format);
                return dummy;
            }
            if (f.base_address == 0)
            {
                LogOnce(0x2100, "texture without a base level (mips only)");
                return dummy;
            }
            uint32_t width = (f.dimension == xenos::DataDimension::k1D ? f.size_1d.width
                : volume ? f.size_3d.width : f.size_2d.width) + 1;
            uint32_t height = f.dimension == xenos::DataDimension::k1D ? 1 : (volume ? f.size_3d.height : f.size_2d.height) + 1;
            uint32_t depth = volume ? f.size_3d.depth + 1 : 1;

            // Guest layout (Xenia's texture_util).
            uint32_t bw = (width + fi.block - 1) / fi.block, bh = (height + fi.block - 1) / fi.block;
            uint32_t offX = 0, offY = 0;
            uint32_t log2w = Log2Ceil(width), log2h = Log2Ceil(height);
            if (f.packed_mips && std::min(log2w, log2h) <= 4)
            {
                // The base level itself sits in the packed-mip tile.
                if (log2w > log2h)
                    offY = 16 / fi.block;
                else
                    offX = 16 / fi.block;
            }
            uint32_t pitchTexels = f.pitch ? f.pitch << 5 : width;
            uint32_t pitchBlocks = (pitchTexels + fi.block - 1) / fi.block;
            uint32_t base = (f.base_address << 12) & 0x1FFFFFFF;
            if (base == s_logTextureUse)
            {
                s_drawUsesLoggedTexture = true;
                s_loggedTextureFetch = std::format("slot {} dim {} fetch {:08X} {:08X} {:08X} {:08X} {:08X} {:08X}", slot, int(dim),
                    s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6], s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6 + 1],
                    s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6 + 2], s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6 + 3],
                    s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6 + 4], s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6 + 5]);
            }
            uint32_t pitchAligned = 0, rowBytes = 0, size;
            if (f.tiled)
            {
                pitchAligned = (std::max(pitchBlocks, bw + offX) + 31) & ~31u;
                uint32_t heightAligned = (bh + offY + 31) & ~31u;
                size = (pitchAligned * heightAligned) << fi.bpbLog2;
            }
            else
            {
                rowBytes = ((pitchBlocks << fi.bpbLog2) + 255) & ~255u;
                size = rowBytes * (bh + offY);
            }
            // Faces after the first follow at the 4 KB-aligned slice stride.
            uint32_t sliceStride = f.tiled ? (((pitchAligned * ((bh + offY + 31) & ~31u)) << fi.bpbLog2) + 4095) & ~4095u
                                           : ((rowBytes * ((bh + offY + 31) & ~31u)) + 4095) & ~4095u;
            size += sliceStride * (layers - 1);
            uint32_t heightAligned3D = (bh + 31) & ~31u;
            if (volume)
            {
                // Tiled: one Tiled3D volume (depth in groups of 4); linear:
                // slices at the 2D slice stride.
                size = f.tiled ? (pitchAligned * heightAligned3D * ((depth + 3) & ~3u)) << fi.bpbLog2
                               : sliceStride * depth;
            }
            size = std::min(size, uint32_t(kSharedSize) - base);

            // Mip levels (2D and cube): up to mip_max_level, when the texture
            // has a mip address. Settings > Video > Mipmaps off
            // (NFSMW_TEXTURE_MIPS=0) loads the base only; a change drops the
            // cache (ApplyLiveSettings), so every texture is loaded again.
            const bool loadMips = s_loadMips;
            uint32_t maxLevel = 0, mipBase = 0, mipSize = 0;
            MipLayout mips{ width, height, fi.block, fi.bpbLog2, layers, f.tiled != 0, f.packed_mips != 0, 0 };
            if (loadMips && !volume && !f.stacked && f.dimension != xenos::DataDimension::k1D && f.mip_address != 0)
            {
                uint32_t sizeMaxLevel = 31 - uint32_t(__builtin_clz(std::max(width, height)));
                maxLevel = std::min<uint32_t>(f.mip_max_level, sizeMaxLevel);
                if (maxLevel > 0)
                {
                    uint32_t log2Short = Log2Ceil(std::min(width, height));
                    mips.packedLevel = log2Short > 4 ? log2Short - 4 : 0;
                    mipBase = (f.mip_address << 12) & 0x1FFFFFFF;
                    uint32_t lastStorage = mips.StorageLevel(maxLevel);
                    mipSize = mips.Offset(lastStorage) + mips.SliceStride(lastStorage) * layers;
                    mipSize = std::min(mipSize, uint32_t(kSharedSize) - mipBase);
                }
            }
            // Drawn into scaled targets: a texture that scaled resolves wrote
            // (and nothing since) is sampled at host resolution, base level
            // only (resolved surfaces have no mips).
            // (Mipmapped textures keep their levels on the 1x path.)
            // At 1.5x only a texture of whole 2x2 blocks (even width and
            // height; gpu/render_scale.h).
            bool scaledCandidate = !cube && !volume && f.tiled && layers == 1 && !f.stacked &&
                ScaledFormat(fi) && offX == 0 && offY == 0 && maxLevel == 0 && s_scaled.scale.Exact(width, height) &&
                s_scaled.scale.HostPixels(width, height) * 4 <= s_scaled.scratchSize;
            bool scaled = s_drawScaled && s_scaled.enabled && scaledCandidate && ScaledTextureValid(base, size);
            sampler = GetSampler(f, maxLevel);
            if (out)
                *out = { nullptr, base, size, mipBase, mipSize, scaledCandidate };

            uint64_t key = scaled ? 0x5CA1ED5CA1EDull : 0xCBF29CE484222325ull;
            for (uint64_t v : { uint64_t(base), uint64_t(format), uint64_t(width), uint64_t(height), uint64_t(f.tiled),
                     uint64_t(pitchTexels), uint64_t(f.endianness), uint64_t(f.packed_mips), uint64_t(layers), uint64_t(depth),
                     uint64_t(mipBase), uint64_t(maxLevel) })
                key = (key ^ v) * 0x100000001B3ull;
            auto& slotRef = s_textures[key];
            if (!slotRef)
            {
                slotRef = std::make_unique<Texture>();
                Texture& t = *slotRef;
                t.format = fi.vk;
                t.width = width;
                t.height = height;
                t.guestStart = base;
                t.guestSize = size;
                t.cube = cube;
                t.volume = volume;
                t.layers = layers;
                t.depth = depth;
                t.levels = maxLevel + 1;
                t.mipStart = mipBase;
                t.mipSize = mipSize;
                t.scaled = scaled;
                if (scaled)
                {
                    t.width = s_scaled.scale.x.Size(width);
                    t.height = s_scaled.scale.y.Size(height);
                    s_scaled.textures++;
                }
                // A 1x texture the untile can store directly (see
                // s_untileStorePipeline): 2D, or a cube (NFSMW_DIRECT_CUBE).
                bool direct1x = !scaled && s_untileDirect && !volume && maxLevel == 0 && fi.block == 1 &&
                    (fi.convert != 0 || fi.bpbLog2 == 2) && (cube ? s_directCube && layers == 6 && !f.stacked : layers == 1) &&
                    DirectFormat(fi.vk, cube);
                if ((scaled && s_scaled.direct) || direct1x)
                {
                    // The untile stores its words through an R32_UINT view
                    // (every scaled format is 32 bits a texel, as is each
                    // direct 1x one), a 2D one of each layer (a cube's faces).
                    VkFormat formats[2] = { fi.vk, VK_FORMAT_R32_UINT };
                    VkImageFormatListCreateInfo list{ VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO, nullptr, 2, formats };
                    VkImageViewUsageCreateInfo sampled{ VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO, nullptr, VK_IMAGE_USAGE_SAMPLED_BIT };
                    // A cube's image and views are untried on some drivers
                    // (MoltenVK: 2D R32_UINT views of a cube's layers): one
                    // turned down is staged instead, below.
                    t.image = CreateImage(VK_IMAGE_TYPE_2D, fi.vk, { t.width, t.height, 1 }, layers,
                        VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | (direct1x ? VK_IMAGE_USAGE_TRANSFER_DST_BIT : 0),
                        cube ? VK_IMAGE_VIEW_TYPE_CUBE : VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT,
                        VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT | (cube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0),
                        1, &list, &sampled, cube);
                    VkImageViewCreateInfo vci{ VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO };
                    vci.image = t.image.image;
                    vci.viewType = VK_IMAGE_VIEW_TYPE_2D;
                    vci.format = VK_FORMAT_R32_UINT;
                    bool made = t.image.image != VK_NULL_HANDLE;
                    for (uint32_t layer = 0; layer < layers && made; layer++)
                    {
                        vci.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, layer, 1 };
                        VkResult result = vkCreateImageView(s_dev, &vci, nullptr, &t.storeViews[layer]);
                        if (!cube)
                            Check(result, "vkCreateImageView(scaled store)");
                        if (result != VK_SUCCESS)
                        {
                            t.storeViews[layer] = VK_NULL_HANDLE;
                            made = false;
                        }
                    }
                    t.storeView = t.storeViews[0];
                    if (!made)
                    {
                        fprintf(stderr, "[renderer] cube %ux%u format %d can't be stored directly (image or view refused): staged\n",
                            width, height, int(fi.vk));
                        for (VkImageView& view : t.storeViews)
                        {
                            if (view)
                                vkDestroyImageView(s_dev, view, nullptr);
                            view = VK_NULL_HANDLE;
                        }
                        t.storeView = VK_NULL_HANDLE;
                        if (t.image.image)
                            DestroyImage(t.image);
                        s_directCubeFormats[fi.vk] = false;  // (once a format: DirectFormat says no from now on)
                    }
                    else if (direct1x)
                    {
                        t.tiled = f.tiled != 0;
                        t.pitch = f.tiled ? pitchAligned : rowBytes;
                        t.bpbLog2 = fi.bpbLog2;
                        t.convert = fi.convert;
                        t.endian = uint32_t(f.endianness);
                        t.offX = offX;
                        t.offY = offY;
                        t.sliceStride = sliceStride;
                        // (A resolve into a cube takes the face rule, see Resolve.)
                        if (!cube)
                            s_directByBase[base] = &t;
                        if (s_fusePartial || cube)
                        {
                            s_directByStart[base] = &t;
                            s_maxDirectSize = std::max(s_maxDirectSize, t.guestSize);
                        }
                    }
                }
                if (!t.image.image)
                    t.image = CreateImage(volume ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D, fi.vk, { t.width, t.height, depth }, layers,
                        VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
                        cube ? VK_IMAGE_VIEW_TYPE_CUBE : volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT,
                        cube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0, t.levels);
            }
            Texture& t = *slotRef;
            {
                // No page written anywhere since the last check: still valid
                // (a draw binds ~7,500 textures a frame; writes are rare).
                uint64_t epoch = writewatch::WriteEpoch();
                bool scan = !(g_cpOpt & CP_OPT_TEXTURE_EPOCH) || epoch != t.checkedEpoch;
                t.checkedEpoch = epoch;
                if (!t.uploaded || (scan && (writewatch::WrittenSince(base, size, t.loadedSequence) ||
                        (mipSize && writewatch::WrittenSince(mipBase, mipSize, t.loadedSequence)))))
                {
                    if (s_uploadLog && s_uploadLog->now)
                        LogUpload(t, format, f.tiled != 0);
                    // Snapshot before recording: a store after this faults and
                    // bumps the sequence; the GPU load itself runs later and
                    // sees anything written before. (NFSMW_WATCH_SUBPAGES: the
                    // snapshots open on its pages are compared first, else the
                    // compare Watch makes below would mark its own pages newer
                    // than this load, and it would load again.)
                    writewatch::Settle(base, size);
                    writewatch::Settle(mipBase, mipSize);
                    t.loadedSequence = writewatch::Current();
                    HitchTimer hitchTimer(s_hitch.textureMs);
                    s_hitch.textures++;
                    if (t.scaled)
                    {
                        s_hitch.scaledTextures++;
                        LoadScaledTexture(t, base, pitchAligned, uint32_t(f.endianness));
                        t.uploaded = true;
                        writewatch::Watch(base, size);  // as the 1x path: a CPU write faults and retires it
                        return TextureView(t, f.swizzle);
                    }
                    s_hitch.textureBytes += size + mipSize;
                    EnsureShadow(base, size);
                    if (mipSize)
                        EnsureShadow(mipBase, mipSize);
                    uint32_t outBpbLog2 = fi.convert == 0 ? fi.bpbLog2 : 2;
                    uint32_t rowWords, rowTexels;
                    if (fi.convert != 0 || fi.bpbLog2 >= 2)
                    {
                        rowWords = (bw << outBpbLog2) / 4;
                        rowTexels = bw * fi.block;
                    }
                    else
                    {
                        uint32_t perWord = 4 >> fi.bpbLog2;
                        rowWords = (bw + perWord - 1) / perWord;
                        rowTexels = rowWords * perWord;
                    }
                    for (uint32_t slice = 0; slice < layers * depth; slice++)
                    {
                        if (t.storeView)  // stored directly (one 2D slice, or a cube's face: see s_untileStorePipeline)
                        {
                            EndPass();
                            Begin();
                            UntileConstants uc{ base + slice * sliceStride, f.tiled ? 1u : 0u, f.tiled ? pitchAligned : rowBytes, fi.bpbLog2,
                                offX, offY, bw, bh, uint32_t(f.endianness), fi.convert, 0, 0, 0, 0 };
                            VkDescriptorImageInfo ii{ VK_NULL_HANDLE, t.storeViews[slice], VK_IMAGE_LAYOUT_GENERAL };
                            VkWriteDescriptorSet w[5] = {
                                { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 0, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &s_sharedInfo[0], nullptr },
                                { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 1, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &ii, nullptr, nullptr },
                            };
                            if (!t.uploaded && slice == 0)
                                ToGeneral(t.image.image, VK_IMAGE_ASPECT_COLOR_BIT, layers);
                            s_stateEpoch++;
                            vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_untileStorePipeline);
                            s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_untileStoreLayout, 0, AddSharedParts(w, 2), w);
                            vkCmdPushConstants(s_cmd, s_untileStoreLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uc), &uc);
                            if (!(s_exp & 4))
                                vkCmdDispatch(s_cmd, (bw + 7) / 8, (bh + 7) / 8, 1);
                            FullBarrier();
                            SubmitIfStarving();
                            continue;
                        }
                        VkDeviceSize staging = RingAllocate(VkDeviceSize(rowWords) * 4 * bh);
                        if (staging == ~VkDeviceSize(0))
                        {
                            LogOnce(0x2200, "texture %ux%u format %u is too large for a staging slot", width, height, format);
                            return dummy;
                        }
                        EndPass();
                        Begin();
                        uint32_t layer = volume ? 0 : slice, zSlice = volume ? slice : 0;
                        uint32_t sliceBase = base + (volume ? (f.tiled ? 0 : zSlice * sliceStride) : layer * sliceStride);
                        UntileConstants uc{ sliceBase, f.tiled ? 1u : 0u, f.tiled ? pitchAligned : rowBytes, fi.bpbLog2,
                            offX, offY, bw, bh, uint32_t(f.endianness), fi.convert, uint32_t(staging / 4), rowWords,
                            zSlice, heightAligned3D };
                        if (volume && f.tiled)
                            uc.tiled = 2;
                        VkDescriptorBufferInfo ring{ s_ring, 0, kRingSize };
                        VkWriteDescriptorSet w[5] = {
                            { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 0, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &s_sharedInfo[0], nullptr },
                            { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 1, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &ring, nullptr },
                        };
                        s_stateEpoch++;
                        vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_untilePipeline);
                        s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_untileLayout, 0, AddSharedParts(w, 2), w);
                        vkCmdPushConstants(s_cmd, s_untileLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uc), &uc);
                        uint32_t groupsX = fi.convert != 0 || fi.bpbLog2 >= 2 ? bw : rowWords;
                        if (!(s_exp & 4))
                            vkCmdDispatch(s_cmd, (groupsX + 7) / 8, (bh + 7) / 8, 1);
                        FullBarrier();
                        if (!t.uploaded && slice == 0)
                            ToGeneral(t.image.image, VK_IMAGE_ASPECT_COLOR_BIT, layers);
                        VkBufferImageCopy region{};
                        region.bufferOffset = staging;
                        region.bufferRowLength = rowTexels;
                        region.imageSubresource = { VK_IMAGE_ASPECT_COLOR_BIT, 0, layer, 1 };
                        region.imageOffset = { 0, 0, int32_t(zSlice) };
                        region.imageExtent = { width, height, 1 };
                        if (!(s_exp & 16))  // NFSMW_GPU_EXP bit 16: no staging copy (upload cost experiment)
                            vkCmdCopyBufferToImage(s_cmd, s_ring, t.image.image, VK_IMAGE_LAYOUT_GENERAL, 1, &region);
                        FullBarrier();
                        SubmitIfStarving();
                    }
                    // Mip levels 1..maxLevel, each face from its storage.
                    for (uint32_t level = 1; level <= maxLevel; level++)
                    {
                        uint32_t lw = std::max(width >> level, 1u), lh = std::max(height >> level, 1u);
                        uint32_t lbw = (lw + fi.block - 1) / fi.block, lbh = (lh + fi.block - 1) / fi.block;
                        uint32_t storage = mips.StorageLevel(level);
                        uint32_t levelBase = mipBase + mips.Offset(storage);
                        uint32_t levelOffX, levelOffY;
                        mips.PackedOffset(level, levelOffX, levelOffY);
                        uint32_t levelRowWords, levelRowTexels;
                        if (fi.convert != 0 || fi.bpbLog2 >= 2)
                        {
                            levelRowWords = (lbw << outBpbLog2) / 4;
                            levelRowTexels = lbw * fi.block;
                        }
                        else
                        {
                            uint32_t perWord = 4 >> fi.bpbLog2;
                            levelRowWords = (lbw + perWord - 1) / perWord;
                            levelRowTexels = levelRowWords * perWord;
                        }
                        for (uint32_t layer = 0; layer < layers; layer++)
                        {
                            VkDeviceSize staging = RingAllocate(VkDeviceSize(levelRowWords) * 4 * lbh);
                            if (staging == ~VkDeviceSize(0))
                                break;
                            EndPass();
                            Begin();
                            UntileConstants uc{ levelBase + layer * mips.SliceStride(storage), f.tiled ? 1u : 0u,
                                f.tiled ? mips.PitchBlocks(storage) : mips.RowBytes(storage), fi.bpbLog2,
                                levelOffX, levelOffY, lbw, lbh, uint32_t(f.endianness), fi.convert, uint32_t(staging / 4), levelRowWords,
                                0, 0 };
                            VkDescriptorBufferInfo ring{ s_ring, 0, kRingSize };
                            VkWriteDescriptorSet w[5] = {
                                { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 0, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &s_sharedInfo[0], nullptr },
                                { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 1, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &ring, nullptr },
                            };
                            s_stateEpoch++;
                            vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_untilePipeline);
                            s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_untileLayout, 0, AddSharedParts(w, 2), w);
                            vkCmdPushConstants(s_cmd, s_untileLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(uc), &uc);
                            uint32_t groupsX = fi.convert != 0 || fi.bpbLog2 >= 2 ? lbw : levelRowWords;
                            vkCmdDispatch(s_cmd, (groupsX + 7) / 8, (lbh + 7) / 8, 1);
                            FullBarrier();
                            VkBufferImageCopy region{};
                            region.bufferOffset = staging;
                            region.bufferRowLength = levelRowTexels;
                            region.imageSubresource = { VK_IMAGE_ASPECT_COLOR_BIT, level, layer, 1 };
                            region.imageExtent = { lw, lh, 1 };
                            if (!(s_exp & 16))  // NFSMW_GPU_EXP bit 16: no staging copy (upload cost experiment)
                                vkCmdCopyBufferToImage(s_cmd, s_ring, t.image.image, VK_IMAGE_LAYOUT_GENERAL, 1, &region);
                            FullBarrier();
                        }
                    }
                    t.uploaded = true;
                    writewatch::Watch(base, size);
                    GuardGpu(base, size, false, "texture", true);  // the untile runs later
                    if (mipSize)
                    {
                        writewatch::Watch(mipBase, mipSize);
                        GuardGpu(mipBase, mipSize, false, "texture mips", true);
                    }
                    if (s_textureVerify)
                        HashLoadedTexture(t);
                    s_stats.uploads++;
                }
                else if (s_textureVerify)
                {
                    VerifyTexture(t, epoch);
                }
            }
            if (out && !t.scaled)
                out->texture = &t;
            return TextureView(t, f.swizzle);
        }

        // Texture lookups kept per fetch slot and dimension. Dense scenes
        // look up 5,000-10,000 textures a frame, nearly all with a fetch
        // constant seen very recently, and the full lookup (layout, three
        // hash maps, the sampler) was ~16% of the command processor (Deck
        // profile). An entry is a lookup's result, used again while the
        // fetch constant's 6 dwords are the same and nothing else it depends
        // on moved: the texture cache (DropTextures), any setting
        // (anisotropy is in the sampler) and the scaled resolve memory bump
        // `generation`; for a texture the scaled path could take, whether
        // the draw renders into scaled targets is part of the key. Validity
        // is the full lookup's: unchanged while no page anywhere was written
        // (WriteEpoch), else checked in place (the texture's write
        // sequences, and whether a scaled copy became current) instead of
        // repeating the whole lookup. Each slot keeps its last lookup; a
        // table by the fetch constant's hash keeps textures that alternate
        // in a slot (about a sixth of the lookups in the career intro).
        // Placeholders and scaled textures aren't kept. NFSMW_TEXTURE_CACHE=0
        // turns it off; NFSMW_CP_CACHE_VERIFY=1 also does the full lookup on
        // every hit and logs any difference (it then returns the full
        // lookup's result).
        struct TextureCacheEntry
        {
            uint32_t fetch[6] = {};
            uint32_t generation = 0;  // TextureCache::generation when made; 0: empty
            uint8_t dim = 0;
            bool scaledCandidate = false, drawScaled = false;
            uint64_t epoch = 0;  // WriteEpoch() it was last valid at
            Texture* texture = nullptr;
            VkImageView view = VK_NULL_HANDLE;
            VkSampler sampler = VK_NULL_HANDLE;
            uint32_t base = 0, size = 0, mipBase = 0, mipSize = 0;
        };
        struct TextureCache
        {
            TextureCacheEntry slots[3][32];
            TextureCacheEntry recent[1024];
            uint32_t generation = 1;
            uint64_t settingsGeneration = 0;
            bool on = false, verify = false;
            uint64_t hits = 0, recentHits = 0, revalidated = 0, misses = 0, stale = 0, uncached = 0, mismatches = 0, raced = 0;
        };
        // On the heap (as the shader cache): the renderer's globals move by a pointer, not by the tables (4K aliasing, see NOTES).
        TextureCache* const s_textureCache = [] {
            auto* c = new TextureCache();
            const char* v = std::getenv("NFSMW_TEXTURE_CACHE");
            // (Built on the epoch rule: without it the full lookup rescans every time.)
            c->on = (g_cpOpt & CP_OPT_TEXTURE_CACHE) && (g_cpOpt & CP_OPT_TEXTURE_EPOCH) && !(v && v[0] == '0') && s_logTextureUse == 0;
            c->verify = std::getenv("NFSMW_CP_CACHE_VERIFY") != nullptr;
            return c;
        }();

        enum class TextureEntryState { kOtherKey, kStale, kValid };

        // Whether entry `e` holds this lookup's result now (see above).
        // `checked`: the texture's checkedEpoch before this looked.
        TextureEntryState CheckTextureEntry(TextureCacheEntry& e, TexDim dim, const uint32_t* fetch, bool drawScaled, uint64_t epoch,
            uint64_t& checked)
        {
            TextureCache& c = *s_textureCache;
            if (e.generation != c.generation || e.dim != uint8_t(dim) || memcmp(e.fetch, fetch, sizeof(e.fetch)) != 0 ||
                (e.scaledCandidate && e.drawScaled != drawScaled))
                return TextureEntryState::kOtherKey;
            Texture& t = *e.texture;
            checked = t.checkedEpoch;
            if (e.epoch == epoch)
                return TextureEntryState::kValid;
            // Some page was written since: the full lookup's checks, in its
            // order (its scaled copy first).
            bool valid = !(e.scaledCandidate && drawScaled && ScaledTextureValid(e.base, e.size)) &&
                (checked == epoch || (t.uploaded && writewatch::WriteSequence(e.base, e.size) <= t.loadedSequence &&
                    (!e.mipSize || writewatch::WriteSequence(e.mipBase, e.mipSize) <= t.loadedSequence)));
            if (!valid)
            {
                c.stale++;
                return TextureEntryState::kStale;
            }
            t.checkedEpoch = epoch;
            e.epoch = epoch;
            c.revalidated++;
            return TextureEntryState::kValid;
        }

        TextureCacheEntry& RecentTextureEntry(const uint32_t* fetch, TexDim dim)
        {
            uint64_t a, b, d;
            memcpy(&a, fetch, 8);
            memcpy(&b, fetch + 2, 8);
            memcpy(&d, fetch + 4, 8);
            uint64_t h = (a * 0x9E3779B97F4A7C15ull) ^ (b * 0xC2B2AE3D27D4EB4Full) ^ (d * 0x165667B19E3779F9ull) ^ uint64_t(dim);
            return s_textureCache->recent[(h * 0x9E3779B97F4A7C15ull) >> 54];
        }

        VkImageView GetTexture(uint32_t slot, TexDim dim, VkSampler& sampler)
        {
            TextureCache& c = *s_textureCache;
            if (!c.on)
                return LookupTexture(slot, dim, sampler, nullptr);
            const uint32_t* fetch = &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + slot * 6];
            TextureCacheEntry& e = c.slots[uint32_t(dim)][slot & 31];
            const bool drawScaled = s_drawScaled && s_scaled.enabled;
            const uint64_t epoch = writewatch::WriteEpoch();
            uint64_t checked = 0;
            TextureEntryState state = CheckTextureEntry(e, dim, fetch, drawScaled, epoch, checked);
            if (state == TextureEntryState::kOtherKey)
            {
                TextureCacheEntry& r = RecentTextureEntry(fetch, dim);
                state = CheckTextureEntry(r, dim, fetch, drawScaled, epoch, checked);
                if (state == TextureEntryState::kValid)
                {
                    e = r;
                    c.recentHits++;
                }
            }
            if (state == TextureEntryState::kValid)
            {
                c.hits++;
                if (s_textureVerify)
                    VerifyTexture(*e.texture, epoch);
                if (!c.verify)
                {
                    sampler = e.sampler;
                    return e.view;
                }
                // The full lookup as if this hit hadn't happened: a reload
                // it does is one the hit would have missed, unless a page
                // was written during the check (another thread): the full
                // lookup then saw memory the hit, a moment earlier, didn't.
                e.texture->checkedEpoch = checked;
                uint32_t loads = s_hitch.textures;
                TextureLookup lookup;
                VkImageView view = LookupTexture(slot, dim, sampler, &lookup);
                bool reloaded = s_hitch.textures != loads;
                if (reloaded && writewatch::WriteEpoch() != epoch)
                {
                    c.raced++;
                    reloaded = false;
                }
                if (view != e.view || sampler != e.sampler || lookup.texture != e.texture || reloaded)
                {
                    if (c.mismatches++ < 50)
                        fprintf(stderr, "[cpverify] texture slot %u dim %u: cached view %p sampler %p texture %p, full lookup %p %p %p%s "
                            "(fetch %08X %08X %08X %08X %08X %08X, epoch %llu, entry epoch %llu, scaled %d/%d)\n",
                            slot, uint32_t(dim), (void*)e.view, (void*)e.sampler, (void*)e.texture, (void*)view, (void*)sampler,
                            (void*)lookup.texture, reloaded ? ", reloaded" : "", fetch[0], fetch[1], fetch[2], fetch[3],
                            fetch[4], fetch[5], (unsigned long long)epoch, (unsigned long long)e.epoch, int(e.scaledCandidate), int(drawScaled));
                    e.generation = 0;
                    RecentTextureEntry(fetch, dim).generation = 0;
                }
                return view;
            }
            c.misses++;
            TextureLookup lookup;
            VkImageView view = LookupTexture(slot, dim, sampler, &lookup);
            if (!lookup.texture)
            {
                c.uncached++;
                e.generation = 0;
                return view;
            }
            memcpy(e.fetch, fetch, sizeof(e.fetch));
            e.generation = c.generation;
            e.dim = uint8_t(dim);
            e.scaledCandidate = lookup.scaledCandidate;
            e.drawScaled = drawScaled;
            e.epoch = epoch;  // read before the lookup: a write meanwhile is checked next time
            e.texture = lookup.texture;
            e.view = view;
            e.sampler = sampler;
            e.base = lookup.base;
            e.size = lookup.size;
            e.mipBase = lookup.mipBase;
            e.mipSize = lookup.mipSize;
            RecentTextureEntry(fetch, dim) = e;
            return view;
        }

        // ---------------------------------------------------------------
        // Resolves

        VkPipeline s_resolvePipeline = VK_NULL_HANDLE;
        VkPipeline s_resolveCoalescedPipeline = VK_NULL_HANDLE;  // scaled resolves (see COALESCED in the shader)
        // The same for a source of another scale denominator ([coalesced][den x - 1][den y - 1]: 1.5x is 3/2), made
        // at the first resolve that needs it (ResolvePipeline); [..][0][0] are the two above.
        VkPipeline s_resolveDenPipelines[2][2][2] = {};
        VkPipeline ResolvePipeline(bool coalesced, const gpu::RenderScale& scale);

        // Coalesced scaled resolves: 2.8 -> 1.1 ms of scaled resolves a frame
        // on the Steam Machine (RADV), but 2.6-3.4 -> 4.1-4.4 ms on an M1 Pro
        // (MoltenVK), so not on Apple GPUs unless NFSMW_SCALED_COALESCE=1.
        bool ScaledCoalesce(const VkPhysicalDeviceProperties& props)
        {
            const char* coalesce = std::getenv("NFSMW_SCALED_COALESCE");
            return coalesce ? coalesce[0] != '0' : props.vendorID != 0x106B;
        }

        // The resolve pipelines a scale will use, made now (at start or at a
        // live change, with the menu open) rather than at its first resolve
        // mid-frame: a new one costs ~0.4 s on MoltenVK (once; the pipeline
        // cache keeps it), 23 ms on RADV.
        void PrewarmResolvePipelines(const gpu::RenderScale& scale)
        {
            if (scale.x.den == 1 && scale.y.den == 1)
                return;
            VkPhysicalDeviceProperties props;
            vkGetPhysicalDeviceProperties(s_vk->physical, &props);
            ResolvePipeline(false, scale);
            if (ScaledCoalesce(props))
                ResolvePipeline(true, scale);
        }
        VkPipelineLayout s_resolveLayout = VK_NULL_HANDLE;
        VkDescriptorSetLayout s_resolveSetLayout = VK_NULL_HANDLE;

        struct ResolveConstants
        {
            int32_t originX, originY;
            uint32_t width, height;
            uint32_t destBase, destPitch;
            uint32_t format, endian;
            uint32_t flags;  // bit 0: swap red/blue; bit 1: depth source
            float expScale;
            uint32_t srcWidth, srcHeight;
            int32_t srcOriginX, srcOriginY;  // host pixels
            uint32_t scaleX, scaleY;         // host pixels per guest pixel in the source (/ the pipeline's SB_DEN_X, SB_DEN_Y)
            uint32_t scaledWrite, slotWords; // also write the samples to the scaled memory
            // Also store into a cached direct texture (binding 4), as its
            // untile would have from the words (see Resolve); texX, texY:
            // where the destination's texel 0,0 is in it (NFSMW_FUSE_PARTIAL,
            // NFSMW_DIRECT_CUBE; pushed only with either, see s_resolvePush).
            uint32_t texWrite, texEndian, texConvert, texWidth, texHeight;
            uint32_t texX, texY;
        };
        // The constants pushed to the resolve shader: without NFSMW_FUSE_PARTIAL
        // and NFSMW_DIRECT_CUBE, those before texX, as before them, and the
        // shader is built without TEX_OFFSET (the same code and push).
        const uint32_t s_resolvePush = s_fusePartial || s_directCube ? sizeof(ResolveConstants) : offsetof(ResolveConstants, texX);

        const char* kResolveGlsl = R"(#version 460
#extension GL_EXT_samplerless_texture_functions : require
layout(local_size_x = 8, local_size_y = 8) in;
layout(std430, set = 0, binding = 0) buffer SharedMemory { uint g_mem[]; };)" GLSL_SHARED_MEMORY_WRITE R"(
layout(set = 0, binding = 1) uniform texture2D src;
layout(std430, set = 0, binding = 2) buffer ScaledPool { uint s_pool[]; };
layout(std430, set = 0, binding = 3) readonly buffer ScaledTable { uint s_table[]; };
layout(set = 0, binding = 4, r32ui) writeonly uniform uimage2D t_image;
layout(push_constant) uniform P
{
    ivec2 origin;
    uvec2 size;
    uint destBase, destPitch, format, endian, flags;
    float expScale;
    uvec2 srcSize;
    ivec2 srcOrigin;    // where the rectangle is in the host target (see untiled rendering), host pixels
    uvec2 scale;        // host pixels per guest pixel (internal resolution): scale / kDen
    uint scaledWrite, slotWords;
    uint texWrite, texEndian, texConvert, texWidth, texHeight;  // also into a cached texture (binding 4)
#ifdef TEX_OFFSET
    uint texX, texY;    // the destination's texel 0,0 in it (whole tiles)
#endif
} p;
)" GLSL_SCALE_CODE R"(
// The scale's denominators are the pipeline's own (CreateResolvePipeline: a
// pair of pipelines for each, made at the first resolve that needs it), not
// push constants: at a whole scale (1, 1) every block and weight expression
// folds back to the arithmetic from before 1.5x, with no integer division.
#ifndef SB_DEN_X
#define SB_DEN_X 1u
#endif
#ifndef SB_DEN_Y
#define SB_DEN_Y 1u
#endif
const uvec2 kDen = uvec2(SB_DEN_X, SB_DEN_Y);

uint tiled2D(uint x, uint y, uint pitch, uint bppLog2)
{
    uint outer = (((y >> 5) * (pitch >> 5)) + (x >> 5)) << 6;
    uint inner = (((y >> 1) & 7u) << 3) | (x & 7u);
    uint oi = (outer | inner) << bppLog2;
    uint bank = (y >> 4) & 1u;
    uint pipe = ((x >> 3) & 3u) ^ (((y >> 3) & 1u) << 1);
    return ((y & 1u) << 4) | (pipe << 6) | (bank << 11) | (oi & 0xFu) | (((oi >> 4) & 1u) << 5) |
        (((oi >> 5) & 7u) << 8) | ((oi >> 8) << 12);
}

uint swapWord(uint v, uint e)
{
    if (e == 1u) return ((v << 8) & 0xFF00FF00u) | ((v >> 8) & 0x00FF00FFu);
    if (e == 2u || e >= 4u) return (v << 24) | ((v << 8) & 0x00FF0000u) | ((v >> 8) & 0x0000FF00u) | (v >> 24);
    if (e == 3u) return (v >> 16) | (v << 16);
    return v;
}

uint bppLog2(uint f)
{
    if (f == 2u) return 0u;                                  // 8
    if (f == 3u || f == 4u || f == 10u || f == 15u || f == 24u || f == 30u) return 1u;
    if (f == 26u || f == 32u || f == 34u || f == 37u) return 3u;
    return 2u;
}

// px() at 1.5x (a den of 2): the host pixels the guest pixel covers, each
// weighted by how much of it lies inside (gpu/render_scale.h's box filter:
// 2/3 and 1/3, in 1/den of a pixel, num of them a whole guest pixel), or
// (depth) the one holding its centre. The rectangle's origin is whole blocks
// (even), so its pixels' host coordinates count from srcOrigin.
vec4 pxFraction(uint x, uint y)
{
    if ((p.flags & 2u) != 0u)
        return texelFetch(src, clamp(p.srcOrigin + ivec2(sbCenterHost(x, p.scale.x, kDen.x), sbCenterHost(y, p.scale.y, kDen.y)),
            ivec2(0), ivec2(p.srcSize) - 1), 0);
    vec4 v = vec4(0.0);
    uint x1 = sbHostSize(x + 1u, p.scale.x, kDen.x), y1 = sbHostSize(y + 1u, p.scale.y, kDen.y);
    for (uint j = sbHostEdge(y, p.scale.y, kDen.y); j < y1; j++)
        for (uint i = sbHostEdge(x, p.scale.x, kDen.x); i < x1; i++)
            v += float(sbOverlap(x, i, p.scale.x, kDen.x) * sbOverlap(y, j, p.scale.y, kDen.y)) *
                texelFetch(src, clamp(p.srcOrigin + ivec2(i, j), ivec2(0), ivec2(p.srcSize) - 1), 0);
    v /= float(p.scale.x * p.scale.y);
    v *= p.expScale;
    if ((p.flags & 1u) != 0u) v = v.bgra;
    return v;
}

vec4 px(uint x, uint y)
{
    // Guest pixel (x, y) of the rectangle: its sx x sy block of host pixels,
    // averaged (colour) or its first sample (depth).
    if (kDen.x * kDen.y > 1u) return pxFraction(x, y);
    ivec2 c = clamp(p.srcOrigin + ivec2(x * p.scale.x, y * p.scale.y), ivec2(0), ivec2(p.srcSize) - 1);
    vec4 v = texelFetch(src, c, 0);
    if ((p.flags & 2u) != 0u) return v;
    if (p.scale.x * p.scale.y > 1u)
    {
        for (uint j = 0u; j < p.scale.y; j++)
            for (uint i = 0u; i < p.scale.x; i++)
                if (i + j > 0u)
                    v += texelFetch(src, clamp(c + ivec2(i, j), ivec2(0), ivec2(p.srcSize) - 1), 0);
        v /= float(p.scale.x * p.scale.y);
    }
    v *= p.expScale;
    if ((p.flags & 1u) != 0u) v = v.bgra;
    return v;
}

uint unorm(float v, float m) { return uint(clamp(v, 0.0, 1.0) * m + 0.5); }
)" GLSL_DEPTH24_PACK R"(
uint pack32(vec4 c)
{
    switch (p.format)
    {
    case 6u: case 14u: return packUnorm4x8(c);
    case 7u: return unorm(c.r, 1023.0) | (unorm(c.g, 1023.0) << 10) | (unorm(c.b, 1023.0) << 20) | (unorm(c.a, 3.0) << 30);
    case 22u: case 23u: return packDepth24(c.r) << 8;  // (not unorm: 1.0 overflows 24 bits, see depth24_glsl.h)
    case 25u: return packSnorm2x16(c.rg);
    case 31u: return packHalf2x16(c.rg);
    case 33u: case 36u: return floatBitsToUint(c.r);
    default: return packUnorm4x8(c);
    }
}

uint pack16(vec4 c)
{
    switch (p.format)
    {
    case 3u: return unorm(c.r, 31.0) | (unorm(c.g, 31.0) << 5) | (unorm(c.b, 31.0) << 10) | (unorm(c.a, 1.0) << 15);
    case 4u: return unorm(c.r, 31.0) | (unorm(c.g, 63.0) << 5) | (unorm(c.b, 31.0) << 11);
    case 15u: return unorm(c.r, 15.0) | (unorm(c.g, 15.0) << 4) | (unorm(c.b, 15.0) << 8) | (unorm(c.a, 15.0) << 12);
    case 10u: return unorm(c.r, 255.0) | (unorm(c.g, 255.0) << 8);
    case 30u: return packHalf2x16(vec2(c.r, 0.0)) & 0xFFFFu;
    default: return unorm(c.r, 65535.0);
    }
}

#ifdef COALESCED
// A scaled resolve's samples stored coalesced: a group is 32x2 pixels, its
// threads in the order of their words in the 32-bit tiled layout (64
// consecutive words when aligned), and the samples go to the pool through
// shared memory so that each store writes consecutive pool words. Each
// thread storing its own pixel's samples writes words `s` apart across the
// group, several times the memory transactions: most of a scaled resolve's
// cost. (A pipeline of its own: the shared memory would slow the others.)
// By block (gpu/render_scale.h): a pixel at a whole scale, 2x2 pixels at
// 1.5x (a group is 16 whole blocks: its rows start even, as its columns).
shared uint g_samples[64 * 16];  // by block, then sample (up to 64 pixels' worth at 4x4)
shared uint g_offsets[64];       // each block's samples in the pool; ~0u: none
#endif

void main()
{
    uint bl = bppLog2(p.format);
    uint perWord = bl >= 2u ? 1u : (4u >> bl);
#ifdef COALESCED
    uint l = gl_LocalInvocationIndex;
    uvec2 id = gl_WorkGroupID.xy * uvec2(32u, 2u) + uvec2((l & 3u) | ((l >> 3u) << 2u), (l >> 2u) & 1u);
#else
    uvec2 id = gl_GlobalInvocationID.xy;
#endif
    uint gx = id.x * perWord, gy = id.y;
    bool inside = gx < p.size.x && gy < p.size.y;
#ifndef COALESCED
    if (!inside) return;  // (a coalesced write needs every thread at its barrier)
#endif
    uint x = uint(p.origin.x) + gx, y = uint(p.origin.y) + gy;
    uint word = ((p.destBase + tiled2D(x, y, p.destPitch, bl)) & 0x1FFFFFFFu) >> 2;
    uint w;
    if (bl == 2u && p.scaledWrite != 0u)
    {
        // Each host sample this guest pixel owns into its block's place in
        // the page's scaled slot (see ScaledMemory, gpu/render_scale.h): at a
        // whole scale all its sx x sy, at 1.5x one, two or four of its 2x2
        // block's 3x3, from where its texel sits in the block (its word's
        // bits 0 and 2; the rectangle's origin is even, so gx and gy agree).
        uint a = word << 2;
        uint slot = inside ? s_table[a >> 12] : 0u;
        uint s = p.scale.x * p.scale.y;  // samples a block
        uint k = (a & 4095u) >> 2;
        uint o = (slot - 1u) * p.slotWords + sbBlockIndex(k, kDen.x, kDen.y) * s;
        uint subX = sbSubX(k, kDen.x), subY = sbSubY(k, kDen.y);
        uint i0 = sbHostEdge(subX, p.scale.x, kDen.x), i1 = sbHostEdge(subX + 1u, p.scale.x, kDen.x);
        uint j0 = sbHostEdge(subY, p.scale.y, kDen.y), j1 = sbHostEdge(subY + 1u, p.scale.y, kDen.y);
#ifdef COALESCED
        uint lb = sbBlockIndex(l, kDen.x, kDen.y);  // the block's in this group (l is the word order)
#endif
        if (slot != 0u)
        {
            ivec2 c0 = p.srcOrigin + ivec2(sbHostEdge(gx, p.scale.x, kDen.x), sbHostEdge(gy, p.scale.y, kDen.y)) - ivec2(i0, j0);
            for (uint j = j0; j < j1; j++)
                for (uint i = i0; i < i1; i++)
                {
                    vec4 v = texelFetch(src, clamp(c0 + ivec2(i, j), ivec2(0), ivec2(p.srcSize) - 1), 0);
                    if ((p.flags & 2u) == 0u)
                    {
                        v *= p.expScale;
                        if ((p.flags & 1u) != 0u) v = v.bgra;
                    }
#ifdef COALESCED
                    g_samples[lb * s + j * p.scale.x + i] = swapWord(pack32(v), p.endian);
#else
                    s_pool[o + j * p.scale.x + i] = swapWord(pack32(v), p.endian);
#endif
                }
        }
#ifdef COALESCED
        if (subX == 0u && subY == 0u)  // (every pixel at a whole scale)
            g_offsets[lb] = slot != 0u ? o : ~0u;
        barrier();
        uint samples = 64u / (kDen.x * kDen.y) * s;
        for (uint q = l; q < samples; q += 64u)
        {
            uint po = g_offsets[q / s];
            if (po != ~0u)
                s_pool[po + q % s] = g_samples[q];
        }
#endif
    }
    if (!inside) return;
    if (bl == 3u)
    {
        vec4 c = px(gx, gy);
        uint w0, w1;
        if (p.format == 26u) { w0 = packSnorm2x16(c.rg); w1 = packSnorm2x16(c.ba); }
        else if (p.format == 32u) { w0 = packHalf2x16(c.rg); w1 = packHalf2x16(c.ba); }
        else { w0 = floatBitsToUint(c.r); w1 = floatBitsToUint(c.g); }
        if (p.endian == 4u) { uint t = w0; w0 = swapWord(w1, 2u); w1 = swapWord(t, 2u); }
        else { w0 = swapWord(w0, p.endian); w1 = swapWord(w1, p.endian); }
        memStore(word, w0);
        memStore(word + 1u, w1);
        return;
    }
    if (bl == 2u) w = pack32(px(gx, gy));
    else if (bl == 1u) w = pack16(px(gx, gy)) | (pack16(px(gx + 1u, gy)) << 16);
    else w = unorm(px(gx, gy).r, 255.0) | (unorm(px(gx + 1u, gy).r, 255.0) << 8) |
        (unorm(px(gx + 2u, gy).r, 255.0) << 16) | (unorm(px(gx + 3u, gy).r, 255.0) << 24);
    uint m = swapWord(w, p.endian);
    memStore(word, m);
    // A cached direct 1x texture of this surface: its texel as the untile
    // would read the word (its endianness, D24 -> float). Or (TEX_OFFSET) one
    // holding it whole tiles in: 32-bit texels make a 4 KB tile, and
    // tiled2D's address is that tile's index in bits 12 and up, the rest from
    // x & 31 and y & 31 only, so the destination's x, y is the texture's
    // x + texX, y + texY.
#ifdef TEX_OFFSET
    if (p.texWrite != 0u && bl == 2u && x + p.texX < p.texWidth && y + p.texY < p.texHeight)
#else
    if (p.texWrite != 0u && bl == 2u && x < p.texWidth && y < p.texHeight)
#endif
    {
        uint v = swapWord(m, p.texEndian);
        // (D24: m is packDepth24's, clamped, so the far plane stores 1.0 here
        // too, not 0.)
        uint o = p.texConvert == 4u ? floatBitsToUint(float(v >> 8) / 16777215.0) : v;
#ifdef TEX_OFFSET
        imageStore(t_image, ivec2(int(x + p.texX), int(y + p.texY)), uvec4(o, 0u, 0u, 0u));
#else
        imageStore(t_image, ivec2(int(x), int(y)), uvec4(o, 0u, 0u, 0u));
#endif
    }
}
)";

        uint32_t ResolveBppLog2(uint32_t f)
        {
            if (f == 2) return 0;
            if (f == 3 || f == 4 || f == 10 || f == 15 || f == 24 || f == 30) return 1;
            if (f == 26 || f == 32 || f == 34 || f == 37) return 3;
            return 2;
        }

        // The guest bytes a resolve of `r` writes, as offsets from the
        // destination base: the 4 KB blocks of the 32x32 tiles the rectangle
        // touches (Tiled2D puts the tile index, shifted by the bytes per
        // pixel, in address bits 12 and up). Not every row of the pitch down
        // to a 32-row boundary: at world entry NFSMW resolves the right
        // 640x720 half of a 1280x720 texture through a destination base 20
        // tiles into it, so whole rows from there ran 80 KB past the
        // texture, into Direct3D's command ring right after it. In shadow
        // mode (discrete GPUs) the pages a resolve marks are read back whole
        // from the GPU copy, which put old bytes over the game's newest
        // commands: a malformed indirect buffer, a hang.
        // NFSMW_RESOLVE_EXTENT=0 marks whole rows again.
        std::pair<uint32_t, uint32_t> ResolveExtent(const Rect& r, uint32_t pitchAligned, uint32_t bppLog2)
        {
            static const bool tiles = [] { const char* v = std::getenv("NFSMW_RESOLVE_EXTENT"); return !v || v[0] != '0'; }();
            if (!tiles)
                return { 0, (pitchAligned * ((uint32_t(r.y1) + 31) & ~31u)) << bppLog2 };
            uint32_t tilesPerRow = pitchAligned >> 5;
            uint32_t first = (uint32_t(r.y0) >> 5) * tilesPerRow + (uint32_t(r.x0) >> 5);
            uint32_t last = (uint32_t(r.y1 - 1) >> 5) * tilesPerRow + (uint32_t(r.x1 - 1) >> 5);
            return { ((first << (6 + bppLog2)) >> 8) << 12, (((((last + 1) << (6 + bppLog2)) - 1) >> 8) + 1) << 12 };
        }

        bool ResolveFormatSupported(uint32_t f)
        {
            switch (f)
            {
            case 2: case 3: case 4: case 6: case 7: case 10: case 14: case 15: case 22: case 23: case 24:
            case 25: case 26: case 30: case 31: case 32: case 33: case 34: case 36: case 37:
                return true;
            default:
                return false;
            }
        }

        int32_t ToFixed16p8(float f)
        {
            float v = std::clamp(f, -8388608.0f / 256.0f, 8388607.0f / 256.0f) * 256.0f;
            return int32_t(v + (v < 0 ? -0.5f : 0.5f));
        }

        // Clear a rectangle of a target (resolve with clear enabled).
        void ClearRect(RenderTarget* rt, const Rect& r)
        {
            VkClearAttachment ca{};
            if (rt->depth)
            {
                uint32_t v = s_regs[XE_GPU_REG_RB_DEPTH_CLEAR];
                ca.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
                float depth;
                if (xenos::DepthRenderTargetFormat(rt->format) == xenos::DepthRenderTargetFormat::kD24FS8)
                {
                    // 20e4 float: 4-bit exponent (bias 15), 20-bit mantissa.
                    uint32_t f24 = v >> 8;
                    uint32_t e = f24 >> 20, m = f24 & 0xFFFFF;
                    depth = e == 0 ? std::ldexp(float(m), -34) : std::ldexp(1.0f + float(m) / 1048576.0f, int(e) - 15);
                }
                else
                {
                    depth = float(v >> 8) / 16777215.0f;
                }
                ca.clearValue.depthStencil = { std::clamp(depth, 0.0f, 1.0f), v & 0xFF };
            }
            else
            {
                uint32_t lo = s_regs[XE_GPU_REG_RB_COLOR_CLEAR];
                uint32_t hi = s_regs[XE_GPU_REG_RB_COLOR_CLEAR_LO];
                ca.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
                ca.colorAttachment = 0;
                float* c = ca.clearValue.color.float32;
                auto half = [](uint32_t h) {
                    uint32_t s = (h >> 15) & 1, e = (h >> 10) & 0x1F, m = h & 0x3FF;
                    float v = e == 0 ? std::ldexp(float(m), -24) : e == 31 ? 65504.0f : std::ldexp(1.0f + m / 1024.0f, int(e) - 15);
                    return s ? -v : v;
                };
                switch (rt->vkFormat)
                {
                case VK_FORMAT_R8G8B8A8_UNORM:
                    for (int i = 0; i < 4; i++)
                        c[i] = float((lo >> (8 * i)) & 0xFF) / 255.0f;
                    break;
                case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
                    c[0] = float(lo & 0x3FF) / 1023.0f;
                    c[1] = float((lo >> 10) & 0x3FF) / 1023.0f;
                    c[2] = float((lo >> 20) & 0x3FF) / 1023.0f;
                    c[3] = float(lo >> 30) / 3.0f;
                    break;
                case VK_FORMAT_R16G16_SFLOAT:
                    c[0] = half(lo & 0xFFFF);
                    c[1] = half(lo >> 16);
                    break;
                case VK_FORMAT_R16G16B16A16_SFLOAT:
                    c[0] = half(lo & 0xFFFF);
                    c[1] = half(lo >> 16);
                    c[2] = half(hi & 0xFFFF);
                    c[3] = half(hi >> 16);
                    break;
                case VK_FORMAT_R32_SFLOAT:
                    memcpy(&c[0], &lo, 4);
                    break;
                case VK_FORMAT_R32G32_SFLOAT:
                    memcpy(&c[0], &lo, 4);
                    memcpy(&c[1], &hi, 4);
                    break;
                default:
                    for (int i = 0; i < 4; i++)
                        c[i] = 0.0f;
                    break;
                }
            }
            // `r` is in guest pixels; the clear covers its host pixels.
            int32_t x0 = rt->scale.x.EdgeAt(std::clamp(r.x0, 0, int32_t(rt->guestWidth)));
            int32_t y0 = rt->scale.y.EdgeAt(std::clamp(r.y0, 0, int32_t(rt->guestHeight)));
            int32_t x1 = rt->scale.x.EdgeAt(std::clamp(r.x1, 0, int32_t(rt->guestWidth)));
            int32_t y1 = rt->scale.y.EdgeAt(std::clamp(r.y1, 0, int32_t(rt->guestHeight)));
            x1 = std::max(x1, x0);
            y1 = std::max(y1, y0);
            if (x1 <= x0 || y1 <= y0)
                return;
            // Deferred (see s_pendingClears); a pass open on the target now
            // must end so the clear lands before its next use.
            if (s_passActive && (s_passDepth == rt || std::find(s_passColor, s_passColor + 4, rt) != s_passColor + 4))
                EndPass();
            VkRect2D rect{ { x0, y0 }, { uint32_t(x1 - x0), uint32_t(y1 - y0) } };
            // NFSMW_CLEAR_FLUSH_RECT: tile bands left pending by the resolves
            // between them become one clear (a load-op clear when it covers the
            // next pass). Only with the target's newest pending clear, so no
            // clear of the target lies between the two (values compared as
            // bytes: `ca` is zeroed, so the bytes a format leaves unused agree).
            if (s_clearFlushRect)
                for (size_t i = s_pendingClears.size(); i-- > 0;)
                    if (s_pendingClears[i].rt == rt)
                    {
                        if (!memcmp(&s_pendingClears[i].value, &ca.clearValue, sizeof(VkClearValue)) &&
                            Adjoins(s_pendingClears[i].rect, rect))
                        {
                            rect = Union(s_pendingClears[i].rect, rect);
                            s_pendingClears.erase(s_pendingClears.begin() + i);
                        }
                        break;
                    }
            // A newer clear covering an older one replaces it.
            std::erase_if(s_pendingClears, [&](const PendingClear& pc) { return pc.rt == rt && Contains(rect, pc.rect); });
            s_pendingClears.push_back({ rt, rect, ca.clearValue });
        }

        // NFSMW_LOG_SHADOWMAP=1 (diagnostic): every 5 s, the first depth
        // resolve from a surface of pitch 1600 (the sun's shadow map) is
        // copied back after it, and a [shadowmap] line gives the share of its
        // texels that are exactly 0 (nearest) and exactly 0xFFFFFF (far: the
        // clear, no caster there). Before d24-fix every far texel read 0
        // (depth24_glsl.h). A GPU copy into host memory: guest memory may be
        // the GPU's copy in VRAM (shadow mode), uncached to the CPU. Once a
        // swap finds the copy's submission done, a thread of its own counts
        // the 2.56M texels. (Counted at the swap, on the command processor,
        // they made the next frame late: after every sample on the Steam
        // Machine, 3-5 ms more CP time and a ~20 ms frame.) The command
        // processor still records the copy (10 MB) and its two barriers
        // every 5 s, so time perf with the probe off. Off (the default): a
        // constant tested per resolve and per swap, nothing allocated.
        const bool s_logShadowMap = [] { const char* v = std::getenv("NFSMW_LOG_SHADOWMAP"); return v && v[0] && v[0] != '0'; }();
        struct ShadowMapProbe
        {
            VkBuffer buffer = VK_NULL_HANDLE;
            VkDeviceMemory memory = VK_NULL_HANDLE;
            const uint8_t* host = nullptr;
            bool coherent = true;
            VkDeviceSize size = 0;
            bool failed = false;
            uint64_t submission = 0, frame = 0;  // the copy's submission, and the frame
            double seconds = 0;
            uint32_t rangeBase = 0, rangeBytes = 0, destBase = 0, destPitch = 0, endian = 0, format = 0, srcBase = 0;
            Rect rect{};
            std::chrono::steady_clock::time_point next{};
        };
        std::unique_ptr<ShadowMapProbe> s_shadowMapProbe;
        std::chrono::steady_clock::time_point s_shadowMapClock{};  // the first swap
        // Idle: a resolve may record a copy. Recorded: the copy waits for its
        // submission. Counting: the counting thread reads the buffer, and the
        // command processor neither refills nor reallocates it until the
        // thread sets idle (release; the resolve's check acquires). Static
        // storage: a count still running at exit outlives s_shadowMapProbe.
        enum : uint8_t { kProbeIdle, kProbeRecorded, kProbeCounting };
        std::atomic<uint8_t> s_shadowMapState{ kProbeIdle };

        // What the counting thread needs, copied: it reads the buffer only.
        struct ShadowMapSample
        {
            const uint8_t* host;
            uint32_t rangeBase, rangeBytes, destBase, destPitch, endian, format, srcBase;
            Rect rect;
            uint64_t frame;
            double seconds;
        };

        void ShadowMapCount(ShadowMapSample p)
        {
            SetHostThreadName("nfsmw-shadowmap");
            hostcpu::LeaveReservedCore();  // started by the command processor
#ifdef __APPLE__
            pthread_set_qos_class_self_np(QOS_CLASS_UTILITY, 0);  // (not the CP's interactive class)
#endif
            // By 32x32 tiles: 32-bit texels make a tile 4 KB of consecutive
            // bytes (Tiled2D), so a tile wholly inside the rectangle is read
            // in order; the rest texel by texel.
            // (farTexels, not "far": <windows.h> defines far and near as nothing.)
            uint64_t texels = 0, zero = 0, farTexels = 0, outside = 0;
            auto count = [&](uint32_t offset) {
                if (uint64_t(offset) + 4 > p.rangeBytes)
                {
                    outside++;
                    return;
                }
                uint32_t word;
                memcpy(&word, p.host + offset, 4);
                uint32_t d = GpuSwap(word, p.endian >= 4 ? 2 : p.endian) >> 8;
                texels++;
                zero += d == 0;
                farTexels += d == 0xFFFFFF;
            };
            auto offsetOf = [&](uint32_t x, uint32_t y) { return ((p.destBase + Tiled2D(x, y, p.destPitch, 2)) & 0x1FFFFFFF) - p.rangeBase; };
            const uint32_t x0 = uint32_t(p.rect.x0), y0 = uint32_t(p.rect.y0), x1 = uint32_t(p.rect.x1), y1 = uint32_t(p.rect.y1);
            for (uint32_t ty = y0 >> 5; ty <= (y1 - 1) >> 5; ty++)
                for (uint32_t tx = x0 >> 5; tx <= (x1 - 1) >> 5; tx++)
                {
                    uint32_t ax = std::max(x0, tx * 32), bx = std::min(x1, tx * 32 + 32);
                    uint32_t ay = std::max(y0, ty * 32), by = std::min(y1, ty * 32 + 32);
                    if (bx - ax == 32 && by - ay == 32)
                        for (uint32_t i = 0, base = offsetOf(tx * 32, ty * 32); i < 1024; i++)
                            count(base + i * 4);
                    else
                        for (uint32_t y = ay; y < by; y++)
                            for (uint32_t x = ax; x < bx; x++)
                                count(offsetOf(x, y));
                }
            auto pct = [&](uint64_t n) { return texels ? 100.0 * double(n) / double(texels) : 0.0; };
            fprintf(stderr, "[shadowmap] frame %llu (%.1f s): depth resolve b%u p1600 fmt %u rect (%d,%d)-(%d,%d) -> %08X: %llu texels, "
                "zero %.2f%%, far %.2f%%%s\n", (unsigned long long)p.frame, p.seconds, p.srcBase, p.format, p.rect.x0, p.rect.y0,
                p.rect.x1, p.rect.y1, p.destBase, (unsigned long long)texels, pct(zero), pct(farTexels),
                outside ? std::format(" ({} texels outside the copy)", outside).c_str() : "");
            s_shadowMapState.store(kProbeIdle, std::memory_order_release);
        }

        void ShadowMapProbeRecord(uint32_t rangeBase, uint32_t rangeBytes, const ResolveConstants& rc, const Rect& r, uint32_t srcBase)
        {
            if (!s_shadowMapProbe)
                s_shadowMapProbe = std::make_unique<ShadowMapProbe>();
            ShadowMapProbe& p = *s_shadowMapProbe;
            auto now = std::chrono::steady_clock::now();
            if (p.failed || s_shadowMapState.load(std::memory_order_acquire) != kProbeIdle || now < p.next || rangeBytes == 0)
                return;
            if (p.size < rangeBytes)
            {
                if (p.buffer)
                {
                    vkDestroyBuffer(s_dev, p.buffer, nullptr);
                    vkFreeMemory(s_dev, p.memory, nullptr);
                    p.buffer = VK_NULL_HANDLE;
                    p.memory = VK_NULL_HANDLE;
                    p.host = nullptr;
                    p.size = 0;
                }
                VkBufferCreateInfo bci{ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
                bci.size = rangeBytes;
                bci.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
                VkMemoryRequirements req{};
                VkMemoryAllocateInfo mai{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
                void* mapped = nullptr;
                const VkMemoryPropertyFlags visible = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
                bool ok = vkCreateBuffer(s_dev, &bci, nullptr, &p.buffer) == VK_SUCCESS;
                if (ok)
                {
                    vkGetBufferMemoryRequirements(s_dev, p.buffer, &req);
                    mai.allocationSize = req.size;
                    // Cached host memory when there is some (the CPU reads the
                    // copy), else any host-visible coherent type. (Inline: the
                    // memory plan, gpu/memory_plan.h, replaced the renderer's
                    // FindMemoryTypePreferring; the windows branch does the same.)
                    const VkMemoryPropertyFlags cached = visible | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
                    mai.memoryTypeIndex = UINT32_MAX;
                    for (uint32_t i = 0; i < s_memProps.memoryTypeCount && mai.memoryTypeIndex == UINT32_MAX; i++)
                        if ((req.memoryTypeBits & (1u << i)) && (s_memProps.memoryTypes[i].propertyFlags & cached) == cached)
                            mai.memoryTypeIndex = i;
                    if (mai.memoryTypeIndex == UINT32_MAX)
                        mai.memoryTypeIndex = FindMemoryType(req.memoryTypeBits, visible | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
                    ok = (s_memProps.memoryTypes[mai.memoryTypeIndex].propertyFlags & visible) == visible &&
                        vkAllocateMemory(s_dev, &mai, nullptr, &p.memory) == VK_SUCCESS;
                }
                ok = ok && vkBindBufferMemory(s_dev, p.buffer, p.memory, 0) == VK_SUCCESS &&
                    vkMapMemory(s_dev, p.memory, 0, VK_WHOLE_SIZE, 0, &mapped) == VK_SUCCESS;
                if (!ok)
                {
                    fprintf(stderr, "[shadowmap] no host-visible buffer for the copy (%u bytes): off\n", rangeBytes);
                    if (p.buffer)
                        vkDestroyBuffer(s_dev, p.buffer, nullptr);
                    if (p.memory)
                        vkFreeMemory(s_dev, p.memory, nullptr);
                    p.buffer = VK_NULL_HANDLE;
                    p.memory = VK_NULL_HANDLE;
                    p.failed = true;
                    return;
                }
                p.host = static_cast<const uint8_t*>(mapped);
                p.size = rangeBytes;
                p.coherent = (s_memProps.memoryTypes[mai.memoryTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
            }
            // The resolve's writes -> the copy -> the host's reads (after
            // the submission's fence).
            VkMemoryBarrier mb{ VK_STRUCTURE_TYPE_MEMORY_BARRIER };
            mb.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
            mb.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
            vkCmdPipelineBarrier(s_cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &mb, 0, nullptr, 0, nullptr);
            VkBufferCopy copy{ rangeBase, 0, rangeBytes };
            vkCmdCopyBuffer(s_cmd, s_shared, p.buffer, 1, &copy);
            mb.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
            mb.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
            vkCmdPipelineBarrier(s_cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, nullptr, 0, nullptr);
            s_shadowMapState.store(kProbeRecorded, std::memory_order_relaxed);
            p.submission = s_submissionCounter;
            p.frame = s_frame;
            p.seconds = s_shadowMapClock.time_since_epoch().count()
                ? std::chrono::duration<double>(now - s_shadowMapClock).count() : 0.0;
            p.rangeBase = rangeBase;
            p.rangeBytes = rangeBytes;
            p.destBase = rc.destBase;
            p.destPitch = rc.destPitch;
            p.endian = rc.endian;
            p.format = rc.format;
            p.srcBase = srcBase;
            p.rect = r;
            p.next = now + std::chrono::seconds(5);
        }

        // At a swap: the recorded copy, once its submission is done, goes to
        // a counting thread (ShadowMapCount).
        void ShadowMapProbeOnSwap()
        {
            if (!s_shadowMapClock.time_since_epoch().count())
                s_shadowMapClock = std::chrono::steady_clock::now();
            if (!s_shadowMapProbe || s_shadowMapState.load(std::memory_order_relaxed) != kProbeRecorded)
                return;
            ShadowMapProbe& p = *s_shadowMapProbe;
            {
                std::lock_guard lock(s_submitMutex);
                bool done = p.submission < s_submissionCounter &&
                    (s_inFlight.empty() || s_submissions[s_inFlight.front()].number > p.submission);
                if (!done)
                    return;
            }
            if (!p.coherent)
            {
                VkMappedMemoryRange range{ VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, nullptr, p.memory, 0, VK_WHOLE_SIZE };
                vkInvalidateMappedMemoryRanges(s_dev, 1, &range);
            }
            s_shadowMapState.store(kProbeCounting, std::memory_order_relaxed);
            std::thread(ShadowMapCount, ShadowMapSample{ p.host, p.rangeBase, p.rangeBytes, p.destBase, p.destPitch, p.endian, p.format,
                p.srcBase, p.rect, p.frame, p.seconds }).detach();
        }

        void Resolve()
        {
            s_stats.resolves++;
            auto copyControl = Reg<reg::RB_COPY_CONTROL>();
            auto surfaceInfo = Reg<reg::RB_SURFACE_INFO>();
            uint32_t pitch = surfaceInfo.surface_pitch;
            if (pitch == 0)
                return;

            // The rectangle comes from the three vertices in vertex fetch 0
            // (Direct3D 9 writes them with the CPU).
            uint32_t fetch0 = s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0];
            uint32_t fetch1 = s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_1];
            uint32_t vertexAddress = fetch0 & 0x1FFFFFFC;
            uint32_t vertexEndian = fetch1 & 3;
            if ((fetch0 & 3) != 3 || ((fetch1 >> 2) & 0xFFFFFF) != 6)
            {
                LogOnce(0x100, "resolve with unexpected vertex fetch constant %08X %08X", fetch0, fetch1);
                return;
            }
            float halfPixel = Reg<reg::PA_SU_VTX_CNTL>().pix_center == xenos::PixelCenter::kD3DZero ? 0.5f : 0.0f;
            int32_t v[6];
            for (int i = 0; i < 6; i++)
            {
                uint32_t bits = GpuSwap(LoadPhysical(vertexAddress + i * 4), vertexEndian);
                float f;
                memcpy(&f, &bits, 4);
                v[i] = ToFixed16p8(f + halfPixel);
            }
            Rect r;
            r.x0 = (std::min({ v[0], v[2], v[4] }) + 127) >> 8;
            r.y0 = (std::min({ v[1], v[3], v[5] }) + 127) >> 8;
            r.x1 = (std::max({ v[0], v[2], v[4] }) + 127) >> 8;
            r.y1 = (std::max({ v[1], v[3], v[5] }) + 127) >> 8;
            if (Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable)
            {
                auto wo = Reg<reg::PA_SC_WINDOW_OFFSET>();
                r.x0 += wo.window_x_offset;
                r.x1 += wo.window_x_offset;
                r.y0 += wo.window_y_offset;
                r.y1 += wo.window_y_offset;
            }
            Rect sc = GuestScissor();
            r.x0 = std::clamp(r.x0, sc.x0, sc.x1);
            r.x1 = std::clamp(r.x1, sc.x0, sc.x1);
            r.y0 = std::clamp(r.y0, sc.y0, sc.y1);
            r.y1 = std::clamp(r.y1, sc.y0, sc.y1);
            r.x0 &= ~7;
            r.y0 &= ~7;
            r.x1 = (r.x1 + 7) & ~7;
            r.y1 = (r.y1 + 7) & ~7;
            r.x1 = std::min(r.x1, int32_t(pitch & ~7u));
            r.x0 = std::min(r.x0, r.x1);
            if (r.x1 <= r.x0 || r.y1 <= r.y0)
                return;

            bool depthSource = copyControl.copy_src_select >= 4;
            // NFSMW_LOG_RESOLVES=1: describe the first resolves in detail.
            // NFSMW_LOG_RESOLVES_PITCH=<p>: only resolves from surfaces of that pitch.
            static const bool logResolves = std::getenv("NFSMW_LOG_RESOLVES") != nullptr;
            static const int logPitch = [] { const char* v = std::getenv("NFSMW_LOG_RESOLVES_PITCH"); return v ? std::atoi(v) : -1; }();
            static uint32_t logged = 0;
            if ((logResolves || (logPitch >= 0 && pitch == uint32_t(logPitch))) && logged < 200)
            {
                logged++;
                auto wo = Reg<reg::PA_SC_WINDOW_OFFSET>();
                fprintf(stderr, "[resolve] rect (%d,%d)-(%d,%d) wo (%d,%d) woEn %u src %u pitch %u msaa %u | dest %08X pitch %u h %u info %08X "
                    "(fmt %u endian %u swap %u bias %d) | ctl %08X color %08X depth %08X clear c%08X d%08X\n",
                    r.x0, r.y0, r.x1, r.y1, wo.window_x_offset, wo.window_y_offset,
                    uint32_t(Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable), copyControl.copy_src_select, pitch,
                    uint32_t(surfaceInfo.msaa_samples), s_regs[XE_GPU_REG_RB_COPY_DEST_BASE],
                    Reg<reg::RB_COPY_DEST_PITCH>().copy_dest_pitch, Reg<reg::RB_COPY_DEST_PITCH>().copy_dest_height,
                    s_regs[XE_GPU_REG_RB_COPY_DEST_INFO], uint32_t(Reg<reg::RB_COPY_DEST_INFO>().copy_dest_format),
                    uint32_t(Reg<reg::RB_COPY_DEST_INFO>().copy_dest_endian), uint32_t(Reg<reg::RB_COPY_DEST_INFO>().copy_dest_swap),
                    int(Reg<reg::RB_COPY_DEST_INFO>().copy_dest_exp_bias), copyControl.value, s_regs[XE_GPU_REG_RB_COLOR_INFO],
                    s_regs[XE_GPU_REG_RB_DEPTH_INFO], s_regs[XE_GPU_REG_RB_COLOR_CLEAR], s_regs[XE_GPU_REG_RB_DEPTH_CLEAR]);
                if (logPitch >= 0)
                    for (int i = 0; i < 6; i++)
                    {
                        uint32_t bits = GpuSwap(LoadPhysical(vertexAddress + i * 4), vertexEndian);
                        float f;
                        memcpy(&f, &bits, 4);
                        fprintf(stderr, "[resolve]   v%d.%c = %.1f\n", i / 2, "xy"[i & 1], f);
                    }
            }
            RenderTarget* source;
            if (depthSource)
            {
                auto di = Reg<reg::RB_DEPTH_INFO>();
                source = GetTarget(di.depth_base, uint32_t(di.depth_format), true, pitch);
            }
            else
            {
                static const uint32_t infoRegs[4] = { XE_GPU_REG_RB_COLOR_INFO, XE_GPU_REG_RB_COLOR1_INFO,
                    XE_GPU_REG_RB_COLOR2_INFO, XE_GPU_REG_RB_COLOR3_INFO };
                auto ci = RegAt<reg::RB_COLOR_INFO>(infoRegs[copyControl.copy_src_select & 3]);
                source = GetTarget(ci.color_base, uint32_t(ci.color_format), false, pitch);
            }

            // Copy.
            auto destInfo = Reg<reg::RB_COPY_DEST_INFO>();
            auto destPitch = Reg<reg::RB_COPY_DEST_PITCH>();
            uint32_t destBase = s_regs[XE_GPU_REG_RB_COPY_DEST_BASE];
            uint32_t format = depthSource
                ? (xenos::DepthRenderTargetFormat(Reg<reg::RB_DEPTH_INFO>().depth_format) == xenos::DepthRenderTargetFormat::kD24FS8 ? 23u : 22u)
                : uint32_t(destInfo.copy_dest_format);
            bool copy = copyControl.copy_command == xenos::CopyCommand::kRaw ||
                copyControl.copy_command == xenos::CopyCommand::kConvert;
            if (copy && destBase != 0)
            {
                auto [extentStart, extentEnd] = ResolveExtent(r, (destPitch.copy_dest_pitch + 31) & ~31u, ResolveBppLog2(format));
                uint32_t rangeBase = (destBase & 0x1FFFFFFF) + extentStart, rangeBytes = extentEnd - extentStart;
                // Shadow mode: pages the resolve covers only in part keep
                // the CPU's latest data around it.
                EnsureShadow(rangeBase, rangeBytes);
                if (destInfo.copy_dest_array)
                    LogOnce(0x200, "resolve to a 3D/array destination is not implemented (copying slice 0)");
                if (!ResolveFormatSupported(format))
                {
                    LogOnce(0x300 + format, "resolve to texture format %u not implemented", format);
                }
                else
                {
                    EndPass();
                    Begin();
                    // NFSMW_UNTILED_AREA: a resolve reading, inside what the
                    // untiled draws cut to their viewports could reach (their
                    // attachments' common size), past what they drew in the
                    // last frame untiled draws drew its source (a resolve
                    // after the swap reads that frame's), reads pixels the
                    // host used to overdraw (and the console never drew).
                    // None seen in menus, loading, pre-race or races; should
                    // one appear, that target's draws keep the whole target
                    // from then on, so it differs from before for this one
                    // frame. Blind: a cut inside another (their union is the
                    // larger). Needless (costing only the saving): a band only
                    // a clear covered, a target drawn since by non-untiled draws.
                    if (s_untiledArea && source->cut && !source->wholeArea)
                    {
                        auto [shiftX, shiftY] = TileShift();
                        int32_t x0 = source->scale.x.EdgeAt(r.x0 + shiftX);
                        int32_t y0 = source->scale.y.EdgeAt(r.y0 + shiftY);
                        int32_t x1 = std::min({ source->scale.x.EdgeAt(r.x1 + shiftX), int32_t(source->width), int32_t(source->cutReach.width) });
                        int32_t y1 = std::min({ source->scale.y.EdgeAt(r.y1 + shiftY), int32_t(source->height), int32_t(source->cutReach.height) });
                        VkRect2D read{ { x0, y0 }, { uint32_t(std::max(x1 - x0, 0)), uint32_t(std::max(y1 - y0, 0)) } };
                        if (read.extent.width && read.extent.height && !Contains(source->cutDrawn, read))
                        {
                            source->wholeArea = true;
                            s_ua.resolvesPast++;
                            LogOnce((0x31ull << 56) | (uint64_t(source->base) << 24) | (uint64_t(source->pitch) << 1) | uint64_t(source->depth),
                                "untiled area: frame %llu resolve of target b%u p%u%s reads %ux%u@%d,%d, past what its cut draws drew (%ux%u@%d,%d "
                                "in frame %llu; dest %08X): its draws keep the whole target from now on",
                                (unsigned long long)s_frame, source->base, source->pitch, source->depth ? " (depth)" : "", read.extent.width,
                                read.extent.height, read.offset.x, read.offset.y, source->cutDrawn.extent.width, source->cutDrawn.extent.height,
                                source->cutDrawn.offset.x, source->cutDrawn.offset.y, (unsigned long long)source->areaFrame, destBase);
                        }
                    }
                    if (s_clearFlushRect)
                    {
                        // What the shader reads (px() and the scaled path:
                        // the rectangle's host pixels from rc.srcOrigin, each
                        // axis clamped to the target; at 1.5x its edges are
                        // even, so whole), so the clears it doesn't see stay
                        // pending.
                        auto [shiftX, shiftY] = TileShift();
                        auto texel = [](int32_t v, uint32_t size) { return std::clamp(v, 0, int32_t(size) - 1); };
                        int32_t x0 = texel(source->scale.x.EdgeAt(r.x0 + shiftX), source->width);
                        int32_t y0 = texel(source->scale.y.EdgeAt(r.y0 + shiftY), source->height);
                        int32_t x1 = texel(source->scale.x.EdgeAt(r.x1 + shiftX) - 1, source->width) + 1;
                        int32_t y1 = texel(source->scale.y.EdgeAt(r.y1 + shiftY) - 1, source->height) + 1;
                        VkRect2D read{ { x0, y0 }, { uint32_t(x1 - x0), uint32_t(y1 - y0) } };
                        FlushPendingClears(source, &read);
                    }
                    else
                    {
                        FlushPendingClears(source);
                    }
                    EndPass();
                    Begin();
                    ResolveConstants rc{};
                    rc.originX = r.x0;
                    rc.originY = r.y0;
                    rc.width = uint32_t(r.x1 - r.x0);
                    rc.height = uint32_t(r.y1 - r.y0);
                    rc.destBase = destBase & 0x1FFFFFFF;
                    rc.destPitch = (destPitch.copy_dest_pitch + 31) & ~31u;
                    rc.format = format;
                    rc.endian = uint32_t(destInfo.copy_dest_endian);
                    rc.flags = (destInfo.copy_dest_swap ? 1u : 0u) | (depthSource ? 2u : 0u);
                    rc.expScale = std::ldexp(1.0f, destInfo.copy_dest_exp_bias);
                    rc.srcWidth = source->width;
                    rc.srcHeight = source->height;
                    auto [shiftX, shiftY] = TileShift();
                    rc.srcOriginX = source->scale.x.EdgeAt(r.x0 + shiftX);
                    rc.srcOriginY = source->scale.y.EdgeAt(r.y0 + shiftY);
                    rc.scaleX = source->scale.x.num;
                    rc.scaleY = source->scale.y.num;
                    // At 1.5x the shader counts host pixels from srcOrigin
                    // and the destination's blocks from its origin: both
                    // need even coordinates (resolves are 8-pixel aligned
                    // and the tile offsets even, so they are).
                    const bool wholeBlocks = (r.x0 + shiftX) >= 0 && (r.y0 + shiftY) >= 0 &&
                        source->scale.x.Exact(uint32_t(r.x0 + shiftX)) && source->scale.y.Exact(uint32_t(r.y0 + shiftY)) &&
                        source->scale.x.Exact(uint32_t(r.x0)) && source->scale.y.Exact(uint32_t(r.y0));
                    if (!wholeBlocks)
                        LogOnce(0x2500, "resolve at (%d,%d) (tile shift %d,%d) from a %s target isn't whole blocks: kept at 1x only, "
                            "its 1x average up to half a host pixel off", r.x0, r.y0, shiftX, shiftY, gpu::ScaleName(source->scale).c_str());
                    uint32_t bl = ResolveBppLog2(format);
                    uint32_t firstPage = rangeBase >> 12;
                    uint32_t lastPage = std::min<uint32_t>((rangeBase + rangeBytes - 1) >> 12, kGuestPages - 1);
                    // A 32-bit resolve from a scaled target keeps its samples.
                    bool scaledWrite = !source->scale.IsOne() && wholeBlocks && bl == 2 && !destInfo.copy_dest_array &&
                        EnsureScaledMemory() && source->scale == s_scaled.scale;
                    if (scaledWrite)
                    {
                        std::vector<std::pair<uint32_t, uint32_t>> fresh;
                        for (uint32_t page = firstPage; page <= lastPage; page++)
                        {
                            // A slot whose copy went stale (a CPU write, a 1x
                            // resolve) starts over from the 1x data too: the
                            // resolve may not cover all of it.
                            bool stale = s_scaled.pageSlot[page] != 0 && !ScaledPageValid(page);
                            bool isFresh;
                            uint32_t slot = ScaledSlot(page, isFresh);
                            if (isFresh || stale)
                                fresh.push_back({ page, slot });
                        }
                        FlushScaledTable();
                        FillFreshSlots(fresh);
                        rc.scaledWrite = 1;
                        rc.slotWords = s_scaled.slotWords;
                        s_scaled.resolvedBytes += s_scaled.scale.HostPixels(rc.width, rc.height) * 4;
                    }

                    // A cached direct 1x texture of exactly this surface (its
                    // address, tiling, pitch and 32-bit texels), current until
                    // now, whose memory this resolve rewrites: the resolve
                    // stores its texels too, and the texture stays current. The
                    // Steam Frame uploaded each 1600x1600 shadow map again after
                    // its resolve (~0.4 ms, twice a frame).
                    // Else (NFSMW_FUSE_PARTIAL, NFSMW_DIRECT_CUBE) one holding
                    // the destination whole 32x32 tiles in, and all this resolve
                    // writes (a strip of the scene, a cube's face): the same,
                    // at those tiles' texels (see the shader). Its other texels
                    // must still be what it loaded: checked again after
                    // MarkWritten, below.
                    Texture* fused = nullptr;
                    Texture* direct = nullptr;
                    bool partial = false;
                    uint32_t layer = 0, texX = 0, texY = 0;  // partial: the face, and the destination's texel 0,0 in it
                    // Why not (NFSMW_LOG_UPLOADS).
                    const char* unfused = !s_resolveToTexture ? "off" : scaledWrite ? "scaled resolve" : bl != 2 ? "not 32-bit"
                        : "no direct texture there";
                    // Whether `t` can take this resolve's words as its texels at all.
                    auto unfit = [&](const Texture& t) -> const char* {
                        return !(t.uploaded && t.storeView && !t.scaled) ? "texture not loaded"
                            : !t.tiled || t.bpbLog2 != 2 || (t.convert != 0 && t.convert != 4) ? "texture layout"
                            : t.pitch != rc.destPitch ? "pitch differs"
                            : t.offX != 0 || t.offY != 0 ? "texture offset"
                            : nullptr;
                    };
                    if (!scaledWrite && bl == 2 && s_resolveToTexture)
                        if (auto it = s_directByBase.find(rc.destBase); it != s_directByBase.end())
                        {
                            Texture& t = *(direct = it->second);
                            unfused = unfit(t);
                            if (!unfused)
                                unfused = !(t.guestStart >= rangeBase && uint64_t(t.guestStart) + t.guestSize <= uint64_t(rangeBase) + rangeBytes)
                                    ? "texture beyond the resolve's range"
                                    : writewatch::WrittenSince(t.guestStart, t.guestSize, t.loadedSequence) ? "texture written since its load"
                                    : nullptr;
                            if (!unfused)
                                fused = &t;
                        }
                    if (!fused && !scaledWrite && bl == 2 && s_resolveToTexture && (s_fusePartial || s_directCube))
                    {
                        // Each direct texture holding the destination, the
                        // nearest start first, until one takes it: textures
                        // are kept (dead ones too) until the cache is dropped,
                        // so an older one may start inside a larger one. Why
                        // not: the nearest's reason, or the nearest's that
                        // could take its words at all (unfit).
                        Texture* why = nullptr;
                        bool whyFits = false;
                        for (auto it = s_directByStart.upper_bound(rc.destBase); !fused && it != s_directByStart.begin();)
                        {
                            if (rc.destBase - (--it)->first >= s_maxDirectSize)
                                break;  // none starting earlier reaches it
                            Texture& t = *it->second;
                            if (rc.destBase - t.guestStart >= t.guestSize)
                                continue;
                            // Its face holding the destination (a cube's: its
                            // layers are the guest's faces in order, as the
                            // staging path copies them), and the tile there:
                            // 32-bit texels make a tile 4 KB, so a destination
                            // whole tiles in is the texture's texels from
                            // (tile % tiles a row, tile / tiles a row) x 32,
                            // as the untile reads them.
                            uint32_t face = t.cube ? (rc.destBase - t.guestStart) / t.sliceStride : 0;
                            uint32_t sliceStart = t.guestStart + face * t.sliceStride;
                            uint64_t sliceEnd = uint64_t(t.guestStart) + t.guestSize;
                            if (t.cube)
                                sliceEnd = std::min<uint64_t>(sliceEnd, uint64_t(sliceStart) + t.sliceStride);
                            uint32_t rel = rc.destBase - sliceStart, tilesPerRow = std::max(t.pitch >> 5, 1u);
                            uint32_t x = (rel >> 12) % tilesPerRow * 32, y = (rel >> 12) / tilesPerRow * 32;
                            const char* reason = unfit(t);
                            bool fits = !reason;
                            if (fits)
                                reason = face >= t.layers ? "texture layout"
                                    : rel % 4096 != 0 ? "not whole tiles into the texture"
                                    : (rel != 0 || !t.cube) && !s_fusePartial ? "partial fusion off"
                                    : x + uint32_t(r.x1) > rc.destPitch ? "beyond the texture's columns"
                                    : !(rangeBase >= sliceStart && uint64_t(rangeBase) + rangeBytes <= sliceEnd)
                                    ? (t.cube ? "beyond the face" : "beyond the texture")
                                    : writewatch::WrittenSince(t.guestStart, t.guestSize, t.loadedSequence) ? "texture written since its load"
                                    : nullptr;
                            if (!reason)
                            {
                                fused = direct = &t;
                                unfused = nullptr;
                                partial = true;
                                layer = face;
                                texX = x;
                                texY = y;
                            }
                            else if (!why || (fits && !whyFits))
                            {
                                why = direct = &t;
                                whyFits = fits;
                                unfused = reason;
                            }
                        }
                    }
                    if (s_uploadLog)
                    {
                        UploadLog& log = *s_uploadLog;
                        log.resolves[rc.destBase] = { s_frame, unfused };
                        if (log.now)
                        {
                            log.resolvesNow++;
                            log.fusedNow += fused != nullptr;
                            std::string part = !partial ? std::string()
                                : fused->cube ? std::format(" (face {}, texels from {},{})", layer, texX, texY)
                                : std::format(" (texels from {},{})", texX, texY);
                            fprintf(stderr, "[upload] frame %llu resolve dest %08X range %08X+%u rect %ux%u fmt %u pitch %u %s %s%s",
                                (unsigned long long)s_frame, rc.destBase, rangeBase, rangeBytes, rc.width, rc.height, format, rc.destPitch,
                                depthSource ? "depth" : "color", fused ? "fused" : "unfused: ", fused ? part.c_str() : unfused);
                            std::set<std::string> kinds;
                            if (direct)
                                fprintf(stderr, " | texture %08X+%u %ux%u pitch %u%s%s", direct->guestStart, direct->guestSize, direct->width,
                                    direct->height, direct->pitch, direct->uploaded ? " " : "",
                                    direct->uploaded ? DescribeWrites(*direct, kinds).c_str() : "");
                            fprintf(stderr, "\n");
                        }
                    }
                    if (fused)
                    {
                        rc.texWrite = 1;
                        rc.texEndian = fused->endian;
                        rc.texConvert = fused->convert;
                        rc.texWidth = fused->width;
                        rc.texHeight = fused->height;
                        rc.texX = texX;
                        rc.texY = texY;
                        LogOnce(0x2400, "resolves store into cached textures too (the first: %ux%u at %08X)", fused->width,
                            fused->height, rc.destBase);
                        if (partial)
                            LogOnce(0x2800, "resolves store into part of cached textures too (the first: %ux%u%s at %08X, texels from %u,%u)",
                                fused->width, fused->height, fused->cube ? " cube" : "", rc.destBase, texX, texY);
                    }

                    VkDescriptorImageInfo ii{ VK_NULL_HANDLE, source->sampleView, VK_IMAGE_LAYOUT_GENERAL };
                    VkDescriptorImageInfo ti{ VK_NULL_HANDLE, fused ? fused->storeViews[layer] : s_dummyStorage.view, VK_IMAGE_LAYOUT_GENERAL };
                    VkDescriptorBufferInfo pool = DummyStorage(s_scaled.pool);
                    VkDescriptorBufferInfo table = DummyStorage(s_scaled.table);
                    VkWriteDescriptorSet w[8]{};
                    w[0] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 0, 0, 1,
                        VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &s_sharedInfo[0], nullptr };
                    w[1] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 1, 0, 1,
                        VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, &ii, nullptr, nullptr };
                    w[2] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 2, 0, 1,
                        VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &pool, nullptr };
                    w[3] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 3, 0, 1,
                        VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &table, nullptr };
                    w[4] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 4, 0, 1,
                        VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &ti, nullptr, nullptr };
                    s_stateEpoch++;
                    bool coalesced = scaledWrite && s_scaled.coalesce;  // 32x2 pixels a group
                    vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, ResolvePipeline(coalesced, source->scale));
                    s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_resolveLayout, 0, AddSharedParts(w, 5), w);
                    vkCmdPushConstants(s_cmd, s_resolveLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, s_resolvePush, &rc);
                    uint32_t perWord = bl >= 2 ? 1 : (4 >> bl);
                    if (!(s_exp & 2))
                        vkCmdDispatch(s_cmd, coalesced ? (rc.width + 31) / 32 : (rc.width / perWord + 7) / 8,
                            coalesced ? (rc.height + 1) / 2 : (rc.height + 7) / 8, 1);
                    if (scaledWrite)
                        FullBarrier();  // (above 1x: as before)
                    else
                        ResolveBarrier(rangeBase, rangeBytes, fused ? fused->guestStart : 0);
                    if (s_logShadowMap && depthSource && pitch == 1600 && bl == 2)
                        ShadowMapProbeRecord(rangeBase, rangeBytes, rc, r, source->base);
                    SubmitIfStarving();
                    // Write sequences are per host page: guest pages of the
                    // edge host pages outside this resolve whose scaled copy
                    // was current stay current (this write isn't theirs).
                    std::vector<uint32_t> siblings;
                    if (s_scaled.enabled && s_scaled.hostPageGuestPages > 1)
                    {
                        uint32_t g = s_scaled.hostPageGuestPages;
                        for (uint32_t edge : { firstPage, lastPage })
                            for (uint32_t q = edge / g * g; q < edge / g * g + g && q < kGuestPages; q++)
                                if ((q < firstPage || q > lastPage) && ScaledPageValid(q))
                                    siblings.push_back(q);
                    }
                    uint64_t written = writewatch::MarkWritten(rangeBase, rangeBytes, writewatch::Writer::kResolve, rc.destBase);
                    // The fused texture holds this write: its pages' sequence
                    // now, and any write after it is newer (0: per-page
                    // sequences, NFSMW_WATCH_RANGES=0, so it is loaded again).
                    // A partial one's pages outside this range keep theirs,
                    // which the check above found no newer than its load: a
                    // CPU store there since (a fault racing this) is older
                    // than `written`, and would never be loaded. Then it stays
                    // as it was (its range is newer now: it loads again). The
                    // exact rule's texture is all in the range. Quiesce first:
                    // a fault that took its sequence before `written` may not
                    // have stored it on its pages yet.
                    uint32_t rangeEnd = rangeBase + rangeBytes;
                    if (partial)
                        writewatch::Quiesce();
                    bool keep = !partial || (!writewatch::WrittenSince(fused->guestStart, rangeBase - fused->guestStart, fused->loadedSequence) &&
                        !writewatch::WrittenSince(rangeEnd, fused->guestStart + fused->guestSize - rangeEnd, fused->loadedSequence));
                    if (fused && written && keep)
                    {
                        fused->loadedSequence = written;
                        fused->verifyHashes.clear();  // (NFSMW_TEXTURE_VERIFY: its memory is the GPU's to change now)
                    }
                    if (fused && !keep && s_uploadLog)
                    {
                        s_uploadLog->resolves[rc.destBase].unfused = "texture written outside it meanwhile";
                        if (s_uploadLog->now)
                        {
                            s_uploadLog->fusedNow--;
                            fprintf(stderr, "[upload] frame %llu resolve dest %08X: stored into %08X, but the texture was written outside "
                                "it meanwhile: loaded again\n", (unsigned long long)s_frame, rc.destBase, fused->guestStart);
                        }
                    }
                    // CPU writes must fault to move the sequence on (and so
                    // retire the scaled copy), whatever else protects the pages.
                    if (scaledWrite)
                        writewatch::Watch(rangeBase, rangeBytes);
                    if (s_scaled.enabled)
                    {
                        for (uint32_t page = firstPage; page <= lastPage; page++)
                            s_scaled.pageSeq[page] = scaledWrite ? writewatch::WriteSequence(page << 12, 4096) : 0;
                        for (uint32_t q : siblings)
                            s_scaled.pageSeq[q] = writewatch::WriteSequence(q << 12, 4096);
                    }
                    GuardGpu(rangeBase, rangeBytes, true, "resolve");
                    writewatch::MarkGpuOwned(rangeBase, rangeBytes);
                }
            }

            // Clears (in host target coordinates, see TileShift).
            {
                auto [shiftX, shiftY] = TileShift();
                r.x0 += shiftX;
                r.x1 += shiftX;
                r.y0 += shiftY;
                r.y1 += shiftY;
            }
            if (copyControl.color_clear_enable && !depthSource)
                ClearRect(source, r);
            if (copyControl.depth_clear_enable)
            {
                auto di = Reg<reg::RB_DEPTH_INFO>();
                ClearRect(GetTarget(di.depth_base, uint32_t(di.depth_format), true, pitch), r);
            }
            if (copyControl.color_clear_enable && depthSource)
                LogOnce(0x400, "color clear requested with a depth resolve source");
        }

        void CreateComputePipeline(const char* glsl, const VkDescriptorSetLayoutBinding* bindings, uint32_t bindingCount,
            uint32_t pushSize, VkDescriptorSetLayout& setLayout, VkPipelineLayout& layout, VkPipeline& pipeline,
            const char* preamble = "")
        {
            std::string log;
            auto spirv = CompileGlsl(glsl, GLSLANG_STAGE_COMPUTE, log, preamble);
            if (spirv.empty())
            {
                fprintf(stderr, "[renderer] compute shader failed:\n%s\n", log.c_str());
                abort();
            }
            VkDescriptorSetLayoutCreateInfo dslci{ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO };
            dslci.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
            dslci.bindingCount = bindingCount;
            dslci.pBindings = bindings;
            Check(vkCreateDescriptorSetLayout(s_dev, &dslci, nullptr, &setLayout), "compute set layout");
            VkPushConstantRange range{ VK_SHADER_STAGE_COMPUTE_BIT, 0, pushSize };
            VkPipelineLayoutCreateInfo plci{ VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO };
            plci.setLayoutCount = 1;
            plci.pSetLayouts = &setLayout;
            plci.pushConstantRangeCount = 1;
            plci.pPushConstantRanges = &range;
            Check(vkCreatePipelineLayout(s_dev, &plci, nullptr, &layout), "compute pipeline layout");
            VkShaderModule module = CreateModule(spirv);
            VkComputePipelineCreateInfo cpci{ VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO };
            cpci.stage = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_COMPUTE_BIT,
                module, "main", nullptr };
            cpci.layout = layout;
            Check(vkCreateComputePipelines(s_dev, s_pipelineCache, 1, &cpci, nullptr, &pipeline), "compute pipeline");
        }

        // Scaled memory's GPU side (see ScaledMemory): made on the first
        // scaled resolve.
        const char* kScaledFillGlsl = R"(#version 460
layout(local_size_x = 64) in;
layout(std430, set = 0, binding = 0) readonly buffer SharedMemory { uint g_mem[]; };)" GLSL_SHARED_MEMORY_READ R"(
layout(std430, set = 0, binding = 1) buffer ScaledPool { uint s_pool[]; };
layout(std430, set = 0, binding = 2) readonly buffer List { uint l_pairs[]; };
layout(push_constant) uniform P { uint count, slotWords, listWord, pad; uvec2 scale, den; } p;
)" GLSL_SCALE_CODE R"(
// A new slot starts as its page's 1x data, each word in the host samples its
// texel owns (all s at a whole scale; at 1.5x one, two or four of its
// block's): a resolve covering the page only in part leaves no garbage
// around it.
void main()
{
    uint n = gl_GlobalInvocationID.x;
    uint pair = n >> 10, k = n & 1023u;
    if (pair >= p.count) return;
    uint page = l_pairs[p.listWord + pair * 2u], slot = l_pairs[p.listWord + pair * 2u + 1u];
    uint v = memLoad(page * 1024u + k);
    uint o = slot * p.slotWords + sbBlockIndex(k, p.den.x, p.den.y) * (p.scale.x * p.scale.y);
    uint subX = sbSubX(k, p.den.x), subY = sbSubY(k, p.den.y);
    uint i1 = sbHostEdge(subX + 1u, p.scale.x, p.den.x), j1 = sbHostEdge(subY + 1u, p.scale.y, p.den.y);
    for (uint j = sbHostEdge(subY, p.scale.y, p.den.y); j < j1; j++)
        for (uint i = sbHostEdge(subX, p.scale.x, p.den.x); i < i1; i++)
            s_pool[o + j * p.scale.x + i] = v;
}
)";
        VkPipeline s_scaledFillPipeline = VK_NULL_HANDLE;
        VkPipelineLayout s_scaledFillLayout = VK_NULL_HANDLE;
        VkDescriptorSetLayout s_scaledFillSetLayout = VK_NULL_HANDLE;

        // A scaled texture's texels from the scaled memory (the 32-bit tiled
        // layout of the guest texture, each texel's samples): stored into the
        // texture through its R32_UINT view, or (TO_BUFFER) linear into the
        // scratch buffer for a copy.
        const char* kScaledUntileGlsl = R"(#version 460
layout(local_size_x = 8, local_size_y = 8) in;
layout(std430, set = 0, binding = 0) readonly buffer ScaledPool { uint s_pool[]; };
layout(std430, set = 0, binding = 1) readonly buffer ScaledTable { uint s_table[]; };
#ifdef TO_BUFFER
layout(std430, set = 0, binding = 2) writeonly buffer Out { uint o_mem[]; };
#else
layout(set = 0, binding = 2, r32ui) writeonly uniform uimage2D o_image;
#endif
// width, height: the texture's host size (whole blocks); scale / den: host
// pixels per guest pixel.
layout(push_constant) uniform P { uint base, pitch, width, height, endian, slotWords; uvec2 scale, den; } p;
)" GLSL_SCALE_CODE R"(
uint tiled2D(uint x, uint y, uint pitch, uint bppLog2)
{
    uint outer = (((y >> 5) * (pitch >> 5)) + (x >> 5)) << 6;
    uint inner = (((y >> 1) & 7u) << 3) | (x & 7u);
    uint oi = (outer | inner) << bppLog2;
    uint bank = (y >> 4) & 1u;
    uint pipe = ((x >> 3) & 3u) ^ (((y >> 3) & 1u) << 1);
    return ((y & 1u) << 4) | (pipe << 6) | (bank << 11) | (oi & 0xFu) | (((oi >> 4) & 1u) << 5) |
        (((oi >> 5) & 7u) << 8) | ((oi >> 8) << 12);
}

uint swapWord(uint v, uint e)
{
    if (e == 1u) return ((v << 8) & 0xFF00FF00u) | ((v >> 8) & 0x00FF00FFu);
    if (e == 2u) return (v << 24) | ((v << 8) & 0x00FF0000u) | ((v >> 8) & 0x0000FF00u) | (v >> 24);
    if (e == 3u) return (v >> 16) | (v << 16);
    return v;
}

void main()
{
    uvec2 h = gl_GlobalInvocationID.xy;
    uint w = p.width;
    if (h.x >= w || h.y >= p.height) return;
    // Host texel h: sample (h mod scale) of the block (h / scale), whose
    // first texel is guest (block x den) (gpu/render_scale.h).
    uvec2 b = h / p.scale;
    uint a = (p.base + tiled2D(b.x * p.den.x, b.y * p.den.y, p.pitch, 2u)) & 0x1FFFFFFFu;
    uint slot = s_table[a >> 12];
    uint v = 0u;
    if (slot != 0u)
        v = s_pool[(slot - 1u) * p.slotWords + sbSampleWord((a & 4095u) >> 2, h.x - b.x * p.scale.x, h.y - b.y * p.scale.y,
            p.scale.x, p.scale.y, p.den.x, p.den.y)];
#ifdef TO_BUFFER
    o_mem[h.y * w + h.x] = swapWord(v, p.endian);
#else
    imageStore(o_image, ivec2(h), uvec4(swapWord(v, p.endian)));
#endif
}
)";
        VkPipeline s_scaledUntilePipeline = VK_NULL_HANDLE;  // TO_BUFFER
        VkPipelineLayout s_scaledUntileLayout = VK_NULL_HANDLE;
        VkDescriptorSetLayout s_scaledUntileSetLayout = VK_NULL_HANDLE;
        VkPipeline s_scaledStorePipeline = VK_NULL_HANDLE;  // into the image
        VkPipelineLayout s_scaledStoreLayout = VK_NULL_HANDLE;
        VkDescriptorSetLayout s_scaledStoreSetLayout = VK_NULL_HANDLE;

        bool CreateDeviceBuffer(VkDeviceSize size, VkBufferUsageFlags usage, VkBuffer& buffer, VkDeviceMemory& memory, const char* what,
            VkDeviceSize* allocated = nullptr)
        {
            VkBufferCreateInfo bci{ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
            bci.size = size;
            bci.usage = usage;
            if (vkCreateBuffer(s_dev, &bci, nullptr, &buffer) != VK_SUCCESS)
            {
                fprintf(stderr, "[renderer] %s: vkCreateBuffer failed\n", what);
                return false;
            }
            VkMemoryRequirements req;
            vkGetBufferMemoryRequirements(s_dev, buffer, &req);
            VkMemoryAllocateInfo mai{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
            mai.allocationSize = req.size;
            mai.memoryTypeIndex = FindMemoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
            if (vkAllocateMemory(s_dev, &mai, nullptr, &memory) != VK_SUCCESS)
            {
                fprintf(stderr, "[renderer] %s: vkAllocateMemory (%llu MB) failed\n", what, (unsigned long long)(size >> 20));
                vkDestroyBuffer(s_dev, buffer, nullptr);
                buffer = VK_NULL_HANDLE;
                return false;
            }
            vkBindBufferMemory(s_dev, buffer, memory, 0);
            if (allocated)
                *allocated = req.size;
            return true;
        }

        // Scaled-texture checks by range, redone only when something was
        // written since (the write epoch moved).
        struct ScaledCheck
        {
            uint64_t epoch = ~0ull;
            bool valid = false;
        };
        std::unordered_map<uint64_t, ScaledCheck> s_scaledChecks;

        // The pool, page table and scratch buffer (the caller has the GPU idle).
        void FreeScaledBuffers()
        {
            for (auto [buffer, memory] : { std::pair{ &s_scaled.pool, &s_scaled.poolMemory }, std::pair{ &s_scaled.table, &s_scaled.tableMemory },
                     std::pair{ &s_scaled.scratch, &s_scaled.scratchMemory } })
            {
                if (*buffer)
                    vkDestroyBuffer(s_dev, *buffer, nullptr);
                if (*memory)
                    vkFreeMemory(s_dev, *memory, nullptr);
                *buffer = VK_NULL_HANDLE;
                *memory = VK_NULL_HANDLE;
            }
            s_scaled.poolBytes = s_scaled.tableBytes = s_scaled.scratchBytes = 0;
        }

        // Back to "not made yet" (a new internal resolution; the GPU idle):
        // the next scaled resolve makes it again at the new scale. Every page
        // falls back to its 1x copy until a scaled resolve writes it again.
        // `keepBuffers` (the new scale is above 1x too): the pool and the
        // page table stay allocated, as their sizes don't depend on the
        // scale; only the slots are laid out again.
        void ResetScaledMemory(bool keepBuffers)
        {
            if (!keepBuffers)
                FreeScaledBuffers();
            s_scaled.tried = s_scaled.enabled = false;
            s_scaled.scale = {};
            s_scaled.slotWords = s_scaled.slots = 0;
            s_scaled.scratchSize = 0;
            std::vector<uint32_t>().swap(s_scaled.pageSlot);
            std::vector<uint64_t>().swap(s_scaled.pageSeq);
            std::vector<uint32_t>().swap(s_scaled.slotPage);
            std::vector<uint8_t>().swap(s_scaled.slotRef);
            s_scaled.dirty.clear();
            s_scaled.clock = 0;
            s_scaledChecks.clear();
            s_textureCache->generation++;  // lookups decided with the old scale
        }

        bool EnsureScaledMemory()
        {
            if (s_scaled.tried)
                return s_scaled.enabled;
            s_scaled.tried = true;
            if (const char* v = std::getenv("NFSMW_SCALED_RESOLVE"); v && v[0] == '0')
                return false;
            auto start = std::chrono::steady_clock::now();
            s_scaled.scale = s_renderScale;
            if (s_scaled.scale.IsOne())
                return false;
            VkPhysicalDeviceProperties props;
            vkGetPhysicalDeviceProperties(s_vk->physical, &props);
            uint64_t megabytes = 512;
            if (const char* v = std::getenv("NFSMW_SCALED_MB"))
                megabytes = std::clamp<uint64_t>(std::strtoull(v, nullptr, 10), 16, 4096);
            // The whole budget, whatever the scale: kept across live changes
            // of the internal resolution (ResetScaledMemory). Bound whole, so
            // within the device's storage buffer range (NFSMW_SCALED_MB=4096
            // is past a 4 GB - 1 limit).
            VkDeviceSize poolSize = std::min<VkDeviceSize>(VkDeviceSize(megabytes) << 20, props.limits.maxStorageBufferRange) & ~VkDeviceSize(0xFFFF);
            s_scaled.slotWords = s_scaled.scale.SlotWords();
            s_scaled.slots = uint32_t(std::min<uint64_t>(poolSize / (4ull * s_scaled.slotWords), kGuestPages));
            // Scaled textures up to 1280x1024 (at scale). The untile stores
            // into them through an R32_UINT view when the device can for every
            // scaled format (a mutable-format image with extended usage: the
            // format itself needn't be storable); otherwise they load through
            // a scratch buffer that size (the ring's slots are too small) and a
            // copy, one more full write and read of the texture.
            s_scaled.scratchSize = VkDeviceSize(s_scaled.scale.HostPixels(1280, 1024)) * 4;
            {
                // (The image's format list is Vulkan 1.2.)
                const char* v = std::getenv("NFSMW_SCALED_DIRECT");
                VkFormatProperties fp;
                vkGetPhysicalDeviceFormatProperties(s_vk->physical, VK_FORMAT_R32_UINT, &fp);
                s_scaled.direct = !(v && v[0] == '0') && props.apiVersion >= VK_API_VERSION_1_2 &&
                    (fp.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT);
                for (uint32_t f = 0; f < 64 && s_scaled.direct; f++)
                {
                    TextureFormatInfo fi;
                    VkImageFormatProperties ifp;
                    if (GetTextureFormat(f, fi) && ScaledFormat(fi))
                        s_scaled.direct = vkGetPhysicalDeviceImageFormatProperties(s_vk->physical, fi.vk, VK_IMAGE_TYPE_2D,
                            VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT,
                            VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT, &ifp) == VK_SUCCESS;
                }
            }
            s_scaled.coalesce = ScaledCoalesce(props);
            // A scratch buffer kept from a smaller scale is too small.
            if (s_scaled.scratch && s_scaled.scratchBytes < s_scaled.scratchSize)
            {
                vkDestroyBuffer(s_dev, s_scaled.scratch, nullptr);
                vkFreeMemory(s_dev, s_scaled.scratchMemory, nullptr);
                s_scaled.scratch = VK_NULL_HANDLE;
                s_scaled.scratchMemory = VK_NULL_HANDLE;
                s_scaled.scratchBytes = 0;
            }
            if ((!s_scaled.pool && !CreateDeviceBuffer(poolSize, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, s_scaled.pool, s_scaled.poolMemory,
                     "scaled pool", &s_scaled.poolBytes)) ||
                (!s_scaled.table && !CreateDeviceBuffer(VkDeviceSize(kGuestPages) * 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
                    s_scaled.table, s_scaled.tableMemory, "scaled page table", &s_scaled.tableBytes)) ||
                (!s_scaled.direct && !s_scaled.scratch && !CreateDeviceBuffer(s_scaled.scratchSize, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
                    s_scaled.scratch, s_scaled.scratchMemory, "scaled scratch", &s_scaled.scratchBytes)))
            {
                fprintf(stderr, "[renderer] scaled resolve memory unavailable: textures stay 1x\n");
                FreeScaledBuffers();
                return false;
            }
            s_scaled.pageSlot.assign(kGuestPages, 0);
            s_scaled.pageSeq.assign(kGuestPages, 0);
            s_scaled.slotPage.assign(s_scaled.slots, 0);
            s_scaled.slotRef.assign(s_scaled.slots, 0);
            long hostPage = sysconf(_SC_PAGESIZE);
            s_scaled.hostPageGuestPages = hostPage > 4096 ? uint32_t(hostPage / 4096) : 1;
            // The pipelines take the scale in their push constants: made
            // once, kept when the memory is made again at another scale.
            VkDescriptorSetLayoutBinding b[3] = {
                { 0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 2, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
            };
            if (!s_scaledFillPipeline)
            {
                auto bindings = WithSharedParts(b, 3);
                CreateComputePipeline(kScaledFillGlsl, bindings.data(), uint32_t(bindings.size()), 8 * sizeof(uint32_t),
                    s_scaledFillSetLayout, s_scaledFillLayout, s_scaledFillPipeline);
            }
            if (s_scaled.direct)
            {
                b[2].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
                if (!s_scaledStorePipeline)
                    CreateComputePipeline(kScaledUntileGlsl, b, 3, 10 * sizeof(uint32_t), s_scaledStoreSetLayout, s_scaledStoreLayout,
                        s_scaledStorePipeline);
            }
            else if (!s_scaledUntilePipeline)
                CreateComputePipeline(kScaledUntileGlsl, b, 3, 10 * sizeof(uint32_t), s_scaledUntileSetLayout, s_scaledUntileLayout,
                    s_scaledUntilePipeline, "#define TO_BUFFER\n");
            EndPass();
            Begin();
            vkCmdFillBuffer(s_cmd, s_scaled.table, 0, VK_WHOLE_SIZE, 0);
            FullBarrier();
            s_scaled.enabled = true;
            s_textureCache->generation++;  // lookups decided without the scaled path
            fprintf(stderr, "[renderer] scaled resolve memory: %u slots of %u KB (%llu MB), scale %s, %s writes, textures %s (%.1f ms)\n",
                s_scaled.slots, s_scaled.slotWords / 256, (unsigned long long)(poolSize >> 20), gpu::ScaleName(s_scaled.scale).c_str(),
                s_scaled.coalesce ? "coalesced" : "per-thread", s_scaled.direct ? "stored directly" : "through a scratch buffer",
                std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count());
            return true;
        }

        bool ScaledTextureValid(uint32_t base, uint32_t size)
        {
            uint64_t epoch = writewatch::WriteEpoch();
            ScaledCheck& check = s_scaledChecks[(uint64_t(base) << 32) | size];
            if (check.epoch != epoch)
            {
                check.epoch = epoch;
                check.valid = ScaledRangeValid(base, size);
                if (!check.valid)
                    s_scaled.fallbacks++;
            }
            return check.valid;
        }

        void LoadScaledTexture(Texture& t, uint32_t base, uint32_t pitchBlocks, uint32_t endian)
        {
            VkDeviceSize bytes = VkDeviceSize(t.width) * t.height * 4;
            if (bytes > s_scaled.scratchSize)
            {
                LogOnce(0x2300, "scaled texture %ux%u is larger than the scratch buffer", t.width, t.height);
                return;
            }
            EndPass();
            Begin();
            s_scaled.loads++;
            s_scaled.untiledBytes += bytes;
            VkDescriptorBufferInfo bi[3] = { { s_scaled.pool, 0, VK_WHOLE_SIZE }, { s_scaled.table, 0, VK_WHOLE_SIZE },
                { s_scaled.scratch, 0, VK_WHOLE_SIZE } };
            VkWriteDescriptorSet w[3];
            for (uint32_t k = 0; k < 3; k++)
                w[k] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, k, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
                    nullptr, &bi[k], nullptr };
            const gpu::RenderScale& sc = s_scaled.scale;
            struct { uint32_t base, pitch, width, height, endian, slotWords, scaleX, scaleY, denX, denY; } pc = { base, pitchBlocks,
                t.width, t.height, endian, s_scaled.slotWords, sc.x.num, sc.y.num, sc.x.den, sc.y.den };
            s_stateEpoch++;
            if (t.storeView)
            {
                // Straight into the image. Everything before is behind a full
                // barrier already (as every pass and dispatch is; with
                // NFSMW_PASS_BARRIER a pass's narrower one still holds compute
                // work), including the draws that sampled the last load and
                // the resolves that wrote the pool, so only the stores need
                // ordering after.
                if (!t.uploaded)
                    ToGeneral(t.image.image, VK_IMAGE_ASPECT_COLOR_BIT, 1);
                VkDescriptorImageInfo ii{ VK_NULL_HANDLE, t.storeView, VK_IMAGE_LAYOUT_GENERAL };
                w[2] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 2, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
                    &ii, nullptr, nullptr };
                vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_scaledStorePipeline);
                s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_scaledStoreLayout, 0, 3, w);
                vkCmdPushConstants(s_cmd, s_scaledStoreLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
                vkCmdDispatch(s_cmd, (t.width + 7) / 8, (t.height + 7) / 8, 1);
                ComputeBarrier();
                return;
            }
            vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_scaledUntilePipeline);
            s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_scaledUntileLayout, 0, 3, w);
            vkCmdPushConstants(s_cmd, s_scaledUntileLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
            vkCmdDispatch(s_cmd, (t.width + 7) / 8, (t.height + 7) / 8, 1);
            FullBarrier();
            if (!t.uploaded)
                ToGeneral(t.image.image, VK_IMAGE_ASPECT_COLOR_BIT, 1);
            VkBufferImageCopy region{};
            region.bufferRowLength = t.width;
            region.imageSubresource = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1 };
            region.imageExtent = { t.width, t.height, 1 };
            vkCmdCopyBufferToImage(s_cmd, s_scaled.scratch, t.image.image, VK_IMAGE_LAYOUT_GENERAL, 1, &region);
            FullBarrier();
        }

        void FillFreshSlots(const std::vector<std::pair<uint32_t, uint32_t>>& fresh)
        {
            for (size_t first = 0; first < fresh.size(); first += 4096)
            {
                uint32_t count = uint32_t(std::min<size_t>(4096, fresh.size() - first));
                VkDeviceSize list = RingAllocate(VkDeviceSize(count) * 8);
                if (list == ~VkDeviceSize(0))
                    return;
                uint32_t* out = reinterpret_cast<uint32_t*>(s_ringHost + list);
                for (uint32_t i = 0; i < count; i++)
                {
                    out[i * 2] = fresh[first + i].first;
                    out[i * 2 + 1] = fresh[first + i].second;
                    EnsureShadow(fresh[first + i].first << 12, 4096);
                }
                EndPass();
                Begin();
                VkDescriptorBufferInfo bi[3] = { s_sharedInfo[0], { s_scaled.pool, 0, VK_WHOLE_SIZE }, { s_ring, 0, kRingSize } };
                VkWriteDescriptorSet w[6];
                for (uint32_t k = 0; k < 3; k++)
                    w[k] = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, k, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
                        nullptr, &bi[k], nullptr };
                const gpu::RenderScale& sc = s_scaled.scale;
                struct { uint32_t count, slotWords, listWord, pad, scaleX, scaleY, denX, denY; } pc = { count, s_scaled.slotWords,
                    uint32_t(list / 4), 0, sc.x.num, sc.y.num, sc.x.den, sc.y.den };
                s_stateEpoch++;
                vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_scaledFillPipeline);
                s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_scaledFillLayout, 0, AddSharedParts(w, 3), w);
                vkCmdPushConstants(s_cmd, s_scaledFillLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
                vkCmdDispatch(s_cmd, (count * 1024 + 63) / 64, 1, 1);
                FullBarrier();
            }
        }

        // Front buffer -> RGBA8 image on the GPU (the presenter blits it), so a
        // swap needs neither a CPU wait nor a CPU untile.
        const char* kFrontGlsl = R"(#version 460
layout(local_size_x = 8, local_size_y = 8) in;
layout(std430, set = 0, binding = 0) readonly buffer SharedMemory { uint g_mem[]; };)" GLSL_SHARED_MEMORY_READ R"(
layout(set = 0, binding = 1, rgba8) writeonly uniform image2D outImage;
layout(std430, set = 0, binding = 2) readonly buffer ScaledPool { uint s_pool[]; };
layout(std430, set = 0, binding = 3) readonly buffer ScaledTable { uint s_table[]; };
// width, height: the output (host pixels when scaled: from the scaled memory,
// at sx / denX x sy / denY host pixels per guest pixel).
layout(push_constant) uniform P { uint base, pitch, tiled, endian, format, swizzle, width, height, sx, sy, slotWords, scaled, denX, denY; } p;
)" GLSL_SCALE_CODE R"(
uint tiled2D(uint x, uint y, uint pitch, uint bppLog2)
{
    uint outer = (((y >> 5) * (pitch >> 5)) + (x >> 5)) << 6;
    uint inner = (((y >> 1) & 7u) << 3) | (x & 7u);
    uint oi = (outer | inner) << bppLog2;
    uint bank = (y >> 4) & 1u;
    uint pipe = ((x >> 3) & 3u) ^ (((y >> 3) & 1u) << 1);
    return ((y & 1u) << 4) | (pipe << 6) | (bank << 11) | (oi & 0xFu) | (((oi >> 4) & 1u) << 5) |
        (((oi >> 5) & 7u) << 8) | ((oi >> 8) << 12);
}

void main()
{
    uvec2 g = gl_GlobalInvocationID.xy;
    if (g.x >= p.width || g.y >= p.height) return;
    uint v;
    if (p.scaled != 0u)
    {
        // Host pixel g: sample (g mod scale) of the block (g / scale), whose
        // first texel is guest (block x den) (gpu/render_scale.h).
        uint bx = g.x / p.sx, by = g.y / p.sy;
        uint a = (p.base + tiled2D(bx * p.denX, by * p.denY, p.pitch, 2u)) & 0x1FFFFFFFu;
        uint slot = s_table[a >> 12];
        v = slot != 0u ? s_pool[(slot - 1u) * p.slotWords + sbSampleWord((a & 4095u) >> 2, g.x - bx * p.sx, g.y - by * p.sy, p.sx, p.sy,
            p.denX, p.denY)] : 0u;
    }
    else
    {
        uint addr = p.base + (p.tiled != 0u ? tiled2D(g.x, g.y, p.pitch, 2u) : (g.y * p.pitch + g.x) * 4u);
        v = memLoad((addr & 0x1FFFFFFFu) >> 2);
    }
    if (p.endian == 1u) v = ((v << 8) & 0xFF00FF00u) | ((v >> 8) & 0x00FF00FFu);
    else if (p.endian == 2u) v = (v << 24) | ((v << 8) & 0x00FF0000u) | ((v >> 8) & 0x0000FF00u) | (v >> 24);
    else if (p.endian == 3u) v = (v >> 16) | (v << 16);
    float c[6];
    if (p.format == 7u || p.format == 54u)
    {
        c[0] = float(v & 0x3FFu) / 1023.0; c[1] = float((v >> 10) & 0x3FFu) / 1023.0;
        c[2] = float((v >> 20) & 0x3FFu) / 1023.0; c[3] = float(v >> 30) / 3.0;
    }
    else
    {
        for (int i = 0; i < 4; i++) c[i] = float((v >> (8 * i)) & 0xFFu) / 255.0;
    }
    c[4] = 0.0; c[5] = 1.0;
    vec3 rgb = vec3(c[min(p.swizzle & 7u, 5u)], c[min((p.swizzle >> 3) & 7u, 5u)], c[min((p.swizzle >> 6) & 7u, 5u)]);
    imageStore(outImage, ivec2(g), vec4(rgb, 1.0));
}
)";

        VkPipeline s_frontPipeline = VK_NULL_HANDLE;
        VkPipelineLayout s_frontLayout = VK_NULL_HANDLE;
        VkDescriptorSetLayout s_frontSetLayout = VK_NULL_HANDLE;
        // The front buffer as presented: 1x, and at host resolution when the
        // scaled resolve memory has it (two images: no stall when the game
        // switches between them, e.g. around movies).
        Image s_front, s_frontScaled;
        uint32_t s_frontWidth = 0, s_frontHeight = 0, s_frontScaledWidth = 0, s_frontScaledHeight = 0;

        // A front image the presenter may still show or be reading: taken
        // back from it, then destroyed once the queue is idle.
        void DestroyFrontImage(Image& front, uint32_t& width, uint32_t& height)
        {
            Flush();
            if (front.image)
                video::ReleaseFrontImage(front.image);
            video::WaitGpuIdle();
            DestroyImage(front);
            width = height = 0;
        }

        void CreateUntilePipeline()
        {
            VkDescriptorSetLayoutBinding b[2] = {
                { 0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
            };
            auto bindings = WithSharedParts(b, 2);
            CreateComputePipeline(kUntileGlsl, bindings.data(), uint32_t(bindings.size()), sizeof(UntileConstants), s_untileSetLayout,
                s_untileLayout, s_untilePipeline);
            b[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
            bindings = WithSharedParts(b, 2);
            CreateComputePipeline(kUntileGlsl, bindings.data(), uint32_t(bindings.size()), sizeof(UntileConstants), s_untileStoreSetLayout,
                s_untileStoreLayout, s_untileStorePipeline, "#define TO_IMAGE\n");
        }

        // The resolve shader for sources whose scale has these denominators
        // (1 or 2 each; the shader's SB_DEN_X, SB_DEN_Y), into `out`.
        void MakeResolvePipeline(bool coalesced, uint32_t denX, uint32_t denY, VkPipeline* out)
        {
            std::string log;
            std::string preamble = coalesced ? "#define COALESCED\n" : "";
            if (s_resolvePush != offsetof(ResolveConstants, texX))
                preamble += "#define TEX_OFFSET\n";
            if (denX != 1)
                preamble += std::format("#define SB_DEN_X {}u\n", denX);
            if (denY != 1)
                preamble += std::format("#define SB_DEN_Y {}u\n", denY);
            auto spirv = CompileGlsl(kResolveGlsl, GLSLANG_STAGE_COMPUTE, log, preamble);
            if (spirv.empty())
            {
                fprintf(stderr, "[renderer] resolve shader failed:\n%s\n", log.c_str());
                abort();
            }
            VkShaderModule module = CreateModule(spirv);
            VkComputePipelineCreateInfo cpci{ VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO };
            cpci.stage = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_COMPUTE_BIT,
                module, "main", nullptr };
            cpci.layout = s_resolveLayout;
            Check(vkCreateComputePipelines(s_dev, s_pipelineCache, 1, &cpci, nullptr, out), "resolve pipeline");
        }

        // The resolve pipeline for a source at `scale` (see s_resolveDenPipelines).
        VkPipeline ResolvePipeline(bool coalesced, const gpu::RenderScale& scale)
        {
            const uint32_t dx = std::clamp(scale.x.den, 1u, 2u), dy = std::clamp(scale.y.den, 1u, 2u);
            VkPipeline& slot = s_resolveDenPipelines[coalesced][dx - 1][dy - 1];
            if (!slot)
            {
                auto start = std::chrono::steady_clock::now();
                MakeResolvePipeline(coalesced, dx, dy, &slot);
                fprintf(stderr, "[renderer] resolve pipeline for scale denominators %ux%u%s made (%.1f ms)\n", dx, dy,
                    coalesced ? ", coalesced" : "", std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count());
            }
            return slot;
        }

        void CreateResolvePipeline()
        {
            VkDescriptorSetLayoutBinding b[5] = {
                { 0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 1, VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 2, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },  // scaled pool
                { 3, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },  // scaled page table
                { 4, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },   // a cached texture (or a dummy)
            };
            auto bindings = WithSharedParts(b, 5);
            VkDescriptorSetLayoutCreateInfo dslci{ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO };
            dslci.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
            dslci.bindingCount = uint32_t(bindings.size());
            dslci.pBindings = bindings.data();
            Check(vkCreateDescriptorSetLayout(s_dev, &dslci, nullptr, &s_resolveSetLayout), "resolve set layout");
            VkPushConstantRange range{ VK_SHADER_STAGE_COMPUTE_BIT, 0, s_resolvePush };
            VkPipelineLayoutCreateInfo plci{ VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO };
            plci.setLayoutCount = 1;
            plci.pSetLayouts = &s_resolveSetLayout;
            plci.pushConstantRangeCount = 1;
            plci.pPushConstantRanges = &range;
            Check(vkCreatePipelineLayout(s_dev, &plci, nullptr, &s_resolveLayout), "resolve pipeline layout");
            for (bool coalesced : { false, true })
                MakeResolvePipeline(coalesced, 1, 1, coalesced ? &s_resolveCoalescedPipeline : &s_resolvePipeline);
            s_resolveDenPipelines[0][0][0] = s_resolvePipeline;
            s_resolveDenPipelines[1][0][0] = s_resolveCoalescedPipeline;
            PrewarmResolvePipelines(s_renderScale);
        }

        // ---------------------------------------------------------------

        // Where guest memory and the upload ring go: gpu/memory_plan.h, which
        // lists the places to try in order. A place the driver refuses (out of
        // memory, say) is logged and the next one tried; only when none is
        // left does the renderer give up.
        memplan::Memory s_planMemory;  // the plan's view of the device's memory
        uint32_t s_guestType = memplan::kNoType;  // the copy's memory type (kNoType: imported)
        std::string s_guestChoice;  // for the [renderer] memory line

        // Test hooks for the plan's fallbacks, which the machines we test on
        // never need:
        //   NFSMW_TEST_BAR_HEAP_MB=<MB>: the heaps holding CPU-mapped VRAM read
        //     as that size to the plan, and the copy's and the ring's
        //     allocations from those types fail (out of device memory) once
        //     they would pass it, as on a GPU without resizable BAR;
        //   NFSMW_TEST_ALLOC_FAIL=<places>: those places fail as out of device
        //     memory (comma-separated: import, copy-vram, copy-system,
        //     copy-cached, ring-vram, ring-cached, ring-system);
        //   NFSMW_TEST_GPU_KIND=integrated|discrete|other: the plan takes the
        //     GPU for that kind (the Deck's choice on the Steam Machine, say).
        struct MemoryTest
        {
            uint64_t barBytes = 0, barUsed = 0;
            std::string fail;  // ",a,b,"
            const char* kind = nullptr;
        };
        MemoryTest s_memoryTest = [] {
            MemoryTest t;
            if (const char* v = std::getenv("NFSMW_TEST_BAR_HEAP_MB"); v && std::atoi(v) > 0)
                t.barBytes = uint64_t(std::atoi(v)) << 20;
            if (const char* v = std::getenv("NFSMW_TEST_ALLOC_FAIL"); v && v[0])
                t.fail = std::string(",") + v + ",";
            t.kind = std::getenv("NFSMW_TEST_GPU_KIND");
            return t;
        }();

        // NFSMW_TEST_ALLOC_FAIL names this place.
        bool MemoryTestRefuses(memplan::Place place, bool ring)
        {
            const char* name = memplan::Name(place, ring);
            if (s_memoryTest.fail.find(std::string(",") + name + ",") == std::string::npos)
                return false;
            fprintf(stderr, "[renderer] NFSMW_TEST_ALLOC_FAIL: %s refused (out of device memory)\n", name);
            return true;
        }

        // vkAllocateMemory for a planned place, through the test hooks.
        VkResult AllocatePlanned(const memplan::Candidate& c, bool ring, VkDeviceSize size, VkDeviceMemory& memory)
        {
            if (MemoryTestRefuses(c.place, ring))
                return VK_ERROR_OUT_OF_DEVICE_MEMORY;
            const VkMemoryPropertyFlags bar = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
            bool mappedVram = (s_memProps.memoryTypes[c.type].propertyFlags & bar) == bar;
            if (mappedVram && s_memoryTest.barBytes && s_memoryTest.barUsed + size > s_memoryTest.barBytes)
            {
                fprintf(stderr, "[renderer] NFSMW_TEST_BAR_HEAP_MB: %llu MB of CPU-mapped VRAM refused (%llu of %llu MB taken)\n",
                    (unsigned long long)(size >> 20), (unsigned long long)(s_memoryTest.barUsed >> 20),
                    (unsigned long long)(s_memoryTest.barBytes >> 20));
                return VK_ERROR_OUT_OF_DEVICE_MEMORY;
            }
            VkMemoryAllocateInfo mai{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
            mai.allocationSize = size;
            mai.memoryTypeIndex = c.type;
            VkResult r = vkAllocateMemory(s_dev, &mai, nullptr, &memory);
            if (r == VK_SUCCESS && mappedVram)
                s_memoryTest.barUsed += size;
            return r;
        }

        // The device's memory as the plan sees it (NFSMW_TEST_BAR_HEAP_MB
        // applied), logged once: bug reports then show what a GPU offers.
        memplan::Memory PlanMemory()
        {
            memplan::Memory m;
            std::string line;
            for (uint32_t h = 0; h < s_memProps.memoryHeapCount; h++)
            {
                m.heaps.push_back(s_memProps.memoryHeaps[h].size);
                line += std::format("{}{} {}{}", h ? ", " : "", h, memplan::Size(m.heaps.back()),
                    (s_memProps.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) ? " device-local" : "");
            }
            line += "; types (type:heap/flags):";
            for (uint32_t i = 0; i < s_memProps.memoryTypeCount; i++)
            {
                m.types.push_back({ s_memProps.memoryTypes[i].propertyFlags, s_memProps.memoryTypes[i].heapIndex });
                line += std::format(" {}:{}/{:X}", i, m.types.back().heap, m.types.back().flags);
            }
            if (s_memoryTest.barBytes)
            {
                memplan::ShrinkMappedVram(m, s_memoryTest.barBytes);
                line += std::format(" (NFSMW_TEST_BAR_HEAP_MB: CPU-mapped VRAM's heaps read as {} MB)", s_memoryTest.barBytes >> 20);
            }
            fprintf(stderr, "[renderer] memory heaps: %s\n", line.c_str());
            return m;
        }

        uint32_t DriverId(const VkPhysicalDeviceProperties& props)
        {
            if (props.apiVersion < VK_API_VERSION_1_2)
                return 0;
            VkPhysicalDeviceDriverProperties driver{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES };
            VkPhysicalDeviceProperties2 props2{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2 };
            props2.pNext = &driver;
            vkGetPhysicalDeviceProperties2(s_vk->physical, &props2);
            return uint32_t(driver.driverID);
        }

        // Guest physical memory imported without a copy (unified memory, or
        // a discrete GPU reading system memory over PCIe).
        bool ImportSharedMemory()
        {
            if (!s_vk->getMemoryHostPointerProperties)  // (no host-memory import)
                return false;
            if (MemoryTestRefuses(memplan::Place::kImport, false))
            {
                fprintf(stderr, "[renderer] importing guest memory failed (%d)\n", int(VK_ERROR_OUT_OF_DEVICE_MEMORY));
                return false;
            }
            VkMemoryHostPointerPropertiesEXT hpp{ VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT };
            if (s_vk->getMemoryHostPointerProperties(s_dev, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
                    s_sharedHost, &hpp) != VK_SUCCESS)
                return false;

            VkExternalMemoryBufferCreateInfo embci{ VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO };
            embci.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
            VkBufferCreateInfo bci{ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
            bci.pNext = &embci;
            bci.size = kSharedSize;
            bci.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
            Check(vkCreateBuffer(s_dev, &bci, nullptr, &s_shared), "vkCreateBuffer(shared)");
            VkMemoryRequirements req;
            vkGetBufferMemoryRequirements(s_dev, s_shared, &req);

            VkImportMemoryHostPointerInfoEXT import{ VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT };
            import.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
            import.pHostPointer = s_sharedHost;
            VkMemoryAllocateInfo mai{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
            mai.pNext = &import;
            mai.allocationSize = kSharedSize;
            mai.memoryTypeIndex = FindMemoryType(req.memoryTypeBits & hpp.memoryTypeBits,
                VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
            VkResult r = vkAllocateMemory(s_dev, &mai, nullptr, &s_sharedMemory);
            if (r == VK_SUCCESS && (r = vkBindBufferMemory(s_dev, s_shared, s_sharedMemory, 0)) != VK_SUCCESS)
            {
                vkFreeMemory(s_dev, s_sharedMemory, nullptr);
                s_sharedMemory = VK_NULL_HANDLE;
            }
            if (r != VK_SUCCESS)
            {
                fprintf(stderr, "[renderer] importing guest memory failed (%d)\n", int(r));
                vkDestroyBuffer(s_dev, s_shared, nullptr);
                s_shared = VK_NULL_HANDLE;
                return false;
            }
            return true;
        }

        // The buffer of guest memory's copy (an unbound one: a place that
        // fails before binding leaves it for the next).
        VkBuffer NewCopyBuffer(VkMemoryRequirements& req)
        {
            VkBufferCreateInfo bci{ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
            bci.size = kSharedSize;
            bci.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
            VkBuffer buffer = VK_NULL_HANDLE;
            Check(vkCreateBuffer(s_dev, &bci, nullptr, &buffer), "vkCreateBuffer(shadow)");
            vkGetBufferMemoryRequirements(s_dev, buffer, &req);
            return buffer;
        }

        // Guest memory's copy for the GPU in `c`'s memory type, kept current
        // page by page from the write watch (writewatch::SyncShadow): on a
        // discrete GPU in VRAM the CPU writes through resizable BAR when it
        // holds the copy, else in system memory; on an integrated one in
        // system memory (memory_plan.h). False, with nothing kept, when the
        // driver refuses it; `buffer` is then still unbound, or replaced.
        bool CreateShadowSharedMemory(VkBuffer& buffer, const VkMemoryRequirements& req, const memplan::Candidate& c)
        {
            VkDeviceMemory memory = VK_NULL_HANDLE;
            void* p = nullptr;
            const char* what = "vkAllocateMemory(shadow)";
            VkResult r = AllocatePlanned(c, false, req.size, memory);
            if (r == VK_SUCCESS && (r = vkMapMemory(s_dev, memory, 0, VK_WHOLE_SIZE, 0, &p)) != VK_SUCCESS)
                what = "vkMapMemory(shadow)";
            else if (r == VK_SUCCESS && (r = vkBindBufferMemory(s_dev, buffer, memory, 0)) != VK_SUCCESS)
            {
                what = "vkBindBufferMemory(shadow)";
                vkDestroyBuffer(s_dev, buffer, nullptr);  // (a buffer binds once)
                VkMemoryRequirements again;
                buffer = NewCopyBuffer(again);
            }
            if (r != VK_SUCCESS)
            {
                fprintf(stderr, "[renderer] %s failed: %d (%s, memory type %u): trying the next place\n", what, int(r),
                    memplan::Name(c.place, false), c.type);
                if (memory)
                    vkFreeMemory(s_dev, memory, nullptr);
                return false;
            }
            s_shared = buffer;
            s_sharedMemory = memory;
            // Write every page of the mapping once, before anything uses it.
            // The first CPU write to a page of mapped VRAM faults it into the
            // page tables; left to the first sync, a burst of new textures
            // paid it mid-frame (on the Steam Machine, copying 7.9 MB of
            // never-copied pages took 3.9 ms instead of 2.9, a scene load's
            // 44 MB 22 ms instead of 14-17), while all 512 MB in order take
            // ~100 ms here. The zeros are never read: SyncShadow copies a page
            // before its first use. NFSMW_SHADOW_PREFAULT=0 leaves the faults
            // to the first sync.
            static const bool prefault = [] { const char* v = std::getenv("NFSMW_SHADOW_PREFAULT"); return !v || v[0] != '0'; }();
            auto prefaultStart = std::chrono::steady_clock::now();
            if (prefault)
                for (VkDeviceSize offset = 0; offset < kSharedSize; offset += 4096)
                    static_cast<volatile uint8_t*>(p)[offset] = 0;
            double prefaultMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - prefaultStart).count();
            writewatch::EnableShadow(static_cast<uint8_t*>(p));
            s_shadowMode = true;
            s_shadowInVram = (s_memProps.memoryTypes[c.type].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0;
            fprintf(stderr, "[renderer] guest memory: GPU copy in %s (memory type %u, flags %X), updated from the write watch%s\n",
                s_shadowInVram ? "VRAM (CPU-mapped)" : "host memory", c.type, s_memProps.memoryTypes[c.type].propertyFlags,
                prefault ? std::format(" (pages touched in {:.0f} ms)", prefaultMs).c_str() : "");
            return true;
        }

        // `failed`, a list of place names, as the log's ", after ... failed".
        std::string AfterFailed(const std::string& failed)
        {
            return failed.empty() ? std::string() : " after " + failed + " failed";
        }

        void CreateSharedMemory()
        {
            s_sharedHost = static_cast<uint8_t*>(g_memory.Translate(vmem::Physical().Base()));
            VkPhysicalDeviceProperties props;
            vkGetPhysicalDeviceProperties(s_vk->physical, &props);
            VkPhysicalDeviceFeatures features;
            vkGetPhysicalDeviceFeatures(s_vk->physical, &features);  // the presenter enables anisotropy when supported
            s_maxAnisotropy = features.samplerAnisotropy ? props.limits.maxSamplerAnisotropy : 1.0f;
            s_autoAnisotropy = props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 16 : 4;
#if defined(__APPLE__) && TARGET_OS_IOS
            // An M-series iPad renders at 1x (ReadRenderScale), where 16x
            // cost nothing measurable (iPad Pro M2, scripted race: 60 fps in
            // the race and in the heavy windows, as at 4x). The A-series
            // GPUs of iPhones are unmeasured: 4x there.
            if (std::string_view(props.deviceName).starts_with("Apple M"))
                s_autoAnisotropy = 16;
#endif
            {
                int anisotropy = settings::GetInt(settings::Id::Anisotropy);
                if (anisotropy == settings::kAnisotropyAuto)
                    anisotropy = s_autoAnisotropy;
                fprintf(stderr, "[renderer] anisotropic filtering: %s\n",
                    anisotropy < 0 ? "the game's" : std::format("{}x", std::min(float(anisotropy), s_maxAnisotropy)).c_str());
            }

            s_planMemory = PlanMemory();
            memplan::GuestInput in;
            in.gpu = props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? memplan::Gpu::kDiscrete
                : props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? memplan::Gpu::kIntegrated : memplan::Gpu::kOther;
            if (const char* kind = s_memoryTest.kind)
            {
                std::string_view k(kind);
                in.gpu = k == "integrated" ? memplan::Gpu::kIntegrated : k == "discrete" ? memplan::Gpu::kDiscrete
                    : k == "other" ? memplan::Gpu::kOther : in.gpu;
                fprintf(stderr, "[renderer] NFSMW_TEST_GPU_KIND=%s: memory planned as for %s GPU\n", kind,
                    in.gpu == memplan::Gpu::kDiscrete ? "a discrete" : in.gpu == memplan::Gpu::kIntegrated ? "an integrated" : "another");
            }
            in.importable = s_vk->getMemoryHostPointerProperties != nullptr;
            in.importRefusal = memplan::ImportRefusal(DriverId(props));
            in.overrides = memplan::ParseOverrides(std::getenv("NFSMW_SHARED_MEMORY"), std::getenv("NFSMW_SHADOW_MEMORY"),
                std::getenv("NFSMW_RING_MEMORY"));
            VkMemoryRequirements req;
            VkBuffer copy = NewCopyBuffer(req);
            in.bits = req.memoryTypeBits;
            memplan::Plan plan = memplan::PlanGuestMemory(s_planMemory, in);
            std::string failed;
            const memplan::Candidate* placed = nullptr;
            for (const memplan::Candidate& c : plan.order)
            {
                if (c.place == memplan::Place::kImport ? ImportSharedMemory() : CreateShadowSharedMemory(copy, req, c))
                {
                    placed = &c;
                    break;
                }
                failed += std::string(failed.empty() ? "" : ", ") + memplan::Name(c.place, false);
            }
            if (!placed)
            {
                fprintf(stderr, "[renderer] guest memory: no place could hold it (%s failed; %s)\n", failed.c_str(), plan.why.c_str());
                abort();
            }
            if (placed->place == memplan::Place::kImport)
            {
                vkDestroyBuffer(s_dev, copy, nullptr);
                fprintf(stderr, "[renderer] guest memory: imported (zero-copy)\n");
                s_guestChoice = "imported (zero-copy)";
            }
            else
            {
                s_guestType = placed->type;
                s_guestChoice = std::format("as a GPU copy in {}, memory type {}", placed->place == memplan::Place::kVram ? "CPU-mapped VRAM"
                    : placed->place == memplan::Place::kCached ? "cached host memory" : "host memory", placed->type);
            }
            s_guestChoice += " (" + plan.why + ")" + AfterFailed(failed);
        }

        void CreateRing()
        {
            VkBufferCreateInfo bci{ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
            bci.size = kRingSize;
            bci.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
            Check(vkCreateBuffer(s_dev, &bci, nullptr, &s_ring), "vkCreateBuffer(ring)");
            VkMemoryRequirements req;
            vkGetBufferMemoryRequirements(s_dev, s_ring, &req);
            // Device-local when the host can map it (resizable BAR) and it
            // fits, so the GPU reads constants and indices from VRAM; else
            // cached system memory (memory_plan.h). NFSMW_RING_MEMORY=cached
            // prefers cached system memory anyway: the CPU's ~40 MB of
            // writes a frame then aren't write-combined PCIe stores (an A/B
            // experiment: on the Steam Machine 2% less CPU, twice the GPU
            // time); =vram|system name the other places.
            memplan::Overrides o = memplan::ParseOverrides(nullptr, nullptr, std::getenv("NFSMW_RING_MEMORY"));
            memplan::Plan plan = memplan::PlanRing(s_planMemory, req.memoryTypeBits, s_guestType, o.ring);
            std::string failed;
            const memplan::Candidate* placed = nullptr;
            for (const memplan::Candidate& c : plan.order)
            {
                VkDeviceMemory memory = VK_NULL_HANDLE;
                void* p = nullptr;
                const char* what = "vkAllocateMemory(ring)";
                VkResult r = AllocatePlanned(c, true, req.size, memory);
                if (r == VK_SUCCESS && (r = vkMapMemory(s_dev, memory, 0, VK_WHOLE_SIZE, 0, &p)) != VK_SUCCESS)
                    what = "vkMapMemory(ring)";
                else if (r == VK_SUCCESS && (r = vkBindBufferMemory(s_dev, s_ring, memory, 0)) != VK_SUCCESS)
                {
                    what = "vkBindBufferMemory(ring)";
                    vkDestroyBuffer(s_dev, s_ring, nullptr);  // (a buffer binds once)
                    Check(vkCreateBuffer(s_dev, &bci, nullptr, &s_ring), "vkCreateBuffer(ring)");
                }
                if (r != VK_SUCCESS)
                {
                    fprintf(stderr, "[renderer] %s failed: %d (%s, memory type %u): trying the next place\n", what, int(r),
                        memplan::Name(c.place, true), c.type);
                    if (memory)
                        vkFreeMemory(s_dev, memory, nullptr);
                    failed += std::string(failed.empty() ? "" : ", ") + memplan::Name(c.place, true);
                    continue;
                }
                fprintf(stderr, "[renderer] upload ring: memory type %u (flags %X)\n", c.type, s_memProps.memoryTypes[c.type].propertyFlags);
                s_ringMemory = memory;
                s_ringHost = static_cast<uint8_t*>(p);
                placed = &c;
                break;
            }
            if (!placed)
            {
                fprintf(stderr, "[renderer] upload ring: no place could hold it (%s failed; %s)\n", failed.c_str(), plan.why.c_str());
                abort();
            }
            // The choice and why, in one line.
            fprintf(stderr, "[renderer] memory: guest memory %s; upload ring in %s, memory type %u (%s)%s\n", s_guestChoice.c_str(),
                placed->place == memplan::Place::kVram ? "CPU-mapped VRAM" : placed->place == memplan::Place::kCached ? "cached host memory"
                    : "host memory", placed->type, plan.why.c_str(), AfterFailed(failed).c_str());
        }

        void CreateSet0()
        {
            VkDescriptorSetLayoutBinding b[3] = {
                { 0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, nullptr },
                { 1, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, 1, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, nullptr },
                { 2, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, nullptr },
            };
            auto bindings = WithSharedParts(b, 3);
            VkDescriptorSetLayoutCreateInfo dslci{ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO };
            dslci.bindingCount = uint32_t(bindings.size());
            dslci.pBindings = bindings.data();
            Check(vkCreateDescriptorSetLayout(s_dev, &dslci, nullptr, &s_set0Layout), "set 0 layout");
            VkDescriptorPoolSize sizes[2] = { { VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, s_splitMemory ? 5u : 2u },
                { VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, 1 } };
            VkDescriptorPoolCreateInfo dpci{ VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO };
            dpci.maxSets = 1;
            dpci.poolSizeCount = 2;
            dpci.pPoolSizes = sizes;
            Check(vkCreateDescriptorPool(s_dev, &dpci, nullptr, &s_descriptorPool), "descriptor pool");
            VkDescriptorSetAllocateInfo dsai{ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO };
            dsai.descriptorPool = s_descriptorPool;
            dsai.descriptorSetCount = 1;
            dsai.pSetLayouts = &s_set0Layout;
            Check(vkAllocateDescriptorSets(s_dev, &dsai, &s_set0), "set 0");
            VkDescriptorBufferInfo constants{ s_ring, 0, kConstantsSize };
            VkDescriptorBufferInfo ring{ s_ring, 0, kRingSize };
            VkWriteDescriptorSet w[6] = {
                { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, s_set0, 0, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &s_sharedInfo[0], nullptr },
                { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, s_set0, 1, 0, 1, VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, nullptr, &constants, nullptr },
                { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, s_set0, 2, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &ring, nullptr },
            };
            vkUpdateDescriptorSets(s_dev, AddSharedParts(w, 3, s_set0), w, 0, nullptr);
        }

        void CreateSamplerAndDummies()
        {
            VkSamplerCreateInfo sci{ VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO };
            sci.magFilter = VK_FILTER_LINEAR;
            sci.minFilter = VK_FILTER_LINEAR;
            sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
            sci.addressModeU = sci.addressModeV = sci.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT;
            sci.maxLod = VK_LOD_CLAMP_NONE;
            Check(vkCreateSampler(s_dev, &sci, nullptr, &s_sampler), "vkCreateSampler");

            VkImageUsageFlags usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
            s_dummy[0] = CreateImage(VK_IMAGE_TYPE_2D, VK_FORMAT_R8G8B8A8_UNORM, { 1, 1, 1 }, 1, usage,
                VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT);
            s_dummy[1] = CreateImage(VK_IMAGE_TYPE_3D, VK_FORMAT_R8G8B8A8_UNORM, { 1, 1, 1 }, 1, usage,
                VK_IMAGE_VIEW_TYPE_3D, VK_IMAGE_ASPECT_COLOR_BIT);
            s_dummy[2] = CreateImage(VK_IMAGE_TYPE_2D, VK_FORMAT_R8G8B8A8_UNORM, { 1, 1, 1 }, 6, usage,
                VK_IMAGE_VIEW_TYPE_CUBE, VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT);
            s_dummyStorage = CreateImage(VK_IMAGE_TYPE_2D, VK_FORMAT_R32_UINT, { 1, 1, 1 }, 1, VK_IMAGE_USAGE_STORAGE_BIT,
                VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT);
            Begin();
            ToGeneral(s_dummyStorage.image, VK_IMAGE_ASPECT_COLOR_BIT, 1);
            VkClearColorValue white{ { 1.0f, 1.0f, 1.0f, 1.0f } };
            for (int i = 0; i < 3; i++)
            {
                uint32_t layers = i == 2 ? 6 : 1;
                ToGeneral(s_dummy[i].image, VK_IMAGE_ASPECT_COLOR_BIT, layers);
                VkImageSubresourceRange range{ VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, layers };
                vkCmdClearColorImage(s_cmd, s_dummy[i].image, VK_IMAGE_LAYOUT_GENERAL, &white, 1, &range);
            }
            FullBarrier();
        }
    }

    namespace
    {
        // ---------------------------------------------------------------
        // Live settings. Internal Resolution and Mipmaps change between two
        // frames (OnSwap, on this thread), with the GPU idle (Flush: command
        // buffers in flight are the only other users of what is replaced).
        // What was built for the old value is made again from what the guest
        // has:
        //   - Internal Resolution: the render targets of the scaled pitches
        //     at the new scale (their EDRAM contents are lost; the game clears
        //     and redraws them every frame, and resolve clears still waiting
        //     move over), the scaled resolve memory (every page back to its
        //     1x copy until a scaled resolve writes it again) and the scaled
        //     textures. The front images follow at the next swap
        //     (PresentFrontBuffer replaces one of a new size), and the
        //     presenter's AA image with them.
        //   - Mipmaps: the whole texture cache, each texture loaded again
        //     from guest memory when a draw next uses it.
        // Auto follows the picture's size (the window, fullscreen, Aspect
        // Ratio) once it has held for kAutoSettleFrames, so a window being
        // dragged doesn't rebuild at every step; a change of the setting
        // itself applies at the next swap.
        uint64_t s_liveGeneration = ~0ull;
        int32_t s_resolutionSetting = 0;  // Internal Resolution as applied
        constexpr uint32_t kAutoSettleFrames = 30;
        gpu::RenderScale s_autoWant;
        uint32_t s_autoSeen = 0;

        void DestroyTexture(Texture& t)
        {
            if (auto it = s_directByBase.find(t.guestStart); it != s_directByBase.end() && it->second == &t)
                s_directByBase.erase(it);
            if (auto it = s_directByStart.find(t.guestStart); it != s_directByStart.end() && it->second == &t)
                s_directByStart.erase(it);
            for (auto& [swizzle, view] : t.views)
                vkDestroyImageView(s_dev, view, nullptr);
            t.views.clear();
            for (uint32_t layer = 1; layer < 6; layer++)  // ([0] is storeView)
                if (t.storeViews[layer])
                    vkDestroyImageView(s_dev, t.storeViews[layer], nullptr);
            std::fill(std::begin(t.storeViews), std::end(t.storeViews), VK_NULL_HANDLE);
            if (t.storeView)
                vkDestroyImageView(s_dev, t.storeView, nullptr);
            t.storeView = VK_NULL_HANDLE;
            DestroyImage(t.image);
        }

        // Drops the cached textures `which` picks (the GPU idle); returns how many.
        template <typename Which>
        size_t DropTextures(Which which)
        {
            s_textureCache->generation++;
            size_t dropped = 0;
            for (auto it = s_textures.begin(); it != s_textures.end();)
            {
                if (!which(*it->second))
                {
                    ++it;
                    continue;
                }
                DestroyTexture(*it->second);
                it = s_textures.erase(it);
                dropped++;
            }
            return dropped;
        }

        void ApplyRenderScale(const gpu::RenderScale& scale, const std::string& how)
        {
            auto start = std::chrono::steady_clock::now();
            const gpu::RenderScale old = s_renderScale;
            Flush();
            s_renderScale = scale;
            PrewarmResolvePipelines(scale);
            size_t targets = 0;
            for (auto& [key, slot] : s_targets)
            {
                RenderTarget& was = *slot;  // (keyed as before: the key has no scale)
                if (!ScaledPitch(was.pitch) || was.scale == scale)
                    continue;
                // Clears still waiting for their target's next pass, in guest
                // pixels (their host edges came from guest ones: GuestEdge
                // gives those back).
                std::vector<PendingClear> moved;
                std::erase_if(s_pendingClears, [&](const PendingClear& pc) {
                    if (pc.rt != &was)
                        return false;
                    const uint32_t x0 = was.scale.x.GuestEdge(uint32_t(pc.rect.offset.x)), y0 = was.scale.y.GuestEdge(uint32_t(pc.rect.offset.y));
                    const uint32_t x1 = was.scale.x.GuestEdge(uint32_t(pc.rect.offset.x) + pc.rect.extent.width);
                    const uint32_t y1 = was.scale.y.GuestEdge(uint32_t(pc.rect.offset.y) + pc.rect.extent.height);
                    moved.push_back({ nullptr, { { int32_t(x0), int32_t(y0) }, { x1 - x0, y1 - y0 } }, pc.value });
                    return true;
                });
                uint32_t base = was.base, format = was.format, pitch = was.pitch;
                bool depth = was.depth, wide = was.wide;
                DestroyTarget(was);
                slot = CreateTarget(base, format, depth, pitch, wide);
                for (PendingClear& pc : moved)
                {
                    pc.rt = slot.get();
                    pc.rect = HostRect(scale, pc.rect.offset.x, pc.rect.offset.y, pc.rect.offset.x + int32_t(pc.rect.extent.width),
                        pc.rect.offset.y + int32_t(pc.rect.extent.height));
                    s_pendingClears.push_back(pc);
                }
                targets++;
            }
            for (RecentTarget& recent : s_recentTargets)
                recent = {};
            size_t textures = DropTextures([](const Texture& t) { return t.scaled; });
            ResetScaledMemory(!scale.IsOne());
            fprintf(stderr, "[renderer] internal resolution %s -> %s (%ux%u, %s): %zu targets and %zu scaled textures replaced in %.1f ms\n",
                gpu::ScaleName(old).c_str(), gpu::ScaleName(scale).c_str(), scale.x.Size(1280), scale.y.Size(720), how.c_str(), targets, textures,
                std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count());
        }

        void ApplyMipmaps(bool mips)
        {
            auto start = std::chrono::steady_clock::now();
            Flush();
            s_loadMips = mips;
            size_t textures = DropTextures([](const Texture&) { return true; });
            fprintf(stderr, "[renderer] mipmaps %s: %zu textures dropped in %.1f ms (loaded again as they are drawn)\n",
                mips ? "on" : "off", textures, std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count());
        }

        void ApplyLiveSettings()
        {
            uint64_t generation = settings::Generation();
            if (generation != s_liveGeneration)
            {
                s_liveGeneration = generation;
                bool mips = settings::GetBool(settings::Id::Mipmaps);
                if (mips != s_loadMips)
                    ApplyMipmaps(mips);
            }
            if (s_scaleFixed)
                return;
            int32_t setting = settings::GetInt(settings::Id::InternalResolution);
            uint32_t regionW = 0, regionH = 0;
            if (setting <= 0)
            {
                video::FrameRegion(regionW, regionH);
                if (regionW == 0 || regionH == 0)
                    return;  // no picture to size it for: keep the scale
            }
            const gpu::RenderScale want = WantedRenderScale(setting, regionW, regionH);
            if (want == s_renderScale)
            {
                s_resolutionSetting = setting;
                s_autoSeen = 0;
                return;
            }
            // Only the picture's size moved (Auto): once it has settled.
            if (setting == s_resolutionSetting)
            {
                if (!(want == s_autoWant))
                {
                    s_autoWant = want;
                    s_autoSeen = 0;
                }
                if (++s_autoSeen < kAutoSettleFrames)
                    return;
            }
            s_resolutionSetting = setting;
            s_autoSeen = 0;
            ApplyRenderScale(want, ScaleReason(setting, regionW, regionH));
        }

        // ---------------------------------------------------------------
        // Occlusion queries counted on the GPU (NFSMW_OCCLUSION=1; see
        // ZpassDone).

        // Before a counted draw, inside its pass: the active segment while
        // the weight is the same, else a new one. None when the slot's
        // queries are used up (the draw isn't counted).
        void ZpdBeginSegment(uint32_t num, uint32_t den)
        {
            if (s_zpdQuery >= 0 && (num != s_zpdNum || den != s_zpdDen))
                ZpdEndSegment();
            if (s_zpdQuery >= 0)
                return;
            if (s_zpdUsed[s_slot] >= kZpdQueries)
            {
                s_zpdCounts.overflow++;
                LogOnce((0x2Aull << 56) | 2, "occlusion query: %u segments in one command buffer: draws past them aren't counted", kZpdQueries);
                return;
            }
            uint32_t q = s_zpdUsed[s_slot]++;
            vkCmdBeginQuery(s_cmd, s_zpdPool[s_slot], q, VK_QUERY_CONTROL_PRECISE_BIT);
            s_zpdQuery = int32_t(q);
            s_zpdNum = num;
            s_zpdDen = den;
            s_submissions[s_slot].zpdSegments.push_back({ s_zpdSegNext++, q, num, den });
            s_zpdCounts.segments++;
        }

        // At each swap. The game's BEGIN, quad and END follow each other in
        // one view, so a bracket still open came from an earlier frame: it
        // is dropped (an open bracket counts every draw and turns off
        // NFSMW_PASS_MERGE bit 1). Every 120 frames, before the [perf] line,
        // the [zpd] line.
        void ZpdOnSwap(uint64_t frame)
        {
            zpd::Bracket first;
            if (uint32_t dropped = s_zpdOpen.DropBefore(frame, &first))
            {
                s_zpdCounts.dropped += dropped;
                LogOnce((0x2Bull << 56) | first.beginAddress, "occlusion query: BEGIN %08X (frame %llu) still open at the swap: dropped (%u in all)",
                    first.beginAddress, (unsigned long long)first.frame, dropped);
                ZpdLog("f%llu swap: %u brackets dropped (the first: BEGIN %08X)", (unsigned long long)frame, dropped, first.beginAddress);
            }
            if (frame % 120 != 0)
                return;
            static ZpdCounts last{};
            static ZpdStores lastStores{};
            ZpdStores stores;
            {
                std::lock_guard lock(s_submitMutex);
                stores = s_zpdStores;
                s_zpdStores.delayMax = 0;
            }
            const ZpdCounts& c = s_zpdCounts;
            uint64_t stored = stores.stored - lastStores.stored;
            fprintf(stderr, "[zpd] per frame: %.1f ZPDs, %.1f ENDs (%.1f with samples), %.1f counted draws in %.1f segments"
                " | CP -> store %.2f ms avg, %.2f ms max | totals: %llu unmatched ENDs, %llu dropped and %llu replaced brackets,"
                " %llu other reports, %llu not available, %llu over the queries, %llu lost, %llu outside the viewport, %llu not clamped\n",
                double(c.zpds - last.zpds) / 120, double(stored) / 120, double(stores.nonZero - lastStores.nonZero) / 120,
                double(c.draws - last.draws) / 120, double(c.segments - last.segments) / 120,
                stored ? (stores.delaySum - lastStores.delaySum) / double(stored) : 0.0, stores.delayMax,
                (unsigned long long)c.unmatched, (unsigned long long)c.dropped, (unsigned long long)c.replaced,
                (unsigned long long)c.unknown, (unsigned long long)s_zpdUnavailable.load(), (unsigned long long)c.overflow,
                (unsigned long long)stores.lost, (unsigned long long)c.clamped, (unsigned long long)c.unclamped);
            last = c;
            lastStores = stores;
        }
    }

    bool Initialize(const uint32_t* regs)
    {
        s_vk = video::GetVulkan();
        if (!s_vk)
            return false;
        s_dev = s_vk->device;
        s_regs = regs;
        {
            VkPhysicalDeviceProperties props;
            vkGetPhysicalDeviceProperties(s_vk->physical, &props);
            ReadRenderScale(props);
            const char* split = std::getenv("NFSMW_SPLIT_MEMORY");
            s_splitMemory = split ? split[0] == '1' : props.limits.maxStorageBufferRange < kSharedSize;
            if (s_splitMemory)
                fprintf(stderr, "[renderer] guest memory bound as 4 parts of %llu MB (storage buffers bind up to %u MB here)\n",
                    (unsigned long long)(kSharedPart >> 20), props.limits.maxStorageBufferRange >> 20);
            s_resolutionSetting = settings::GetInt(settings::Id::InternalResolution);
            s_loadMips = settings::GetBool(settings::Id::Mipmaps);
            s_liveGeneration = settings::Generation();
            uint32_t count = 0;
            vkEnumerateDeviceExtensionProperties(s_vk->physical, nullptr, &count, nullptr);
            std::vector<VkExtensionProperties> extensions(count);
            vkEnumerateDeviceExtensionProperties(s_vk->physical, nullptr, &count, extensions.data());
            for (const VkExtensionProperties& e : extensions)
                s_memoryBudget |= strcmp(e.extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME) == 0;
        }
        vkGetPhysicalDeviceMemoryProperties(s_vk->physical, &s_memProps);
        s_pushDescriptorSet = reinterpret_cast<PFN_vkCmdPushDescriptorSetKHR>(
            vkGetDeviceProcAddr(s_dev, "vkCmdPushDescriptorSetKHR"));
        if (!s_pushDescriptorSet)
        {
            fprintf(stderr, "[renderer] VK_KHR_push_descriptor is missing; rendering disabled\n");
            s_vk = nullptr;
            return false;
        }
        glslang_initialize_process();
        CreatePipelineCache();

        VkCommandPoolCreateInfo cpci{ VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO };
        cpci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
        cpci.queueFamilyIndex = s_vk->queueFamily;
        Check(vkCreateCommandPool(s_dev, &cpci, nullptr, &s_commandPool), "vkCreateCommandPool");
        for (Submission& sub : s_submissions)
        {
            VkCommandBufferAllocateInfo cbai{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO };
            cbai.commandPool = s_commandPool;
            cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
            cbai.commandBufferCount = 1;
            Check(vkAllocateCommandBuffers(s_dev, &cbai, &sub.cmd), "vkAllocateCommandBuffers");
            VkFenceCreateInfo fci{ VK_STRUCTURE_TYPE_FENCE_CREATE_INFO };
            Check(vkCreateFence(s_dev, &fci, nullptr, &sub.fence), "vkCreateFence");
        }
        {
            VkPhysicalDeviceProperties props;
            vkGetPhysicalDeviceProperties(s_vk->physical, &props);
            s_timestampPeriod = props.limits.timestampPeriod;
            VkQueryPoolCreateInfo qpci{ VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO };
            qpci.queryType = VK_QUERY_TYPE_TIMESTAMP;
            qpci.queryCount = kSlots * 2;
            if (const char* v = std::getenv("NFSMW_GPU_TIMING"); v && v[0] == '1')
                if (vkCreateQueryPool(s_dev, &qpci, nullptr, &s_timestamps) != VK_SUCCESS)
                    s_timestamps = VK_NULL_HANDLE;
            if (s_profileFrame >= 0)
            {
                qpci.queryCount = 512;
                Check(vkCreateQueryPool(s_dev, &qpci, nullptr, &s_passQueries), "pass query pool");
            }
        }
        // NFSMW_OCCLUSION=1: a pool of occlusion queries per slot, so a reset
        // never touches queries a submission in flight uses. Each is reset
        // before its first use: here with host reset, else by the slot's
        // first Begin.
        if (s_vk->occlusionCounting)
        {
            VkQueryPoolCreateInfo qpci{ VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO };
            qpci.queryType = VK_QUERY_TYPE_OCCLUSION;
            qpci.queryCount = kZpdQueries;
            s_zpdOn = true;
            for (uint32_t k = 0; k < kSlots && s_zpdOn; k++)
            {
                s_zpdOn = vkCreateQueryPool(s_dev, &qpci, nullptr, &s_zpdPool[k]) == VK_SUCCESS;
                if (!s_zpdOn)
                    s_zpdPool[k] = VK_NULL_HANDLE;
                else if (s_vk->resetQueryPool)
                    s_vk->resetQueryPool(s_dev, s_zpdPool[k], 0, kZpdQueries);
                else
                    s_zpdUsed[k] = kZpdQueries;
            }
            if (s_zpdOn)
                s_zpdResults = std::make_unique<zpd::ResultRing>();
            else
            {
                for (VkQueryPool& pool : s_zpdPool)
                    if (pool)
                        vkDestroyQueryPool(s_dev, pool, nullptr);
                std::fill(std::begin(s_zpdPool), std::end(s_zpdPool), VK_NULL_HANDLE);
                std::fill(std::begin(s_zpdUsed), std::end(s_zpdUsed), 0u);
            }
            fprintf(stderr, s_zpdOn ? "[renderer] occlusion queries counted: %u per command buffer, %u command buffers\n"
                                    : "[renderer] occlusion query pools (%u x %u) couldn't be made: queries report zero samples\n",
                kZpdQueries, kSlots);
        }
        if (s_passBarrierLevel)
            fprintf(stderr, "[renderer] NFSMW_PASS_BARRIER=%u: %s leave the vertex stages out of their barriers%s\n", s_passBarrierLevel,
                s_passBarrierLevel == 1 ? "pass ends" : s_passBarrierLevel == 2 ? "pass ends and 1x resolves" : "1x resolves",
                s_passBarrierAlt ? " (every other [perf] window: NFSMW_PASS_BARRIER_ALT)" : "");
        if (s_untiledArea)
            fprintf(stderr, "[renderer] NFSMW_UNTILED_AREA%s: untiled passes load and store their draws' guest viewport, not the whole target\n",
                s_untiledArea == 2 ? "=alt (every other [perf] window)" : "");
        std::thread(CompletionThread).detach();
        if (s_imagePool)
            std::thread(MemoryThread).detach();

        CreateSharedMemory();
        SetSharedInfo();
        CreateRing();
        CreateSet0();
        CreateSamplerAndDummies();
        CreateResolvePipeline();
        CreateUntilePipeline();
        {
            VkDescriptorSetLayoutBinding b[4] = {
                { 0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 2, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
                { 3, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr },
            };
            auto bindings = WithSharedParts(b, 4);
            CreateComputePipeline(kFrontGlsl, bindings.data(), uint32_t(bindings.size()), 14 * sizeof(uint32_t), s_frontSetLayout,
                s_frontLayout, s_frontPipeline);
        }
        Flush();
        fprintf(stderr, "[renderer] ready: guest physical memory %s (%llu MB)\n",
            !s_shadowMode ? "imported" : s_shadowInVram ? "copied for the GPU, in VRAM" : "copied for the GPU, in host memory",
            (unsigned long long)(kSharedSize >> 20));
        return true;
    }

    bool Enabled()
    {
        return s_vk != nullptr;
    }

    void LoadShader(uint32_t type, const uint32_t* words, uint32_t dwords)
    {
        if (!s_vk || dwords == 0 || type > 1)
            return;
        auto hash = [type](const uint32_t* w, uint32_t n) {
            uint64_t h = 0xCBF29CE484222325ull ^ type;
            for (uint32_t i = 0; i < n; i++)
                h = (h ^ w[i]) * 0x100000001B3ull;
            return h;
        };
        ShaderCache& c = *s_shaderCache;
        ShaderCacheEntry* e = nullptr;
        if (c.on)
        {
            c.loads++;
            e = &c.entries[(uint64_t(reinterpret_cast<uintptr_t>(words)) * 0x9E3779B97F4A7C15ull) >> 54];
            if (e->words == words && e->dwords == dwords && e->type == type)
            {
                c.comparedBytes += size_t(dwords) * 4;
                if (memcmp(e->copy.data(), words, size_t(dwords) * 4) == 0)
                {
                    c.hits++;
                    if (c.verify)
                    {
                        auto it = s_shaders.find(hash(words, dwords));
                        Shader* full = it == s_shaders.end() ? nullptr : it->second.get();
                        if (full != e->shader && c.mismatches++ < 50)
                            fprintf(stderr, "[cpverify] shader at %p (%u dwords, type %u): cached %016llx, hash finds %016llx\n",
                                (const void*)words, dwords, type, (unsigned long long)e->shader->hash,
                                (unsigned long long)(full ? full->hash : 0));
                    }
                    (type == 0 ? s_vs : s_ps) = e->shader;
                    return;
                }
            }
            // Hashed from the copy, so the entry's words and shader agree
            // even if the guest writes them meanwhile.
            e->words = words;
            e->dwords = dwords;
            e->type = type;
            e->copy.assign(words, words + dwords);
            words = e->copy.data();
        }
        uint64_t h = hash(words, dwords);
        auto& slot = s_shaders[h];
        if (!slot)
        {
            HitchTimer hitchTimer(s_hitch.shaderMs);
            s_hitch.shaders++;
            slot = std::make_unique<Shader>();
            slot->stage = type == 0 ? ShaderStage::Vertex : ShaderStage::Pixel;
            slot->hash = h;
            std::vector<uint32_t> ucode(dwords);
            for (uint32_t i = 0; i < dwords; i++)
                ucode[i] = __builtin_bswap32(words[i]);
            slot->info = TranslateShader(slot->stage, ucode.data(), dwords);
            if (!slot->info.ok)
                fprintf(stderr, "[renderer] %s_%016llx: translation failed: %s\n", type == 0 ? "vs" : "ps",
                    (unsigned long long)h, slot->info.error.c_str());
            // Compiled to SPIR-V with its first pipeline, on a pipeline worker.
        }
        if (e)
            e->shader = slot.get();
        (type == 0 ? s_vs : s_ps) = slot.get();
    }

    void Draw(uint32_t initiatorValue, uint32_t dmaBase, uint32_t dmaSizeValue)
    {
        if (!s_vk)
            return;
        reg::VGT_DRAW_INITIATOR initiator;
        initiator.value = initiatorValue;
        auto edramMode = Reg<reg::RB_MODECONTROL>().edram_mode;
        if (edramMode == xenos::EdramMode::kCopy)
        {
            Resolve();
            return;
        }
        if (edramMode != xenos::EdramMode::kColorDepth && edramMode != xenos::EdramMode::kDepthOnly)
            return;
        // Depth-only: no pixel shader runs (depth_only.h); the guest's stays
        // bound for the draws after this one.
        struct RestorePs
        {
            Shader* ps;
            ~RestorePs() { s_ps = ps; }
        } restorePs{ s_ps };
        if (!gpu::RunsGuestPixelShader(edramMode, s_depthOnlyGuestPs))
            s_ps = DepthOnlyPixelShader();
        s_stats.draws++;
        auto skip = [&](uint64_t key, const char* why) {
            s_stats.skipped++;
            LogOnce(key, "draw skipped: %s", why);
        };
        if (!s_vs || !s_ps)
            return skip(1, "no shaders loaded");
        if (!s_vs->info.ok || !s_ps->info.ok)
            return skip(2, "a shader failed to translate");
        uint32_t count = initiator.num_indices;
        if (count == 0)
            return;
        if (initiator.source_select == xenos::SourceSelect::kImmediate)
            return skip(3, "immediate indices");
        auto viz = Reg<reg::PA_SC_VIZ_QUERY>();
        if (viz.viz_query_ena && viz.kill_pix_post_hi_z)
            return;

        auto surface = Reg<reg::RB_SURFACE_INFO>();
        uint32_t pitch = surface.surface_pitch;
        if (pitch == 0)
            return skip(4, "zero surface pitch");
        auto [tileShiftX, tileShiftY] = TileShift();
        if (tileShiftX != 0 || tileShiftY != 0)
        {
            s_stats.tileSkips++;
            return;  // drawn with tile 0 (untiled rendering)
        }
        const bool untiledPass = s_untiled && surface.msaa_samples != xenos::MsaaSamples::k1X &&
            Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable;

        // Render targets.
        RenderTarget* color[4] = {};
        auto colorMask = Reg<reg::RB_COLOR_MASK>();
        uint32_t writeMask = 0;
        if (edramMode == xenos::EdramMode::kColorDepth)
        {
            static const uint32_t infoRegs[4] = { XE_GPU_REG_RB_COLOR_INFO, XE_GPU_REG_RB_COLOR1_INFO,
                XE_GPU_REG_RB_COLOR2_INFO, XE_GPU_REG_RB_COLOR3_INFO };
            for (uint32_t i = 0; i < 4; i++)
            {
                uint32_t mask = (colorMask.value >> (4 * i)) & 0xF;
                if (!(s_ps->info.colorTargetMask & (1u << i)) || mask == 0)
                    continue;
                auto ci = RegAt<reg::RB_COLOR_INFO>(infoRegs[i]);
                color[i] = GetTarget(ci.color_base, uint32_t(ci.color_format), false, pitch);
                writeMask |= mask << (4 * i);
            }
        }
        auto depthControl = Reg<reg::RB_DEPTHCONTROL>();
        RenderTarget* depth = nullptr;
        if (depthControl.z_enable || depthControl.stencil_enable)
        {
            auto di = Reg<reg::RB_DEPTH_INFO>();
            depth = GetTarget(di.depth_base, uint32_t(di.depth_format), true, pitch);
        }
        if (!depth && writeMask == 0)
            return;  // nothing visible (memexport-only draws are not supported yet)
        // NFSMW_OCCLUSION=1, an occlusion query bracket open: this draw is
        // counted (ZpassDone).
        const bool zpdCount = s_zpdOn && !s_zpdOpen.Empty();
        // NFSMW_PASS_MERGE bit 1: nor is a depth-tested draw that writes no
        // colour, depth or stencil (the sun's occlusion quad: LEQUAL, no
        // writes), unless a query counts it: then it is what the query
        // counts, and bit 2 draws it in the view's open pass (depth tested
        // there, its colour masked). Otherwise the command processor answers
        // the queries (EVENT_WRITE_ZPD stores zero samples, VIZ_QUERY sets
        // its bit), so the draw changes nothing; its pass of its own does.
        if ((s_passMerge & 1) && !zpdCount && writeMask == 0 && !(depthControl.z_enable && depthControl.z_write_enable) &&
            !depthControl.stencil_enable)
        {
            LogOnce(s_vs->hash ^ (s_ps->hash << 1) ^ 0x2600, "pass merge: vs_%016llx ps_%016llx writes nothing (depth control %08X): skipped",
                (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash, depthControl.value);
            return;
        }
        // NFSMW_PASS_MERGE bit 2: a draw whose targets are some of the open
        // pass's (the sun's glow, colour only and additive, right after the
        // view's colour and depth draws) is drawn in that pass, with the
        // others attached but untouched: their colour writes masked, depth
        // and stencil tests off (the pipeline must have the pass's formats).
        // On its own it was a pass that loaded and stored its target (an
        // untiled one whole, 1280x2048) behind a full barrier, and the
        // view's pass started again after it. Only when no pixel anything
        // reads is lost: its area with its own targets is its area with the
        // pass's (the same pixels; untiled with NFSMW_UNTILED_AREA, both are
        // its viewport), or, untiled (that area is then the whole
        // target, and the pass's is shorter: the depth target's), with
        // clipping on and its viewport, the frame, inside the pass's
        // targets. What it loses then lies outside its viewport (the host
        // clips to the target, not the viewport) and at or beyond the pass's
        // extent: right only while nothing resolves rows at or beyond the
        // pass's extent (the frame's resolves read inside it). With clipping
        // off a draw may mean to reach past its viewport: it keeps its pass.
        uint32_t adoptedColor = 0;  // colour targets attached for the pass only
        bool adoptedDepth = false;
        if ((s_passMerge & 2) && s_passActive && (!depth || depth == s_passDepth))
        {
            bool subset = true, same = depth == s_passDepth;
            for (int i = 0; i < 4; i++)
            {
                subset = subset && (!color[i] || color[i] == s_passColor[i]);
                same = same && color[i] == s_passColor[i];
            }
            // As the scissor below (keep the two alike), in guest pixels,
            // with these targets (the pass's area is never larger: its
            // targets include these).
            auto areaWith = [&](RenderTarget* const* targets) {
                int32_t w = INT32_MAX, h = INT32_MAX;
                for (int i = 0; i < 5; i++)
                    if (targets[i])
                    {
                        w = std::min(w, int32_t(targets[i]->guestWidth));
                        h = std::min(h, int32_t(targets[i]->guestHeight));
                    }
                Rect sc = untiledPass ? UntiledScissor(targets, w, h) : GuestScissor();
                int32_t x0 = std::clamp(sc.x0, 0, w), y0 = std::clamp(sc.y0, 0, h);
                return Rect{ x0, y0, std::clamp(sc.x1, x0, w), std::clamp(sc.y1, y0, h) };
            };
            bool keeps = false;
            if (subset && !same)
            {
                RenderTarget* const own[5] = { color[0], color[1], color[2], color[3], depth };
                RenderTarget* const pass[5] = { s_passColor[0], s_passColor[1], s_passColor[2], s_passColor[3], s_passDepth };
                Rect a = areaWith(own), b = areaWith(pass);
                keeps = a.x0 == b.x0 && a.y0 == b.y0 && a.x1 == b.x1 && a.y1 == b.y1;
                // (The viewport only where x and y are scaled and offset.)
                auto vte = Reg<reg::PA_CL_VTE_CNTL>();
                if (!keeps && untiledPass && (vte.value & 0xF) == 0xF && !Reg<reg::PA_CL_CLIP_CNTL>().clip_disable)
                {
                    float sx = std::fabs(RegF(XE_GPU_REG_PA_CL_VPORT_XSCALE)), ox = RegF(XE_GPU_REG_PA_CL_VPORT_XOFFSET);
                    float sy = std::fabs(RegF(XE_GPU_REG_PA_CL_VPORT_YSCALE)), oy = RegF(XE_GPU_REG_PA_CL_VPORT_YOFFSET);
                    keeps = ox - sx >= float(b.x0) && ox + sx <= float(b.x1) && oy - sy >= float(b.y0) && oy + sy <= float(b.y1);
                }
            }
            if (keeps)
            {
                for (int i = 0; i < 4; i++)
                    if (!color[i] && s_passColor[i])
                    {
                        color[i] = s_passColor[i];
                        adoptedColor |= 1u << i;
                    }
                adoptedDepth = !depth && s_passDepth;
                depth = s_passDepth;
                LogOnce(s_vs->hash ^ (s_ps->hash << 1) ^ 0x2601, "pass merge: vs_%016llx ps_%016llx drawn in the open pass (%s attached)",
                    (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash,
                    adoptedDepth ? (adoptedColor ? "colour and depth" : "depth") : "colour");
            }
        }
        // Before the textures are looked up: a draw into scaled targets
        // samples scaled textures (every attachment shares the pitch, so the scale).
        s_drawScaled = false;
        for (RenderTarget* rt : { color[0], color[1], color[2], color[3], depth })
            if (rt && !rt->scale.IsOne())
                s_drawScaled = true;

        // NFSMW_LOG_TILES=1: register state of the first draws under MSAA (predicated tiling).
        static const bool logTiles = std::getenv("NFSMW_LOG_TILES") != nullptr;
        if (logTiles && surface.msaa_samples != xenos::MsaaSamples::k1X)
        {
            static uint32_t logged = 0;
            static uint32_t lastKey = 0;
            auto wo = Reg<reg::PA_SC_WINDOW_OFFSET>();
            auto wtl = Reg<reg::PA_SC_WINDOW_SCISSOR_TL>();
            auto wbr = Reg<reg::PA_SC_WINDOW_SCISSOR_BR>();
            auto stl = Reg<reg::PA_SC_SCREEN_SCISSOR_TL>();
            auto sbr = Reg<reg::PA_SC_SCREEN_SCISSOR_BR>();
            uint32_t key = wo.value ^ (wtl.value * 3) ^ (wbr.value * 7) ^ (stl.value * 11) ^ (sbr.value * 13);
            if (key != lastKey && logged < 60)
            {
                logged++;
                lastKey = key;
                fprintf(stderr, "[tiles] draw %llu: msaa %u pitch %u wo (%d,%d) woEn %u | window scissor (%u,%u)-(%u,%u) offDis %u | screen scissor (%d,%d)-(%d,%d) | color %08X depth %08X vp y %.1f+%.1f\n",
                    (unsigned long long)s_stats.draws, uint32_t(surface.msaa_samples), pitch, wo.window_x_offset, wo.window_y_offset,
                    uint32_t(Reg<reg::PA_SU_SC_MODE_CNTL>().vtx_window_offset_enable), wtl.tl_x, wtl.tl_y, wbr.br_x, wbr.br_y,
                    wtl.window_offset_disable, stl.tl_x, stl.tl_y, sbr.br_x, sbr.br_y, s_regs[XE_GPU_REG_RB_COLOR_INFO],
                    s_regs[XE_GPU_REG_RB_DEPTH_INFO], RegF(XE_GPU_REG_PA_CL_VPORT_YOFFSET), RegF(XE_GPU_REG_PA_CL_VPORT_YSCALE));
            }
        }

        // Primitive assembly.
        auto modeCntl = Reg<reg::PA_SU_SC_MODE_CNTL>();
        auto dmaSize = RegAt<reg::VGT_DMA_SIZE>(0);
        dmaSize.value = dmaSizeValue;
        IndexSource src{ initiator.source_select == xenos::SourceSelect::kDMA, dmaBase & 0x1FFFFFFF,
            initiator.index_size == xenos::IndexFormat::kInt32, uint32_t(dmaSize.swap_mode) };
        bool restartEnabled = modeCntl.multi_prim_ib_ena && src.indexed;
        uint32_t resetIndex = s_regs[XE_GPU_REG_VGT_MULTI_PRIM_IB_RESET_INDX] & 0xFFFFFF;

        DrawConstants pc{};
        VkPrimitiveTopology topology;
        bool restart = false;
        bool polygonal = true;
        std::vector<uint32_t> indices;  // converted/expanded (empty: plain vkCmdDraw)
        bool useIndices = false;
        bool directIndices = false;     // converted by ConvertIndices into the ring instead
        uint32_t vertexCount = count;
        auto convertAll = [&](bool allowRestart) {
            if ((g_cpOpt & CP_OPT_FAST_INDICES) &&
                uint64_t(src.base & 0x1FFFFFFF) + uint64_t(count) * (src.is32 ? 4 : 2) <= kSharedSize)
            {
                directIndices = true;
                useIndices = true;
                return;
            }
            indices.resize(count);
            for (uint32_t i = 0; i < count; i++)
            {
                uint32_t v = src.Get(i);
                indices[i] = allowRestart && v == resetIndex ? 0xFFFFFFFFu : v;
            }
            useIndices = true;
        };
        // Splits the draw at reset indices and hands each run to `emit`.
        auto forEachRun = [&](auto&& emit) {
            std::vector<uint32_t> run;
            for (uint32_t i = 0; i < count; i++)
            {
                uint32_t v = src.Get(i);
                if (restartEnabled && v == resetIndex)
                {
                    emit(run);
                    run.clear();
                    continue;
                }
                run.push_back(v);
            }
            emit(run);
            useIndices = true;
        };

        switch (initiator.prim_type)
        {
        case xenos::PrimitiveType::kPointList:
            topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST;
            polygonal = false;
            if (src.indexed) convertAll(false);
            break;
        case xenos::PrimitiveType::kLineList:
            topology = VK_PRIMITIVE_TOPOLOGY_LINE_LIST;
            polygonal = false;
            if (src.indexed) convertAll(false);
            break;
        case xenos::PrimitiveType::kLineStrip:
            topology = VK_PRIMITIVE_TOPOLOGY_LINE_STRIP;
            polygonal = false;
            restart = restartEnabled;
            if (src.indexed) convertAll(restart);
            break;
        case xenos::PrimitiveType::kTriangleList:
            topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
            if (src.indexed) convertAll(false);
            break;
        case xenos::PrimitiveType::kTriangleStrip:
            topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP;
            restart = restartEnabled;
            if (src.indexed) convertAll(restart);
            break;
        case xenos::PrimitiveType::kTriangleFan:
        case xenos::PrimitiveType::kPolygon:
            topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
            forEachRun([&](const std::vector<uint32_t>& r) {
                for (size_t k = 2; k < r.size(); k++)
                    indices.insert(indices.end(), { r[0], r[k - 1], r[k] });
            });
            break;
        case xenos::PrimitiveType::kQuadList:
            topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
            forEachRun([&](const std::vector<uint32_t>& r) {
                for (size_t k = 0; k + 3 < r.size(); k += 4)
                    indices.insert(indices.end(), { r[k], r[k + 1], r[k + 2], r[k], r[k + 2], r[k + 3] });
            });
            break;
        case xenos::PrimitiveType::kQuadStrip:
            topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
            forEachRun([&](const std::vector<uint32_t>& r) {
                for (size_t k = 0; k + 3 < r.size(); k += 2)
                    indices.insert(indices.end(), { r[k], r[k + 1], r[k + 3], r[k], r[k + 3], r[k + 2] });
            });
            break;
        case xenos::PrimitiveType::kLineLoop:
            topology = VK_PRIMITIVE_TOPOLOGY_LINE_LIST;
            polygonal = false;
            forEachRun([&](const std::vector<uint32_t>& r) {
                for (size_t k = 0; k + 1 < r.size(); k++)
                    indices.insert(indices.end(), { r[k], r[k + 1] });
                if (r.size() > 2)
                    indices.insert(indices.end(), { r.back(), r[0] });
            });
            break;
        case xenos::PrimitiveType::kRectangleList:
            // Expanded by the vertex shader (4th corner = v1 + v2 - v0).
            topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
            vertexCount = count / 3 * 6;
            pc.indexInfo = 1u << 3;
            if (src.indexed)
            {
                pc.indexAddress = src.base;
                pc.indexInfo |= (src.is32 ? 1u : 0u) | (src.endian << 1);
            }
            break;
        default:
            s_stats.skipped++;
            LogOnce(0x1000 + uint32_t(initiator.prim_type), "primitive type %u not supported",
                uint32_t(initiator.prim_type));
            return;
        }
        if (useIndices)
            vertexCount = directIndices ? count : uint32_t(indices.size());
        // 16-bit indices when 0xFFFF means what the pipeline says: restart
        // with the guest's reset index 0xFFFF, or no restart and no 0xFFFF
        // index (Metal can't turn primitive restart off).
        bool indices16 = false;
        if (directIndices && !src.is32)
        {
            if (restart)
                indices16 = resetIndex == 0xFFFF;
            else
            {
                const uint16_t* p = reinterpret_cast<const uint16_t*>(s_sharedHost + (src.base & 0x1FFFFFFE));
                indices16 = std::find(p, p + count, uint16_t(0xFFFF)) == p + count;  // byte order doesn't matter
            }
        }
        const VkDeviceSize indexBytes = directIndices ? VkDeviceSize(count) * (indices16 ? 2 : 4) : indices.size() * 4;
        if (vertexCount == 0)
            return;

        // Pipeline state.
        const TextureLayout& layout = GetTextureLayout(*s_vs, *s_ps);
        PipelineKey key{};
        key.vs = s_vs->hash;
        key.ps = s_ps->hash;
        key.topology = topology;
        key.restart = restart;
        static const uint32_t blendRegs[4] = { XE_GPU_REG_RB_BLENDCONTROL0, XE_GPU_REG_RB_BLENDCONTROL1,
            XE_GPU_REG_RB_BLENDCONTROL2, XE_GPU_REG_RB_BLENDCONTROL3 };
        for (int i = 0; i < 4; i++)
        {
            key.blend[i] = ~0u;
            if (!color[i])
                continue;
            key.colorFormats[i] = color[i]->vkFormat;
            uint32_t bc = s_regs[blendRegs[i]] & 0x1FFF1FFF;
            // (One, Zero, Add) for both colour and alpha is "blending off".
            if (bc != 0x00010001 && color[i]->blendable && !(adoptedColor & (1u << i)))
                key.blend[i] = bc;
        }
        key.colorMask = writeMask;  // (none for adopted targets)
        if (depth)
        {
            key.depthFormat = depth->vkFormat;
            key.depthControl = adoptedDepth ? 0u : depthControl.value;  // adopted: no depth or stencil test or write
        }
        auto clipCntl = Reg<reg::PA_CL_CLIP_CNTL>();
        bool depthBias = polygonal && (modeCntl.poly_offset_front_enable || modeCntl.poly_offset_para_enable);
        uint32_t cull = 0;
        if (polygonal && initiator.prim_type != xenos::PrimitiveType::kRectangleList)
            cull = (modeCntl.cull_front ? VK_CULL_MODE_FRONT_BIT : 0) | (modeCntl.cull_back ? VK_CULL_MODE_BACK_BIT : 0);
        auto colorControlKey = Reg<reg::RB_COLORCONTROL>();
        // (The alpha test tests the pixel shader's alpha: none depth-only.)
        bool alphaTest = colorControlKey.alpha_test_enable && colorControlKey.alpha_func != xenos::CompareFunction::kAlways &&
            gpu::RunsGuestPixelShader(edramMode, s_depthOnlyGuestPs);
        key.raster = cull | (modeCntl.face ? 4u : 0u) | (clipCntl.clip_disable ? 8u : 0u) | (depthBias ? 16u : 0u) |
            (alphaTest ? 32u : 0u);

        bool pipelinePending;
        VkPipeline pipeline = GetPipeline(key, layout.pipeline, pipelinePending);
        if (pipelinePending)
            return;  // still compiling (counted in OnSwap's log)
        if (pipeline == VK_NULL_HANDLE)
            return skip(5, "pipeline creation failed");

        // Textures (may upload through the ring and so flush: before the
        // constants are allocated).
        if (s_textureCache->on && settings::Generation() != s_textureCache->settingsGeneration)
        {
            s_textureCache->settingsGeneration = settings::Generation();  // the anisotropy setting, in the samplers
            s_textureCache->generation++;
        }
        VkDescriptorImageInfo textureInfos[96];
        for (uint32_t d = 0; d < 3; d++)
            for (uint32_t n = 0; n < 32; n++)
            {
                if (!(((layout.vsMasks[d] | layout.psMasks[d]) >> n) & 1))
                    continue;
                VkDescriptorImageInfo& ii = textureInfos[d * 32 + n];
                ii = { s_sampler, s_dummy[d].view, VK_IMAGE_LAYOUT_GENERAL };
                if (!(s_exp & 8))  // NFSMW_GPU_EXP bit 8: placeholders only (texture cost experiment)
                    ii.imageView = GetTexture(n, d == 0 ? TexDim::k2D : d == 1 ? TexDim::k3D : TexDim::kCube, ii.sampler);
            }

        // Vertex data the guest may reuse before the GPU runs this draw (its
        // fences are published early, see g_eagerWrites): small ranges are
        // copied into the ring now and the shader reads the copy (vmem);
        // larger ones (static meshes, the triple-buffered dynamic buffers)
        // are guarded by the write watch. A range the GPU itself writes
        // first (a resolve into vertex memory) is never copied from the CPU.
        // Vertex data snapshots, copied into the ring with this draw's
        // constants and indices (one allocation: a submit between two would
        // leave the snapshot in the previous submission's slot).
        struct SnapshotRange { uint32_t base, size; };
        SnapshotRange copies[4];
        uint32_t copyCount = 0;
        VkDeviceSize copyBytes = 0;
        if (g_eagerWrites || s_shadowMode)
        {
            constexpr uint32_t kSnapshotMax = 64 * 1024;
            using Range = SnapshotRange;
            for (const Shader* shader : { s_vs, s_ps })
                for (uint32_t word = 0; word < 3; word++)
                    for (uint32_t bits = shader->info.vertexFetchConstants[word]; bits; bits &= bits - 1)
                    {
                        uint32_t constant = word * 32 + uint32_t(__builtin_ctz(bits));
                        const uint32_t* vf = &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + constant * 2];
                        Range r{ vf[0] & 0x1FFFFFFC, ((vf[1] >> 2) & 0xFFFFFF) * 4 };
                        r.size = std::min<uint32_t>(r.size, uint32_t(kSharedSize) - r.base);
                        if (r.size == 0 || std::any_of(copies, copies + copyCount, [&](const Range& c) { return c.base == r.base && c.size == r.size; }))
                            continue;
                        if (g_eagerWrites && r.size <= kSnapshotMax && copyCount < 4 &&
                            !writewatch::GpuWritePending(r.base, r.size) && !writewatch::AnyGpuOwned(r.base, r.size))
                        {
                            copies[copyCount++] = r;
                            copyBytes += (r.size + 255) & ~255u;
                        }
                        else
                        {
                            EnsureShadow(r.base, r.size);
                            GuardGpu(r.base, r.size, false, "vertices");
                        }
                    }
            // Indices the shader fetches itself.
            if (pc.indexAddress)
            {
                EnsureShadow(pc.indexAddress, count << (src.is32 ? 2 : 1));
                GuardGpu(pc.indexAddress, count << (src.is32 ? 2 : 1), false, "indices");
            }
        }

        // NFSMW_LOG_TEXTURE_USE=<hex base>: dump the shaders of draws that
        // sample the texture at that address (once per shader pair).
        if (s_drawUsesLoggedTexture)
        {
            s_drawUsesLoggedTexture = false;
            static std::unordered_set<uint64_t> dumped;
            if (dumped.insert(s_vs->hash ^ (s_ps->hash << 1)).second)
            {
                std::string path = std::format("build/texuse_vs_{:016x}.glsl", s_vs->hash);
                if (FILE* f = fopen(path.c_str(), "w")) { fputs(s_vs->info.glsl.c_str(), f); fclose(f); }
                path = std::format("build/texuse_ps_{:016x}.glsl", s_ps->hash);
                if (FILE* f = fopen(path.c_str(), "w")) { fputs(s_ps->info.glsl.c_str(), f); fclose(f); }
                fprintf(stderr, "[texuse] %08X (%s) sampled by vs %016llx ps %016llx; vertex fetch constants:", s_logTextureUse, s_loggedTextureFetch.c_str(),
                    (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash);
                for (uint32_t word = 0; word < 3; word++)
                    for (uint32_t bits = s_vs->info.vertexFetchConstants[word]; bits; bits &= bits - 1)
                    {
                        uint32_t c = word * 32 + uint32_t(__builtin_ctz(bits));
                        const uint32_t* vf = &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + c * 2];
                        fprintf(stderr, " [%u: %08X %08X]", c, vf[0], vf[1]);
                        uint32_t words = std::min<uint32_t>((vf[1] >> 2) & 0xFFFFFF, 128);
                        for (uint32_t i = 0; i < words; i++)
                        {
                            uint32_t bits = GpuSwap(LoadPhysical((vf[0] & 0x1FFFFFFC) + i * 4), vf[1] & 3);
                            float fv;
                            memcpy(&fv, &bits, 4);
                            fprintf(stderr, "%s%.3f", i % 20 == 0 ? "\n[texuse]   v: " : " ", fv);
                        }
                    }
                fprintf(stderr, "\n");
            }
        }

        // Per-draw data in the ring (may submit when full, so before
        // recording). Constants are uploaded only when the game wrote any
        // constant register since the last upload (g_constantsDirty) or the
        // last copy is in an older submission's slot; one allocation holds
        // both, since a submit between two would recycle the first.
        VkDeviceSize constantsOffset = s_lastConstantsOffset;
        VkDeviceSize indexOffset = 0;
        // A shader with packed constants reads its used ones from slots
        // 0..N-1 of its stage's range, so the upload's layout is the shader's
        // (null: the full file). ~20 vec4s instead of 256 per stage: constants
        // were 9 KB an upload, ~2,500 uploads a frame into the BAR ring.
        const Shader* constLayoutVs = s_vs->info.constPacked ? s_vs : nullptr;
        const Shader* constLayoutPs = s_ps->info.constPacked ? s_ps : nullptr;
        while (true)
        {
            bool uploadConstants = g_constantsDirty.load(std::memory_order_relaxed) ||
                s_lastConstantsSubmission != s_submissionCounter || !s_recording ||
                constLayoutVs != s_lastConstLayoutVs || constLayoutPs != s_lastConstLayoutPs;
            VkDeviceSize need = (uploadConstants ? kConstantsSize : 0) + (useIndices ? 256 + indexBytes : 0) +
                (copyCount ? 256 + copyBytes : 0);
            if (need == 0)
                break;
            uint64_t before = s_submissionCounter;
            VkDeviceSize at = RingAllocate(need);
            if (!uploadConstants && s_submissionCounter != before)
                continue;  // submitted meanwhile: the cached constants are gone
            if (uploadConstants)
            {
                g_constantsDirty.store(false, std::memory_order_relaxed);
                constantsOffset = at;
                uint8_t* c = s_ringHost + at;
                auto copyStage = [&](const Shader* packed, uint32_t base) {
                    const uint32_t* src = &s_regs[XE_GPU_REG_SHADER_CONSTANT_000_X + base * 4];
                    if (!packed)
                    {
                        memcpy(c + base * 16, src, 256 * 16);
                        return;
                    }
                    uint32_t slot = 0;
                    for (uint32_t q = 0; q < 4; q++)
                        for (uint64_t bits = packed->info.constUsed[q]; bits; bits &= bits - 1, slot++)
                            memcpy(c + (base + slot) * 16, src + (q * 64 + uint32_t(__builtin_ctzll(bits))) * 4, 16);
                };
                copyStage(constLayoutVs, 0);    // pc.vsConstBase
                copyStage(constLayoutPs, 256);  // pc.psConstBase
                s_lastConstLayoutVs = constLayoutVs;
                s_lastConstLayoutPs = constLayoutPs;
                memcpy(c + 512 * 16, &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0], 48 * 16);
                memcpy(c + 560 * 16, &s_regs[XE_GPU_REG_SHADER_CONSTANT_BOOL_000_031], 2 * 16);
                memcpy(c + 562 * 16, &s_regs[XE_GPU_REG_SHADER_CONSTANT_LOOP_00], 8 * 16);
                memset(c + 570 * 16, 0, 4 * 16);
                s_lastConstantsOffset = at;
                s_lastConstantsSubmission = s_submissionCounter;
                s_stats.constantUploads++;
            }
            if (useIndices)
            {
                indexOffset = (at + (uploadConstants ? kConstantsSize : 0) + 255) & ~VkDeviceSize(255);
                if (directIndices)
                {
                    ConvertIndices(s_ringHost + indexOffset, src, count, indices16, restart, resetIndex);
                    // NFSMW_CP_OPT_VERIFY=1: compare with the plain conversion.
                    static const bool verify = std::getenv("NFSMW_CP_OPT_VERIFY") != nullptr;
                    if (verify)
                    {
                        static uint64_t checked = 0, bad = 0;
                        for (uint32_t i = 0; i < count; i++)
                        {
                            uint32_t v = src.Get(i);
                            uint32_t want = restart && v == resetIndex ? 0xFFFFFFFFu : v, got;
                            if (indices16)
                            {
                                uint16_t g16;
                                memcpy(&g16, s_ringHost + indexOffset + size_t(i) * 2, 2);
                                got = restart && g16 == 0xFFFF ? 0xFFFFFFFFu : g16;
                            }
                            else
                                memcpy(&got, s_ringHost + indexOffset + size_t(i) * 4, 4);
                            if (got != want && bad++ < 10)
                                fprintf(stderr, "[cpverify] index mismatch at %u: %08X, want %08X (32-bit %d, endian %u, restart %d)\n",
                                    i, got, want, int(src.is32), src.endian, int(restart));
                        }
                        if (++checked % 100000 == 0)
                            fprintf(stderr, "[cpverify] %llu indexed draws checked, %llu mismatches\n", (unsigned long long)checked, (unsigned long long)bad);
                    }
                }
                else
                    memcpy(s_ringHost + indexOffset, indices.data(), indices.size() * 4);
            }
            if (copyCount)
            {
                VkDeviceSize snapshot = (at + (uploadConstants ? kConstantsSize : 0) + (useIndices ? 256 + indexBytes : 0) + 255) & ~VkDeviceSize(255);
                s_stats.snapshotBytes += copyBytes;
                for (uint32_t i = 0; i < copyCount; i++)
                {
                    memcpy(s_ringHost + snapshot, s_sharedHost + copies[i].base, copies[i].size);
                    pc.vfRedirect[i][0] = copies[i].base;
                    pc.vfRedirect[i][1] = copies[i].size;
                    pc.vfRedirect[i][2] = uint32_t(snapshot);
                    snapshot += (copies[i].size + 255) & ~255u;
                }
            }
            break;
        }

        // The attachments' common size (the pass extent) and this draw's
        // scissor, which is also the area the pass must cover (NFSMW_PASS_MERGE
        // repeats it before adopting the open pass's targets: areaWith).
        // Untiled, tile 0 draws the whole frame: the whole target, or with
        // NFSMW_UNTILED_AREA its viewport (UntiledScissor).
        // In guest pixels; the host viewport and scissor are x (scaleX, scaleY)
        // (every attachment of a pass shares the surface pitch, so its scale),
        // the scissor's edges as gpu/render_scale.h maps them (HostRect).
        uint32_t targetW = UINT32_MAX, targetH = UINT32_MAX;
        gpu::RenderScale scale;
        for (RenderTarget* rt : { color[0], color[1], color[2], color[3], depth })
            if (rt)
            {
                targetW = std::min(targetW, rt->guestWidth);
                targetH = std::min(targetH, rt->guestHeight);
                scale = rt->scale;
            }
        float w = float(targetW), h = float(targetH);
        const float scaleX = scale.x.Value(), scaleY = scale.y.Value();
        RenderTarget* const attached[5] = { color[0], color[1], color[2], color[3], depth };
        Rect sc = GuestScissor();
        int untiledWhy = 4;
        if (untiledPass)
            sc = UntiledScissor(attached, int32_t(w), int32_t(h), &untiledWhy);
        int32_t x0 = std::clamp(sc.x0, 0, int32_t(w)), y0 = std::clamp(sc.y0, 0, int32_t(h));
        int32_t x1 = std::clamp(sc.x1, x0, int32_t(w)), y1 = std::clamp(sc.y1, y0, int32_t(h));
        // NFSMW_UNTILED_AREA: a draw whose area its targets' open pass doesn't
        // cover (a whole-target draw after cut ones, another viewport) would
        // restart that pass over the union, a pass more and the union
        // reloaded, which no untiled draw did before (each one's area was the
        // whole target): those targets keep the whole target from now on, as
        // this draw does, so they restart this once. (Checked before a vertex
        // barrier may end the pass: then it gives way a draw early.)
        if (untiledPass && untiledWhy != 4 && x1 > x0 && y1 > y0 && s_passActive && depth == s_passDepth &&
            std::equal(color, color + 4, s_passColor) &&
            !Contains(s_passArea, HostRect(scale, x0, y0, x1, y1)))
        {
            uint64_t key = 0;
            for (RenderTarget* rt : attached)
                if (rt)
                {
                    rt->wholeArea = true;
                    key = key * 0x9E3779B1ull + ((uint64_t(rt->base) << 20) ^ (uint64_t(rt->pitch) << 1) ^ uint64_t(rt->depth));
                }
            const VkRect2D want = HostRect(scale, x0, y0, x1, y1);
            LogOnce((0x33ull << 56) | (key & 0xFFFFFFFFFFFFFFull), "untiled area: frame %llu vs_%016llx ps_%016llx would restart its targets' "
                "pass (area %ux%u@%d,%d) for %ux%u@%d,%d: they keep the whole target from now on", (unsigned long long)s_frame,
                (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash, s_passArea.extent.width, s_passArea.extent.height,
                s_passArea.offset.x, s_passArea.offset.y, want.extent.width, want.extent.height, want.offset.x, want.offset.y);
            if (untiledWhy == 0)
            {
                x0 = y0 = 0;
                x1 = int32_t(w);
                y1 = int32_t(h);
                untiledWhy = 3;
            }
        }
        if (untiledPass && untiledWhy != 4 && s_logUntiledArea)
        {
            if (untiledWhy != 0)
                s_ua.whole++;
            else if (x1 <= x0 || y1 <= y0)
                s_ua.empty++;
            else
                s_ua.cut++;
            if (untiledWhy == 1)
                s_ua.clipOff++;
            else if (untiledWhy == 2)
                s_ua.noViewport++;
            else if (untiledWhy == 3)
                s_ua.wholeArea++;
            if (untiledWhy == 1 || untiledWhy == 2)
                LogOnce((0x30ull << 56) ^ s_vs->hash ^ (s_ps->hash << 1), "untiled area: vs_%016llx ps_%016llx keeps the whole target (%s; VTE %03X, clip control %08X)",
                    (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash, untiledWhy == 1 ? "clipping off" : "no viewport transform",
                    Reg<reg::PA_CL_VTE_CNTL>().value & 0xFFF, Reg<reg::PA_CL_CLIP_CNTL>().value);
            // Each distinct cut per surface (pitch, MSAA, window scissor),
            // once (the map looked up only when it differs from the last).
            if (untiledWhy == 0)
            {
                static uint64_t lastKey = 0;
                auto wtl = Reg<reg::PA_SC_WINDOW_SCISSOR_TL>();
                auto wbr = Reg<reg::PA_SC_WINDOW_SCISSOR_BR>();
                uint64_t key = (0x32ull << 56) ^ (uint64_t(pitch) << 40) ^ (uint64_t(surface.msaa_samples) << 38) ^ (uint64_t(wbr.br_x) << 50) ^
                    (uint64_t(x0) << 26) ^ (uint64_t(y0) << 14) ^ (uint64_t(x1) << 2) ^ (uint64_t(y1) * 0x9E3779B1ull);
                if (key != lastKey)
                    LogOnce(key, "untiled area: pitch %u msaa %u target %dx%d, window scissor (%u,%u)-(%u,%u): draws cut to (%d,%d)-(%d,%d)%s",
                        pitch, uint32_t(surface.msaa_samples), int32_t(w), int32_t(h), wtl.tl_x, wtl.tl_y, wbr.br_x, wbr.br_y, x0, y0, x1, y1,
                        x1 <= x0 || y1 <= y0 ? " (nothing: not drawn)" : "");
                lastKey = key;
            }
        }
        // NFSMW_UNTILED_AREA, for Resolve's check: the untiled draws cut short
        // of the attachments' common size (a smaller attachment cuts the
        // others as before), even to nothing, what they drew and that size.
        // Whole-target draws lose nothing, nor do the targets a draw adopts
        // (never written). Two cuts whose union isn't exact (disjoint
        // viewports: the gap would count as drawn) fall back to the whole
        // target from the next draw.
        if (untiledPass && s_untiledArea)
        {
            const bool cut = untiledWhy == 0 && (x0 > 0 || y0 > 0 || x1 < int32_t(w) || y1 < int32_t(h));
            const VkRect2D drew = HostRect(scale, x0, y0, x1, y1);
            for (int i = 0; i < 5; i++)
                if (RenderTarget* rt = attached[i]; rt && !(i < 4 ? (adoptedColor >> i) & 1 : adoptedDepth))
                {
                    if (rt->areaFrame != s_frame)
                    {
                        rt->areaFrame = s_frame;
                        rt->cut = false;
                    }
                    if (!cut)
                        continue;
                    if (!rt->cut)
                    {
                        rt->cutDrawn = {};
                        rt->cutReach = {};
                    }
                    rt->cut = true;
                    rt->cutReach = { std::max(rt->cutReach.width, scale.x.Size(uint32_t(w))), std::max(rt->cutReach.height, scale.y.Size(uint32_t(h))) };
                    if (x1 <= x0 || y1 <= y0)
                        continue;
                    const VkRect2D& was = rt->cutDrawn;
                    if (was.extent.width && !Contains(was, drew) && !Contains(drew, was) && !Adjoins(was, drew))
                    {
                        rt->wholeArea = true;
                        LogOnce((0x34ull << 56) | (uint64_t(rt->base) << 24) | (uint64_t(rt->pitch) << 1) | uint64_t(rt->depth),
                            "untiled area: frame %llu target b%u p%u%s cut to %ux%u@%d,%d and %ux%u@%d,%d: its draws keep the whole target from now on",
                            (unsigned long long)s_frame, rt->base, rt->pitch, rt->depth ? " (depth)" : "", was.extent.width, was.extent.height,
                            was.offset.x, was.offset.y, drew.extent.width, drew.extent.height, drew.offset.x, drew.offset.y);
                    }
                    rt->cutDrawn = was.extent.width ? Union(was, drew) : drew;
                }
        }
        if (x1 <= x0 || y1 <= y0)
            return;
        VkRect2D scissor = HostRect(scale, x0, y0, x1, y1);

        // Viewport: the guest transform is folded into the vertex shader
        // (pc.posScale) and the host viewport covers the whole target.
        auto vte = Reg<reg::PA_CL_VTE_CNTL>();
        float sx = vte.vport_x_scale_ena ? RegF(XE_GPU_REG_PA_CL_VPORT_XSCALE) : 1.0f;
        float sy = vte.vport_y_scale_ena ? RegF(XE_GPU_REG_PA_CL_VPORT_YSCALE) : 1.0f;
        float sz = vte.vport_z_scale_ena ? RegF(XE_GPU_REG_PA_CL_VPORT_ZSCALE) : 1.0f;
        float ox = vte.vport_x_offset_ena ? RegF(XE_GPU_REG_PA_CL_VPORT_XOFFSET) : 0.0f;
        float oy = vte.vport_y_offset_ena ? RegF(XE_GPU_REG_PA_CL_VPORT_YOFFSET) : 0.0f;
        float oz = vte.vport_z_offset_ena ? RegF(XE_GPU_REG_PA_CL_VPORT_ZOFFSET) : 0.0f;
        if (modeCntl.vtx_window_offset_enable)
        {
            auto wo = Reg<reg::PA_SC_WINDOW_OFFSET>();
            ox += float(wo.window_x_offset);
            oy += float(wo.window_y_offset);
        }
        if (Reg<reg::PA_SU_VTX_CNTL>().pix_center == xenos::PixelCenter::kD3DZero)
        {
            ox += 0.5f;
            oy += 0.5f;
        }
        // A counted draw stops at its viewport, as the console's does: the
        // host clips to the target, and a cube face's 256x256 viewport lies
        // in a 280-pitch target up to 896 rows tall, so a quad past the
        // face's edge would count samples the console never draws. A pixel
        // is in when its centre is. The edges are clamped to the scissor
        // while still floats (garbage in the viewport registers never
        // converts out of range; NaN lands on an edge). Wholly outside, the
        // draw is not drawn and begins no pass; else the pass is begun with
        // the unclamped area (pass decisions are those of any other draw).
        VkRect2D drawScissor = scissor;
        if (zpdCount)
        {
            if ((vte.value & 0xF) == 0xF && !clipCntl.clip_disable)  // (x and y scaled and offset)
            {
                const float fx = float(scaleX), fy = float(scaleY), ax = std::fabs(sx), ay = std::fabs(sy);
                const float vx0 = (ox - ax) * fx, vx1 = (ox + ax) * fx, vy0 = (oy - ay) * fy, vy1 = (oy + ay) * fy;
                const float lx = float(scissor.offset.x), hx = lx + float(scissor.extent.width);
                const float ly = float(scissor.offset.y), hy = ly + float(scissor.extent.height);
                auto edge = [](float v, float lo, float hi) { return int32_t(std::fmax(lo, std::fmin(std::ceil(v - 0.5f), hi))); };
                const int32_t cx0 = edge(vx0, lx, hx), cx1 = edge(vx1, lx, hx), cy0 = edge(vy0, ly, hy), cy1 = edge(vy1, ly, hy);
                if (cx1 <= cx0 || cy1 <= cy0)
                {
                    s_zpdCounts.clamped++;
                    ZpdLog("f%llu draw vs_%016llx ps_%016llx: nothing inside its viewport %.1f..%.1f x %.1f..%.1f (host px): not drawn",
                        (unsigned long long)s_frame, (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash, vx0, vx1, vy0, vy1);
                    return;
                }
                drawScissor = { { cx0, cy0 }, { uint32_t(cx1 - cx0), uint32_t(cy1 - cy0) } };
            }
            else
            {
                s_zpdCounts.unclamped++;
                LogOnce((0x2Aull << 56) | 1, "occlusion query: a counted draw without its viewport transform or with clipping off "
                    "(VTE %03X, clip control %08X): not clamped to its viewport", vte.value & 0xFFF, clipCntl.value);
            }
        }
        if (s_passBarrier >= 2)
            VertexBarrierIfNeeded(layout.vsMasks, pc.indexAddress, pc.indexAddress ? count << (src.is32 ? 2 : 1) : 0);
        BeginPass(color, depth, scissor);
        if (s_profiling && !s_passInfos.empty())
        {
            PassInfo& pi = s_passInfos.back();
            if (pi.draws++ < 3)
                pi.desc += std::format(" [vs_{:016x} ps_{:016x} n{}]", s_vs->hash, s_ps->hash, vertexCount);
        }
        scissor = drawScissor;
        pc.posScale[0] = 2.0f * sx / w;
        pc.posScale[1] = 2.0f * sy / h;
        pc.posScale[2] = 2.0f * ox / w - 1.0f;
        pc.posScale[3] = 2.0f * oy / h - 1.0f;
        pc.flags = 1u | (vte.vtx_w0_fmt ? 2u : 0u) | (vte.vtx_xy_fmt ? 4u : 0u) | (vte.vtx_z_fmt ? 8u : 0u);
        // Ultrawide (game/ultrawide): the 16:9 guest frame is stretched to the
        // screen and the front-end (HUD and menu) matrix is narrowed by s to
        // match. Full-screen front-end layers (fades to black, the dims behind
        // dialogs and menus, the loading screen's backdrop and band) are
        // authored ~29% past the 16:9 edges; narrowed by s (0.744 at 3440x1440)
        // they stopped ~1.8% short of each screen edge and the world showed
        // through. A front-end draw whose every primitive spans the whole 16:9
        // width keeps its 16:9 extent (still past both screen edges).
        // Vert+ (a screen narrower than 16:9) is the same in y: the front-end
        // matrix's y scale is narrowed by sv (0.75 on a 4:3 iPad), so a layer
        // that ended at the 16:9 frame's top and bottom ends at the 16:9
        // band's. One that spans the whole width (as above) and reaches both
        // band edges is stretched in y to the screen's top and bottom; the
        // loading band, short of them, keeps its height. Any other layer that
        // covers the whole band (a menu backdrop picture, made of one quad or
        // several) is scaled up in x and y alike about the centre until it
        // reaches the screen's top and bottom, its sides cropped ("cover"):
        // stretched in y alone it would be distorted by up to 1/sv. The rest
        // of the front end is clipped to the band (below).
        // Edge bars (gpu/edge_bars.h): the copies of a police vid-cam side
        // bar or the intro's black frame that fill the screen past the 16:9
        // band, drawn right after it; and the black around a full-band
        // backdrop (the results screen), cleared right after it, at 16:9 too.
        // NFSMW_BAR_FILL=0 turns all the edge fills off (an A/B switch).
        // planEdgeFills: the plan for a front-end draw (its registers, what
        // the rules below did, a reader for its ePolys): with `bars`,
        // PlanFill's bars and black frames first (wide and tall screens
        // only), then PlanBackdrop's full-band picture (every screen shape).
        // logEdgeFill: what a plan does, once per kind, screen shape and
        // outcome, in every log (a tester's too).
        static const bool barFill = [] { const char* v = std::getenv("NFSMW_BAR_FILL"); return !(v && v[0] == '0'); }();
        auto planEdgeFills = [&](const float* c, float feY, float aspectX, float aspectY, bool feShader, bool leftAlone, bool bars) {
            xenos::xe_gpu_texture_fetch_t t0;
            memcpy(&t0, &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0], sizeof(t0));
            const gpu::bars::Draw draw{ .prim = uint32_t(initiator.prim_type),
                .count = count,
                .c = c,
                .frontEndY = feY,
                .scaleX = aspectX,
                .scaleY = aspectY,
                .frontEndShader = feShader,
                .leftAlone = leftAlone,
                .occlusion = zpdCount,
                .vte = vte.value,
                .blend0 = s_regs[XE_GPU_REG_RB_BLENDCONTROL0] & 0x1FFF1FFF,
                .t0Sampled = (s_ps->info.texture2DMask & 1) != 0,
                .t0Width = uint32_t(t0.size_2d.width) + 1,
                .t0Height = uint32_t(t0.size_2d.height) + 1,
                .posScale = { pc.posScale[0], pc.posScale[1], pc.posScale[2], pc.posScale[3] },
                .hostW = w * float(scaleX),
                .hostH = h * float(scaleY),
                .scissor = { scissor.offset.x, scissor.offset.y, scissor.offset.x + int32_t(scissor.extent.width),
                    scissor.offset.y + int32_t(scissor.extent.height) } };
            const uint32_t* vf = &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + 95 * 2];
            const uint32_t base = vf[0] & 0x1FFFFFFC, endian = vf[1] & 3;
            const uint32_t words = (vf[1] >> 2) & 0xFFFFFF, first = s_regs[XE_GPU_REG_VGT_INDX_OFFSET];
            auto read = [&](uint32_t k, uint32_t j, uint32_t& word) {
                const uint32_t v = src.Get(k) + first;  // an ePoly: 8 words
                if ((v + 1) * 8 > words)
                    return false;
                word = GpuSwap(LoadPhysical(base + (v * 8 + j) * 4), endian);
                return true;
            };
            gpu::bars::Plan plan = bars ? gpu::bars::PlanFill(draw, read) : gpu::bars::Plan{};
            // The results screen's backdrop (and any opaque picture of that
            // shape: the title's key art): what lies outside the picture,
            // past the band and in the band's overscan rows above and below
            // it, is cleared to black after it, where the world drawn behind
            // it showed (gpu/edge_bars.h).
            if (plan.fill == gpu::bars::Fill::None)
                plan = gpu::bars::PlanBackdrop(draw, read);
            return plan;
        };
        auto logEdgeFill = [&](const gpu::bars::Plan& barPlan, float aspectX, float aspectY) {
            if (barPlan.fill == gpu::bars::Fill::None)
                return;
            static const bool logFrontEnd = std::getenv("NFSMW_LOG_FRONTEND") != nullptr;
            // A match each frame skips building the line once its key is
            // logged.
            const uint32_t kind = uint32_t(barPlan.fill);
            const bool blackout = barPlan.fill == gpu::bars::Fill::Blackout;
            const bool backdrop = barPlan.fill == gpu::bars::Fill::Backdrop;
            const gpu::bars::Box& box = barPlan.box;
            const uint64_t key = (0x36ull << 56) | (uint64_t(kind) << 40) | (uint64_t(barPlan.rectCount) << 36) |
                (uint64_t(std::lround(aspectX * 4096.0f)) << 18) | uint64_t(std::lround(aspectY * 4096.0f));
            static uint64_t loggedKey[5] = { ~0ull, ~0ull, ~0ull, ~0ull, ~0ull };
            if (loggedKey[kind] != key)
            {
                loggedKey[kind] = key;
                char fills[128];
                if (barPlan.rectCount)
                    snprintf(fills, sizeof(fills), "fills %u rectangle(s) %s, first [%d,%d)x[%d,%d) (host px)", barPlan.rectCount,
                        backdrop ? "outside it (past the 16:9 band, and the band's overscan rows)" : "past the 16:9 band", barPlan.rects[0].x0,
                        barPlan.rects[0].x1, barPlan.rects[0].y0, barPlan.rects[0].y1);
                else
                    snprintf(fills, sizeof(fills), "%s", blackout || backdrop ? "fills nothing" : "reaches the screen edge: nothing to fill");
                LogOnce(key,
                    "edge bars: frame %llu vs_%016llx ps_%016llx: %s (clip x %.3f..%.3f y %.3f..%.3f, colour %08X; view scale x %.4f y %.4f) %s",
                    (unsigned long long)s_frame, (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash,
                    backdrop ? "a full-band backdrop (black outside it)" : blackout ? "a black frame"
                        : kind == uint32_t(gpu::bars::Fill::RightBar) ? "the right vid-cam bar" : "the left vid-cam bar",
                    box.x0, box.x1, box.y0, box.y1, barPlan.argb, aspectX, aspectY, fills);
            }
            // NFSMW_LOG_FRONTEND=1: the first frame of each run of frames
            // with a match, bars and black frames apart (one run per vid-cam
            // scene and per cut; any other is a draw this should not take).
            static uint64_t lastBarFrame[3] = { ~0ull - 1, ~0ull - 1, ~0ull - 1 };
            uint64_t& last = lastBarFrame[backdrop ? 2 : blackout ? 1 : 0];
            if (logFrontEnd && s_frame != last && s_frame != last + 1)
            {
                xenos::xe_gpu_texture_fetch_t t0;
                memcpy(&t0, &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0], sizeof(t0));
                fprintf(stderr, "[frontend] edge bars: matching from frame %llu: %s x %.3f..%.3f y %.3f..%.3f colour %08X blend %08X t0 %08X %ux%u, %u rect(s)\n",
                    (unsigned long long)s_frame, backdrop ? "backdrop" : blackout ? "black frame" : kind == uint32_t(gpu::bars::Fill::RightBar) ? "right bar" : "left bar",
                    box.x0, box.x1, box.y0, box.y1, barPlan.argb, s_regs[XE_GPU_REG_RB_BLENDCONTROL0] & 0x1FFF1FFF,
                    (uint32_t(t0.base_address) << 12) & 0x1FFFFFFFu, uint32_t(t0.size_2d.width) + 1, uint32_t(t0.size_2d.height) + 1,
                    barPlan.rectCount);
            }
            last = s_frame;
        };
        gpu::bars::Plan barPlan;
        if (const float s = game::AspectScaleX(), sv = game::AspectScaleY(); s < 1.0f || sv < 1.0f)
        {
            if (const float feX = game::FrontEndScaleX(); feX > 0.0f)
            {
                // A front-end draw: c0..c3 hold the front-end matrix (x scale
                // as the hook left it, no rotation, no projection), and its
                // vertices are ePolys (32 bytes: x, y, z, colour, uv0, uv1;
                // written at 0x82445DC8) at vertex fetch constant 95.
                float c[16];
                memcpy(c, &s_regs[XE_GPU_REG_SHADER_CONSTANT_000_X], sizeof(c));
                const uint32_t* vf = &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + 95 * 2];
                if (std::fabs(c[0] - feX) <= feX * 1e-5f && c[1] == 0.0f && c[2] == 0.0f &&
                    c[12] == 0.0f && c[13] == 0.0f && c[14] == 0.0f && c[15] == 1.0f && ((vf[0] & 3) == 3))
                {
                    // NFSMW_LOG_FRONTEND=1: each distinct full-width layer,
                    // and in Vert+ each draw that covers the band, sits at
                    // one of its edges or reaches past it (below), once, with
                    // its extents in the frame's clip space before any
                    // widening (to check the thresholds); 4096 lines at most.
                    static const bool logFrontEnd = std::getenv("NFSMW_LOG_FRONTEND") != nullptr;
                    // At most 32 lines per shader pair (an animated draw makes
                    // a new line each frame), so a long session still logs
                    // the screen it was played to.
                    auto logOnce = [](uint64_t vs, uint64_t ps, const std::string& line) {
                        static std::unordered_set<std::string> logged;
                        static std::unordered_map<std::string, uint32_t> perPair;
                        if (logged.size() >= 4096 || !logged.insert(line).second)
                            return;
                        uint32_t n = ++perPair[std::format("{:016x}{:016x}", vs, ps)];
                        if (n <= 32)
                            fprintf(stderr, "[frontend] %s\n", line.c_str());
                        else if (n == 33)
                            fprintf(stderr, "[frontend] vs_%016llx ps_%016llx: 32 lines, no more\n", (unsigned long long)vs, (unsigned long long)ps);
                    };
                    // c4..c7 the y column: 0, the y scale as the hook left it
                    // (its sign unverified), 0, the y translation.
                    const float feY = game::FrontEndScaleY();
                    // Vert+ acts on every draw it matches (Hor+ only on
                    // full-width layers): the vertex shader must read the
                    // matrix (registers c0..c3: the 16 floats above) and the
                    // ePolys (fetch constant 95), not just inherit them from
                    // the HUD drawn before it (the mirror's picture, a movie
                    // quad, a post-process pass).
                    const auto& vi = s_vs->info;
                    const bool feShader = ((vi.vertexFetchConstants[95 / 32] >> (95 % 32)) & 1) &&
                        (vi.constRelative || (vi.constUsed[0] & 0xFull) == 0xFull);
                    const bool tall = feShader && sv < 1.0f && feY > 0.0f && c[4] == 0.0f && c[6] == 0.0f &&
                        std::fabs(std::fabs(c[5]) - feY) <= feY * 1e-5f;
                    const bool textured = (s_ps->info.texture2DMask | s_ps->info.texture3DMask | s_ps->info.textureCubeMask) != 0;
                    // Quad, triangle and rectangle lists are measured: up to
                    // 16 quads to act on (layers are a few), more for the log
                    // (which says so). Hor+ stops at the first primitive short
                    // of the 16:9 width; Vert+ needs them all (the band's
                    // cover is their union).
                    const uint32_t prim = uint32_t(initiator.prim_type);
                    const uint32_t per = prim == 13 ? 4 : (prim == 4 || prim == 8) ? 3 : 0;
                    const bool small = count <= 64;
                    const bool scan = per && count % per == 0 && (small || (logFrontEnd && count <= 4096));
                    const bool measureY = tall || logFrontEnd;
                    bool valid = scan, fullWidth = scan, fullHeight = scan && tall;
                    float reach = 1e30f, reachY = 1e30f;  // the nearest screen edge any primitive reaches
                    float allLo = 1e30f, allHi = -1e30f, allLoY = 1e30f, allHiY = -1e30f;
                    // The primitives' areas inside the band (x +-1, s being 1
                    // in Vert+; y +-sv), in clip units: a quad's or rectangle's
                    // box, half a triangle's (a quad's half). The first eight
                    // primitives' extents for the log.
                    float bandArea = 0.0f, minPrimArea = 1e30f;
                    // The band this draw shows on a 16:9 frame: clip y of
                    // its translation's share outside the narrowing, +-sv.
                    const float bandMid = c[7] * (1.0f - sv), bandLo = bandMid - sv, bandHi = bandMid + sv;
                    bool outside = false;  // a primitive wholly above or below the band
                    uint32_t prims = 0;
                    std::string primList;
                    // NFSMW_LOG_FRONTEND=1: up to four primitives that cross
                    // an edge of the 16:9 band (past it by more than half a
                    // percent of the band) and stop short of the screen's
                    // edge, the shape that leaves a strip of the world (the
                    // vid cam's mask did): logged below as "edge candidate"
                    // with what the rules here did to them. And ("inset
                    // candidate") one centred in the band, covering most of
                    // it in x and y and stopping short of its edges in one
                    // or both, the shape of the intro's black frame between
                    // two shots, which the world showed all round.
                    struct EdgeHit
                    {
                        uint32_t prim;
                        const char* edge;  // "+y", "-y", "+x" or "-x" (clip space), or "in" (inset)
                        float lo, hi, past, end;  // band units along that axis; the outer end in clip units
                                                  // (inset: band x, then its reach in clip x and y)
                        float acrossLo, acrossHi;  // band units along the other axis (inset: band y)
                        uint32_t colours[4];
                        uint32_t colourCount;
                    };
                    EdgeHit edgeHits[4];
                    uint32_t edgeHitCount = 0;
                    const float bandMidX = c[3] * (1.0f - s), bandLoX = bandMidX - s, bandHiX = bandMidX + s;
                    if (scan)
                    {
                        const uint32_t base = vf[0] & 0x1FFFFFFC, endian = vf[1] & 3;
                        const uint32_t words = (vf[1] >> 2) & 0xFFFFFF, first = s_regs[XE_GPU_REG_VGT_INDX_OFFSET];
                        for (uint32_t v0 = 0; v0 < count && valid && (fullWidth || measureY); v0 += per)
                        {
                            float lo = 1e30f, hi = -1e30f, loY = 1e30f, hiY = -1e30f;
                            for (uint32_t k = 0; k < per; k++)
                            {
                                uint32_t v = src.Get(v0 + k) + first;
                                if ((v + 1) * 8 > words)
                                {
                                    valid = false;
                                    break;
                                }
                                float p[3];
                                for (uint32_t j = 0; j < 3; j++)
                                {
                                    uint32_t b = GpuSwap(LoadPhysical(base + (v * 8 + j) * 4), endian);
                                    memcpy(&p[j], &b, 4);
                                }
                                float x = p[0] * c[0] + p[1] * c[1] + p[2] * c[2] + c[3];
                                lo = std::min(lo, x);
                                hi = std::max(hi, x);
                                if (measureY)
                                {
                                    float y = p[0] * c[4] + p[1] * c[5] + p[2] * c[6] + c[7];
                                    loY = std::min(loY, y);
                                    hiY = std::max(hiY, y);
                                }
                            }
                            if (!valid)
                                break;
                            // Only layers authored well past both 16:9 edges (the
                            // fades, dims and loading band at +-1.29): the title /
                            // Controls key art (+-1.105) keeps its shape.
                            fullWidth = fullWidth && lo <= -1.2f * s && hi >= 1.2f * s;
                            reach = std::min(reach, std::min(-lo, hi));
                            // Vert+: and down to both edges of the 16:9 band.
                            fullHeight = fullHeight && loY <= -0.99f * sv && hiY >= 0.99f * sv;
                            reachY = std::min(reachY, std::min(-loY, hiY));
                            allLo = std::min(allLo, lo);
                            allHi = std::max(allHi, hi);
                            allLoY = std::min(allLoY, loY);
                            allHiY = std::max(allHiY, hiY);
                            const float primArea = std::max(0.0f, std::min(hi, 1.0f) - std::max(lo, -1.0f)) *
                                std::max(0.0f, std::min(hiY, sv) - std::max(loY, -sv)) * (prim == 4 ? 0.5f : 1.0f);
                            bandArea += primArea;
                            minPrimArea = std::min(minPrimArea, primArea);
                            outside = outside || (measureY && (hiY < bandLo || loY > bandHi));
                            if (logFrontEnd && prims < 8)
                                primList += std::format(" [{:.3f}..{:.3f} {:.3f}..{:.3f}]", lo, hi, loY, hiY);
                            // (The front-end shader's draws only: one that
                            // inherits the constants reads no ePolys.)
                            if (logFrontEnd && feShader && edgeHitCount < 4)
                            {
                                auto hit = [&](const char* edge, float a, float b, float mid, float scale, float past, float end) {
                                    if (edgeHitCount >= 4)
                                        return;
                                    EdgeHit& e = edgeHits[edgeHitCount++];
                                    const bool y = edge[1] == 'y';
                                    const float c0 = y ? lo : loY, c1 = y ? hi : hiY, cMid = y ? bandMidX : bandMid, cScale = y ? s : sv;
                                    e = { prims, edge, (a - mid) / scale, (b - mid) / scale, past / scale, end, (c0 - cMid) / cScale,
                                        (c1 - cMid) / cScale, {}, 0 };
                                    for (uint32_t k = 0; k < per && k < 4; k++)
                                    {
                                        const uint32_t v = src.Get(v0 + k) + first;  // (in range: read above)
                                        e.colours[e.colourCount++] = GpuSwap(LoadPhysical(base + (v * 8 + 3) * 4), endian);
                                    }
                                };
                                const float my = 0.005f * sv, mx = 0.005f * s;
                                if (sv < 1.0f)
                                {
                                    if (hiY > bandHi + my && hiY < 1.0f && loY < bandHi)
                                        hit("+y", loY, hiY, bandMid, sv, hiY - bandHi, hiY);
                                    if (loY < bandLo - my && loY > -1.0f && hiY > bandLo)
                                        hit("-y", loY, hiY, bandMid, sv, bandLo - loY, loY);
                                }
                                if (s < 1.0f)
                                {
                                    if (hi > bandHiX + mx && hi < 1.0f && lo < bandHiX)
                                        hit("+x", lo, hi, bandMidX, s, hi - bandHiX, hi);
                                    if (lo < bandLoX - mx && lo > -1.0f && hi > bandLoX)
                                        hit("-x", lo, hi, bandMidX, s, bandLoX - lo, lo);
                                }
                                // Inset: centred within 2% of the band,
                                // reaching 80% of it or more each way, short
                                // of its edge (99%) in x or y.
                                const float xl = (lo - bandMidX) / s, xh = (hi - bandMidX) / s, yl = (loY - bandMid) / sv, yh = (hiY - bandMid) / sv;
                                const float bandReachX = std::min(-xl, xh), bandReachY = std::min(-yl, yh);
                                if (edgeHitCount < 4 && std::fabs(xl + xh) <= 0.04f && std::fabs(yl + yh) <= 0.04f && bandReachX >= 0.8f &&
                                    bandReachY >= 0.8f && std::min(bandReachX, bandReachY) < 0.99f)
                                {
                                    EdgeHit& e = edgeHits[edgeHitCount++];
                                    e = { prims, "in", xl, xh, std::min(-lo, hi), std::min(-loY, hiY), yl, yh, {}, 0 };
                                    for (uint32_t k = 0; k < per && k < 4; k++)
                                    {
                                        const uint32_t v = src.Get(v0 + k) + first;  // (in range: read above)
                                        e.colours[e.colourCount++] = GpuSwap(LoadPhysical(base + (v * 8 + 3) * 4), endian);
                                    }
                                }
                            }
                            prims++;
                        }
                        fullWidth = fullWidth && valid;
                        fullHeight = fullHeight && valid;
                    }
                    // Saved before any widening: the band below is this
                    // draw's own transform of clip y = +-sv.
                    const float ps1 = pc.posScale[1], ps3 = pc.posScale[3];
                    float kX = 1.0f, kY = 1.0f, kCover = 1.0f;
                    if (s < 1.0f && small && fullWidth && reach < 1.0f)
                    {
                        // Just past the screen edges, not the full 1/s: a
                        // textured layer (the band's stripes) stays ~5% off.
                        kX = std::min(1.0f / s, 1.01f / reach);
                        pc.posScale[0] *= kX;
                        pc.posScale[2] *= kX;
                    }
                    // A flat full-screen layer (a fade or dim) is stretched in
                    // y alone; a textured one (a backdrop picture) is not, it
                    // would be distorted: it is a cover below.
                    const bool extended = sv < 1.0f && small && fullWidth && fullHeight && !textured;
                    if (extended && reachY < 1.0f)
                    {
                        // posScale.y alone: NDC y = y * posScale.y +
                        // posScale.w * w scales about posScale.w, the
                        // viewport's centre, also with a tile's window offset
                        // (NFSMW_TILING=1).
                        kY = std::min(1.0f / sv, 1.01f / reachY);
                        pc.posScale[1] *= kY;
                    }
                    // Cover: the draw's primitives together reach both band
                    // edges in x and y (within 1%) and fill 95% of its area
                    // (corner pieces, a frame's four sides or a list's rows
                    // don't). Scaled by up to 1/sv (just past the screen's
                    // top and bottom, as the fades), the same in x so it keeps
                    // its shape; flat ones too (nothing to distort, and they
                    // must not be clipped: the strips would show undimmed).
                    const bool coversBand = allLo <= -0.99f && allHi >= 0.99f && allLoY <= -0.99f * sv && allHiY >= 0.99f * sv;
                    // And nothing else in the batch: the primitives' areas add
                    // up to the band's (no overlap, no extra sprite drawn with
                    // the same texture) and each is a sixteenth of it at
                    // least (a backdrop in up to 4x4 tiles), so a dialog's
                    // pieces batched with a dim are never zoomed.
                    const bool cover = tall && small && valid && !extended && coversBand && bandArea >= 0.95f * 4.0f * sv &&
                        bandArea <= 1.05f * 4.0f * sv && minPrimArea >= 4.0f * sv / 16.0f;
                    if (const float reachAllY = std::min(-allLoY, allHiY); cover && reachAllY < 1.0f)
                    {
                        // posScale.x and .y alone: about the viewport's centre
                        // (clip 0,0), as above.
                        kCover = std::min(1.0f / sv, 1.01f / reachAllY);
                        pc.posScale[0] *= kCover;
                        pc.posScale[1] *= kCover;
                    }
                    // Vert+: everything else is clipped to the 16:9 band, the
                    // rows a 16:9 frame shows. Menus park elements just past
                    // the frame's top and bottom (clip y beyond +-1, never on
                    // a 16:9 screen), and the narrowed matrix brings them into
                    // the strips above and below the band (the autosave
                    // dialog's white box and warning triangle, top right).
                    // The band is clip y = +-sv through this draw's viewport
                    // transform, in the target's host pixels like the
                    // scissor: posScale covers the render scale (the host
                    // viewport is the target's size x scale) and a tile's
                    // window offset (NFSMW_TILING=1: posScale.w moves with
                    // it). A row is in it when its centre is, as for a quad
                    // that ends there (the edges snapped to 1/256 pixel, as
                    // the rasteriser snaps vertices, so float noise doesn't
                    // decide a tie): at 1x, 720 rows and sv 0.75, rows
                    // 90..629.
                    // A measured draw whose every primitive meets the band
                    // isn't clipped: a bar or gradient authored past the
                    // frame's edge (as the fades) reaches into the strips as
                    // before. One with a primitive wholly outside (parked)
                    // is, and so is one too big to measure (text batches).
                    const bool meetsBand = small && scan && valid && !outside;
                    bool clipped = false;
                    if (tall && !extended && !cover && !meetsBand && vte.vport_y_scale_ena && vte.vport_y_offset_ena)
                    {
                        // (The rows the edge-bar fills leave, by the same rule.)
                        const gpu::bars::Rows band = gpu::bars::BandRows(bandLo, bandHi, ps1, ps3, h * float(scaleY));
                        const int32_t top = std::max(scissor.offset.y, band.top);
                        const int32_t bottom = std::min(scissor.offset.y + int32_t(scissor.extent.height), band.bottom);
                        scissor.offset.y = top;
                        scissor.extent.height = uint32_t(std::max(bottom - top, 0));
                        clipped = true;
                    }
                    // The police vid-cam's mask (career intro) ends with two
                    // solid black side bars that reach ~3% past the 16:9
                    // frame's top and bottom: under Vert+ the world shows in
                    // the rows above and below them (and the CRT bezel's
                    // quadrants, which stop at the same place), under Hor+
                    // past a 2.50:1 screen beside them. The intro's cut
                    // between two shots (~95 s) is a few frames of one black
                    // quad 0.970 x 0.939 of the 16:9 frame: the world shows
                    // all round it. Such a quad (gpu::bars::PlanFill: one
                    // black rectangle of the front-end shader the rules
                    // above left alone; a bar a little past both band edges
                    // in y, from inside the band to well past one side; a
                    // black frame centred, short of the band's edges) is
                    // drawn again after itself, stretched over the frame and
                    // scissored to (its half of) what lies outside the band:
                    // nothing inside the band changes.
                    // NFSMW_BAR_FILL=0 turns it off (an A/B switch: barFill).
                    const uint32_t blend0 = s_regs[XE_GPU_REG_RB_BLENDCONTROL0] & 0x1FFF1FFF;
                    if (barFill)
                    {
                        barPlan = planEdgeFills(c, feY, s, sv, feShader, kX == 1.0f && kY == 1.0f && kCover == 1.0f && !clipped, true);
                        logEdgeFill(barPlan, s, sv);
                    }
                    // The edge candidates (above) that still stop short of
                    // the screen's edge after the rules, each distinct line
                    // once, at most 4 per (t0, edge, kind) and 256 per run:
                    // a sliding or fading element makes a new line each
                    // frame, so the frame number is not part of the line.
                    // Their own cap, not the per-shader-pair one (every
                    // front-end draw shares one shader pair, and the title
                    // screen and menus use it up).
                    for (uint32_t i = 0; i < edgeHitCount; i++)
                    {
                        static std::unordered_set<std::string> logged;
                        static std::unordered_map<uint64_t, uint32_t> perKey;
                        static uint32_t lines = 0;
                        if (lines > 256)
                            break;
                        const EdgeHit& e = edgeHits[i];
                        const bool inset = e.edge[0] == 'i';
                        // One the rules above take past the screen's edge
                        // (widened, stretched or zoomed about the centre: the
                        // fades, dims and loading band) leaves no strip; an
                        // inset one, none when they take it past both.
                        if (inset ? e.past * kX * kCover >= 1.0f && e.end * kY * kCover >= 1.0f
                                  : std::fabs(e.end * (e.edge[1] == 'x' ? kX : kY) * kCover) >= 1.0f)
                            continue;
                        auto fetch = [](uint32_t n) {
                            xenos::xe_gpu_texture_fetch_t f;
                            memcpy(&f, &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + n * 6], sizeof(f));
                            return f;
                        };
                        const xenos::xe_gpu_texture_fetch_t f0 = fetch(0), f1 = fetch(1);
                        const uint32_t t0 = (s_ps->info.texture2DMask & 1) ? (uint32_t(f0.base_address) << 12) & 0x1FFFFFFFu : 0;
                        const uint32_t t1 = (s_ps->info.texture2DMask & 2) ? (uint32_t(f1.base_address) << 12) & 0x1FFFFFFFu : 0;
                        bool same = true;
                        uint32_t minAlpha = 0xFF;
                        for (uint32_t k = 0; k < e.colourCount; k++)
                        {
                            same = same && e.colours[k] == e.colours[0];
                            minAlpha = std::min(minAlpha, e.colours[k] >> 24);
                        }
                        // At most 4 lines per (t0, edge, kind: opaque black,
                        // opaque, translucent): checked before the line is
                        // made, so a capped element costs nothing more. (By
                        // kind too: the game's white texture is under every
                        // flat panel, and a fading glow would use them up.)
                        const uint64_t kind = (minAlpha >= 0xF0 ? 1u : 0u) | (same && (e.colours[0] & 0xFFFFFFu) == 0 ? 2u : 0u);
                        const uint64_t edgeCode = inset ? 4u : (uint64_t(e.edge[0] == '+') << 1) | uint64_t(e.edge[1] == 'x');
                        uint32_t& perT0Edge = perKey[(uint64_t(t0) << 32) | (kind << 3) | edgeCode];
                        if (perT0Edge >= 4)
                            continue;
                        const std::string action = barPlan.rectCount
                            ? std::string(barPlan.fill == gpu::bars::Fill::Backdrop ? "backdrop fill"
                                    : barPlan.fill == gpu::bars::Fill::Blackout ? "black frame fill" : "bar fill")
                            : clipped ? std::format("clipped to rows {}..{}", scissor.offset.y, scissor.offset.y + int32_t(scissor.extent.height))
                            : kCover != 1.0f ? std::format("cover x{:.3f}", kCover)
                            : (kX != 1.0f || kY != 1.0f) ? std::format("widened x{:.3f} y{:.3f}", kX, kY)
                            : std::string("left alone");
                        const std::string textures = std::format("t0 {} t1 {}",
                            t0 ? std::format("{:08X} {}x{} f{}", t0, uint32_t(f0.size_2d.width) + 1, uint32_t(f0.size_2d.height) + 1, uint32_t(f0.format)) : std::string("-"),
                            t1 ? std::format("{:08X}", t1) : std::string("-"));
                        const std::string colour = same ? std::format("{:08X}", e.colours[0]) : std::string("varies");
                        const std::string line = inset
                            ? std::format("vs_{:016x} ps_{:016x} n{} prim {}/{} {}: band x {:.3f}..{:.3f} y {:.3f}..{:.3f}, short of the band's edges by "
                                "{:.3f} in x and {:.3f} in y, reaches clip x {:.3f} y {:.3f}; colour {} alpha {:02X} blend {:08X}: {}",
                                s_vs->hash, s_ps->hash, count, e.prim, prims, textures, e.lo, e.hi, e.acrossLo, e.acrossHi,
                                1.0f - std::min(-e.lo, e.hi), 1.0f - std::min(-e.acrossLo, e.acrossHi), e.past, e.end, colour, minAlpha, blend0, action)
                            : std::format("vs_{:016x} ps_{:016x} n{} prim {}/{} {} {}: band {:.3f}..{:.3f} ({} {:.3f}..{:.3f}), "
                                "{:.3f} past its edge, ends at clip {:.3f} (short of the screen edge); colour {} alpha {:02X} blend {:08X}: {}",
                                s_vs->hash, s_ps->hash, count, e.prim, prims, textures, e.edge, e.lo, e.hi, e.edge[1] == 'y' ? "x" : "y", e.acrossLo,
                                e.acrossHi, e.past, e.end, colour, minAlpha, blend0, action);
                        if (!logged.insert(line).second)
                            continue;
                        perT0Edge++;
                        if (++lines > 256)
                        {
                            fprintf(stderr, "[frontend] edge candidate: cap reached (256 lines), no more\n");
                            break;
                        }
                        fprintf(stderr, "[frontend] %s candidate: frame %llu %s\n", inset ? "inset" : "edge", (unsigned long long)s_frame, line.c_str());
                    }
                    if (logFrontEnd && fullWidth)
                    {
                        logOnce(s_vs->hash, s_ps->hash, std::format("vs_{:016x} ps_{:016x} n{} x {:.3f}..{:.3f} y {:.3f}..{:.3f} c5 {:.4f} (front end y {:.4f}{}{}) "
                            "view scale x {:.4f} y {:.4f}: widened x{:.3f} y{:.3f}{}",
                            s_vs->hash, s_ps->hash, count, allLo, allHi, allLoY, allHiY, c[5], feY, tall ? "" : ", not matched",
                            textured ? ", textured" : "", s, sv, kX, kY, small ? "" : " (n > 64: not acted on)"));
                    }
                    if (logFrontEnd && sv < 1.0f)
                    {
                        // The union's share of the band's area, and the
                        // primitives' summed (overlaps count twice). Logged:
                        // a draw that covers the band, one of a tenth of it
                        // at either edge (a backdrop in tiles drawn one by
                        // one, a header or button strip), and what is clipped
                        // because it reaches past the band.
                        const float boxShare = std::max(0.0f, std::min(allHi, 1.0f) - std::max(allLo, -1.0f)) *
                            std::max(0.0f, std::min(allHiY, sv) - std::max(allLoY, -sv)) / (4.0f * sv);
                        const bool atPlus = allHiY >= 0.99f * sv, atMinus = allLoY <= -0.99f * sv;
                        const bool pastBand = allLoY < -1.001f * sv || allHiY > 1.001f * sv;
                        const std::string action = cover ? std::format("cover x{:.3f}", kCover)
                            : extended ? std::format("widened y{:.3f}", kY)
                            : clipped ? std::format("clipped to rows {}..{}", scissor.offset.y, scissor.offset.y + int32_t(scissor.extent.height))
                            : tall ? "left alone (no viewport y)" : "not matched, left alone";
                        if (!scan || !valid)
                        {
                            if (tall)
                                logOnce(s_vs->hash, s_ps->hash, std::format("band vs_{:016x} ps_{:016x} prim {} n{} (translation c3 {:.3f} c7 {:.3f}): not measured{}: {}",
                                    s_vs->hash, s_ps->hash, prim, count, c[3], c[7], textured ? ", textured" : "", action));
                        }
                        else if (boxShare >= 0.95f || (boxShare >= 0.1f && (atPlus || atMinus)) || (tall && pastBand))
                        {
                            // A cover's 2D textures (address, size): a picture
                            // loaded with the menu, or a copy of the frame
                            // (a resolve: then the cover would show a zoomed
                            // copy of the picture under it).
                            std::string textures;
                            for (uint32_t n = 0; n < 32; n++)
                                if (cover && (s_ps->info.texture2DMask & (1u << n)))
                                {
                                    xenos::xe_gpu_texture_fetch_t f;
                                    memcpy(&f, &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + n * 6], sizeof(f));
                                    textures += std::format(" t{} {:08X} {}x{}", n, (uint32_t(f.base_address) << 12) & 0x1FFFFFFFu,
                                        uint32_t(f.size_2d.width) + 1, uint32_t(f.size_2d.height) + 1);
                                }
                            logOnce(s_vs->hash, s_ps->hash, std::format("band vs_{:016x} ps_{:016x} prim {} n{} ({} primitives) x {:.3f}..{:.3f} y {:.3f}..{:.3f} "
                                "(band y +-{:.3f}; translation c3 {:.3f} c7 {:.3f}): box {:.0f}%, primitives {:.0f}%{}{}{}{}: {}{}{}{}",
                                s_vs->hash, s_ps->hash, prim, count, prims, allLo, allHi, allLoY, allHiY, sv, c[3], c[7], boxShare * 100.0f,
                                bandArea / (4.0f * sv) * 100.0f, textured ? ", textured" : "", atPlus ? ", at +y edge" : "",
                                atMinus ? ", at -y edge" : "", pastBand ? ", past it" : "", action, small ? "" : " (n > 64: not acted on)",
                                textures, primList));
                        }
                    }
                    // Nothing of the band inside the game's own scissor.
                    if (clipped && scissor.extent.height == 0)
                        return;
                }
            }
        }
        else if (barFill)
        {
            // 16:9 (or a 16:9 picture letterboxed on another shape): none of
            // the rules above has anything to do, and no bar or black frame
            // needs a fill, but a full-band backdrop (the results screen)
            // leaves the race showing in the band's overscan rows above and
            // below it: those are cleared to black (gpu::bars::OverscanRows).
            // A front-end draw as above (the front-end matrix in c0..c3,
            // ePolys at fetch constant 95, read by the vertex shader); nothing
            // is measured, and only one quad over a large t0 gets as far as
            // reading its vertices (gpu::bars::BackdropCandidate).
            if (const float feX = game::FrontEndScaleX(); feX > 0.0f)
            {
                float c[16];
                memcpy(c, &s_regs[XE_GPU_REG_SHADER_CONSTANT_000_X], sizeof(c));
                const uint32_t* vf = &s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + 95 * 2];
                if (std::fabs(c[0] - feX) <= feX * 1e-5f && c[1] == 0.0f && c[2] == 0.0f &&
                    c[12] == 0.0f && c[13] == 0.0f && c[14] == 0.0f && c[15] == 1.0f && ((vf[0] & 3) == 3))
                {
                    const auto& vi = s_vs->info;
                    const bool feShader = ((vi.vertexFetchConstants[95 / 32] >> (95 % 32)) & 1) &&
                        (vi.constRelative || (vi.constUsed[0] & 0xFull) == 0xFull);
                    barPlan = planEdgeFills(c, game::FrontEndScaleY(), s, sv, feShader, true, false);
                    logEdgeFill(barPlan, s, sv);
                }
            }
        }
        // NFSMW_LOG_DRAWS=<seconds>: every draw of one frame (target, viewport,
        // scissor, textures, shaders), consecutive duplicates folded.
        if (s_logDrawsNow)
        {
            std::string line = std::format("{}x{} vp s({:.1f},{:.1f}) o({:.1f},{:.1f}) sc({},{})-({},{}) vte{:03X} vs_{:016x} ps_{:016x} tex",
                targetW, targetH, sx, sy, ox, oy, x0, y0, x1, y1, vte.value & 0xFFF, s_vs->hash, s_ps->hash);
            for (uint32_t n = 0; n < 32; n++)
                if ((s_ps->info.texture2DMask | s_ps->info.texture3DMask | s_ps->info.textureCubeMask) & (1u << n))
                    line += std::format(" {}:{:08X}", n, (s_regs[XE_GPU_REG_SHADER_CONSTANT_FETCH_00_0 + n * 6 + 1] & 0x1FFFF000u));
            if (barPlan.rectCount)
                line += std::format(" +edge {} fill x{}", barPlan.fill == gpu::bars::Fill::Backdrop ? "backdrop"
                    : barPlan.fill == gpu::bars::Fill::Blackout ? "black frame" : "bar", barPlan.rectCount);
            static std::string last;
            static uint32_t repeats = 0;
            if (line == last)
                repeats++;
            else
            {
                if (repeats)
                    fprintf(stderr, "[draws]   (x%u more)\n", repeats);
                fprintf(stderr, "[draws] %s\n", line.c_str());
                last = line;
                repeats = 0;
            }
        }
        pc.vsConstBase = 0;
        pc.psConstBase = 256;
        pc.indexOffset = s_regs[XE_GPU_REG_VGT_INDX_OFFSET];
        auto colorControl = Reg<reg::RB_COLORCONTROL>();
        pc.alphaTest = colorControl.alpha_test_enable ? uint32_t(colorControl.alpha_func) : 7u;
        pc.alphaRef = RegF(XE_GPU_REG_RB_ALPHA_REF);

        float zmin = std::clamp(oz, 0.0f, 1.0f), zmax = std::clamp(oz + sz, 0.0f, 1.0f);
        VkViewport viewport{ 0.0f, 0.0f, w * float(scaleX), h * float(scaleY), zmin, zmax };

        // What this command buffer already has: commands that would set the
        // same value again are skipped (most draws change only a few). Set 0
        // stays bound and push constants stay valid across the texture
        // layouts (same set-0 layout and push-constant range); set 1 is
        // pushed again whenever the layout changes. Compute work and a new
        // command buffer reset it (s_stateEpoch).
        struct BoundState
        {
            uint64_t epoch = 0;
            VkPipeline pipeline = VK_NULL_HANDLE;
            VkViewport viewport{};
            VkRect2D scissor{};
            uint32_t stencil[6] = {};
            float blend[4] = {}, bias[3] = {};
            uint32_t setOffset = ~0u;
            VkPipelineLayout textureLayout = VK_NULL_HANDLE;
            uint32_t textureCount = ~0u;
            VkDescriptorImageInfo textures[96];
            DrawConstants pc{};
        };
        static BoundState bound;
        // The first draw after a reset sets everything: dynamic state must be
        // set before a draw uses it, whatever the values.
        const bool fresh = !(g_cpOpt & CP_OPT_SKIP_REDUNDANT) || bound.epoch != s_stateEpoch;
        if (fresh)
        {
            bound = BoundState{};
            bound.epoch = s_stateEpoch;
        }
        if (fresh || bound.pipeline != pipeline)
        {
            vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
            bound.pipeline = pipeline;
        }
        if (fresh || memcmp(&bound.viewport, &viewport, sizeof(viewport)) != 0)
        {
            vkCmdSetViewport(s_cmd, 0, 1, &viewport);
            bound.viewport = viewport;
        }
        // The scissor and the push constants are recorded through these, so
        // the bound state always follows what was recorded: the edge-bar
        // copies (below) change both, and a stale cache would skip the next
        // draw's own scissor and clip it to a fill rectangle.
        auto setScissor = [&](const VkRect2D& r) {
            vkCmdSetScissor(s_cmd, 0, 1, &r);
            bound.scissor = r;
        };
        auto pushConstants = [&](const DrawConstants& c) {
            vkCmdPushConstants(s_cmd, layout.pipeline, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(c), &c);
            bound.pc = c;
        };
        if (fresh || memcmp(&bound.scissor, &scissor, sizeof(scissor)) != 0)
            setScissor(scissor);
        auto front = Reg<reg::RB_STENCILREFMASK>();
        auto back = depthControl.backface_enable ? RegAt<reg::RB_STENCILREFMASK>(XE_GPU_REG_RB_STENCILREFMASK_BF) : front;
        uint32_t stencil[6] = { front.stencilmask, back.stencilmask, front.stencilwritemask, back.stencilwritemask,
            front.stencilref, back.stencilref };
        if (fresh || memcmp(bound.stencil, stencil, sizeof(stencil)) != 0)
        {
            vkCmdSetStencilCompareMask(s_cmd, VK_STENCIL_FACE_FRONT_BIT, front.stencilmask);
            vkCmdSetStencilCompareMask(s_cmd, VK_STENCIL_FACE_BACK_BIT, back.stencilmask);
            vkCmdSetStencilWriteMask(s_cmd, VK_STENCIL_FACE_FRONT_BIT, front.stencilwritemask);
            vkCmdSetStencilWriteMask(s_cmd, VK_STENCIL_FACE_BACK_BIT, back.stencilwritemask);
            vkCmdSetStencilReference(s_cmd, VK_STENCIL_FACE_FRONT_BIT, front.stencilref);
            vkCmdSetStencilReference(s_cmd, VK_STENCIL_FACE_BACK_BIT, back.stencilref);
            memcpy(bound.stencil, stencil, sizeof(stencil));
        }
        float blendConstants[4] = { RegF(XE_GPU_REG_RB_BLEND_RED), RegF(XE_GPU_REG_RB_BLEND_GREEN),
            RegF(XE_GPU_REG_RB_BLEND_BLUE), RegF(XE_GPU_REG_RB_BLEND_ALPHA) };
        if (fresh || memcmp(bound.blend, blendConstants, sizeof(blendConstants)) != 0)
        {
            vkCmdSetBlendConstants(s_cmd, blendConstants);
            memcpy(bound.blend, blendConstants, sizeof(blendConstants));
        }
        float bias[3] = { 0.0f, 0.0f, 0.0f };
        if (depthBias)
        {
            bias[0] = RegF(XE_GPU_REG_PA_SU_POLY_OFFSET_FRONT_OFFSET) * float(1 << 24);
            // The slope factor is per host pixel: scaled with the resolution (Xenia).
            bias[2] = RegF(XE_GPU_REG_PA_SU_POLY_OFFSET_FRONT_SCALE) / 16.0f * std::max(scaleX, scaleY);
        }
        if (fresh || memcmp(bound.bias, bias, sizeof(bias)) != 0)
        {
            vkCmdSetDepthBias(s_cmd, bias[0], bias[1], bias[2]);
            memcpy(bound.bias, bias, sizeof(bias));
        }

        uint32_t dynamicOffset = uint32_t(constantsOffset);
        if (fresh || bound.setOffset != dynamicOffset)
        {
            vkCmdBindDescriptorSets(s_cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, layout.pipeline, 0, 1, &s_set0, 1, &dynamicOffset);
            bound.setOffset = dynamicOffset;
        }

        VkWriteDescriptorSet writes[96];
        uint32_t writeCount = 0;
        for (uint32_t d = 0; d < 3; d++)
            for (uint32_t n = 0; n < 32; n++)
            {
                if (!(((layout.vsMasks[d] | layout.psMasks[d]) >> n) & 1))
                    continue;
                VkWriteDescriptorSet& wds = writes[writeCount++];
                wds = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET };
                wds.dstBinding = d * 32 + n;
                wds.descriptorCount = 1;
                wds.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
                wds.pImageInfo = &textureInfos[d * 32 + n];
            }
        if (writeCount)
        {
            bool same = !fresh && bound.textureLayout == layout.pipeline && bound.textureCount == writeCount;
            for (uint32_t i = 0; same && i < writeCount; i++)  // (not memcmp: the struct has padding)
                same = bound.textures[i].sampler == writes[i].pImageInfo->sampler &&
                    bound.textures[i].imageView == writes[i].pImageInfo->imageView &&
                    bound.textures[i].imageLayout == writes[i].pImageInfo->imageLayout;
            if (!same)
            {
                s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, layout.pipeline, 1, writeCount, writes);
                bound.textureLayout = layout.pipeline;
                bound.textureCount = writeCount;
                for (uint32_t i = 0; i < writeCount; i++)
                    bound.textures[i] = *writes[i].pImageInfo;
            }
        }
        if (fresh || memcmp(&bound.pc, &pc, sizeof(pc)) != 0)
            pushConstants(pc);

        // Guest memory this draw reads when the GPU runs it: vertex data, and
        // indices when the shader fetches them itself.


        if (g_eagerWrites && (++s_drawsSinceCheck & 127) == 0)
        {
            // Every 128 draws: submit when another thread waits on recorded
            // work (write-watch guard), or when the GPU is starving and the
            // recording is at least NFSMW_STARVE_SUBMIT_US old (default
            // 1000). Each submit costs the CP ~20 us on RADV plus a pass
            // restart; every 128 draws was ~25 submits a frame in races.
            static const int64_t starveUs = [] { const char* v = std::getenv("NFSMW_STARVE_SUBMIT_US"); return v ? int64_t(std::atoll(v)) : 1000; }();
            bool submit = writewatch::g_submitRequested.exchange(false);
            if (!submit && std::chrono::steady_clock::now() - s_recordStart >= std::chrono::microseconds(starveUs))
            {
                std::lock_guard lock(s_submitMutex);
                submit = s_inFlight.size() < 2;
            }
            if (submit)
            {
                // Record this draw first: submit right after it.
                s_submitAfterDraw = true;
            }
        }

        if (s_exp & 1)
            return;
        if (zpdCount)
        {
            // Weighted by the guest's samples per pixel (its MSAA; the host
            // renders 1) over host pixels per guest pixel (the render scale).
            const uint64_t segments = s_zpdSegNext;
            // (At 1.5x a host pixel is 4/9 of a guest one: msaa x 4 / 9.)
            ZpdBeginSegment((1u << uint32_t(surface.msaa_samples)) * scale.PixelsDen(), scale.PixelsNum());
            s_zpdCounts.draws++;
            if (ZpdLogging())
            {
                std::string targets = depth ? std::format("D[b{} p{}]", depth->base, depth->pitch) : "no depth";
                for (int i = 0; i < 4; i++)
                    if (color[i])
                        targets += std::format(" C{}[b{} p{}]{}", i, color[i]->base, color[i]->pitch, (adoptedColor >> i) & 1 ? " (pass's)" : "");
                ZpdLog("f%llu draw vs_%016llx ps_%016llx n%u: %s, x %u/%u, %s, scissor %ux%u@%d,%d (host px)",
                    (unsigned long long)s_frame, (unsigned long long)s_vs->hash, (unsigned long long)s_ps->hash, vertexCount,
                    s_zpdQuery < 0 ? "not counted (this command buffer's queries are used up)"
                        : std::format("{} {}", s_zpdSegNext != segments ? "new segment" : "segment", s_zpdSegNext - 1).c_str(),
                    (1u << uint32_t(surface.msaa_samples)) * scale.PixelsDen(), scale.PixelsNum(), targets.c_str(), scissor.extent.width,
                    scissor.extent.height, scissor.offset.x, scissor.offset.y);
            }
        }
        if (useIndices)
        {
            vkCmdBindIndexBuffer(s_cmd, s_ring, indexOffset, indices16 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32);
            vkCmdDrawIndexed(s_cmd, vertexCount, 1, 0, 0, 0);
        }
        else
        {
            vkCmdDraw(s_cmd, vertexCount, 1, 0, 0);
        }
        // Edge bars (above): the same draw again per rectangle, with the
        // stretched posScale and the rectangle as its scissor (inside this
        // draw's own, so inside the pass); through setScissor and
        // pushConstants, so the bound state follows.
        if (barPlan.rectCount && barPlan.fill == gpu::bars::Fill::Backdrop)
        {
            // A backdrop: its rectangles (inside this draw's scissor, so
            // inside the pass) cleared to opaque black in the colour targets
            // it writes; nothing else changes.
            VkClearAttachment ca[4]{};
            uint32_t attachments = 0;
            for (uint32_t i = 0; i < 4; i++)
                if (color[i] && ((writeMask >> (4 * i)) & 0xF))
                {
                    ca[attachments].aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
                    ca[attachments].colorAttachment = i;
                    ca[attachments].clearValue.color = { { 0.0f, 0.0f, 0.0f, 1.0f } };
                    attachments++;
                }
            VkClearRect cr[std::extent_v<decltype(gpu::bars::Plan::rects)>]{};
            for (uint32_t i = 0; i < barPlan.rectCount; i++)
            {
                const gpu::bars::Rect& r = barPlan.rects[i];
                cr[i] = { { { r.x0, r.y0 }, { uint32_t(r.x1 - r.x0), uint32_t(r.y1 - r.y0) } }, 0, 1 };
            }
            if (attachments)
                vkCmdClearAttachments(s_cmd, attachments, ca, barPlan.rectCount, cr);
        }
        else if (barPlan.rectCount)
        {
            DrawConstants fill = pc;
            memcpy(fill.posScale, barPlan.posScale, sizeof(barPlan.posScale));
            pushConstants(fill);
            for (uint32_t i = 0; i < barPlan.rectCount; i++)
            {
                const gpu::bars::Rect& r = barPlan.rects[i];
                setScissor(VkRect2D{ { r.x0, r.y0 }, { uint32_t(r.x1 - r.x0), uint32_t(r.y1 - r.y0) } });
                if (useIndices)
                    vkCmdDrawIndexed(s_cmd, vertexCount, 1, 0, 0, 0);
                else
                    vkCmdDraw(s_cmd, vertexCount, 1, 0, 0);
            }
        }
        if (s_submitAfterDraw)
        {
            s_submitAfterDraw = false;
            SubmitRecorded(8);
        }
    }

    void Submit()
    {
        if (s_vk)
            SubmitRecorded(2);
    }

    void SubmitIfIdle()
    {
        if (!s_vk || !s_recording)
            return;
        size_t inFlight;
        {
            std::lock_guard lock(s_submitMutex);
            inFlight = s_inFlight.size();
        }
        if (inFlight < (g_eagerWrites ? 2u : 1u))
            SubmitRecorded(3);
    }

    void SubmitIfDue()
    {
        // With fences published mid-buffer (events), batching more work per
        // command buffer costs the game nothing while the GPU is busy anyway;
        // each Metal command buffer has a large fixed cost. Submit when the
        // queue is shallow, or when the recording has aged.
        // Eager writes: nothing waits on the recording, so keep the GPU fed.
        static const uint32_t depth = [] { const char* v = std::getenv("NFSMW_SUBMIT_DEPTH"); return v ? uint32_t(std::atoi(v)) : g_eagerWrites ? 2u : 1u; }();
        static const int64_t ageUs = [] { const char* v = std::getenv("NFSMW_SUBMIT_AGE_US"); return v ? int64_t(std::atoll(v)) : 2000; }();
        if (!s_vk || !s_recording)
            return;
        if (!s_fenceEvents)
        {
            SubmitRecorded(4);
            return;
        }
        size_t inFlight;
        {
            std::lock_guard lock(s_submitMutex);
            inFlight = s_inFlight.size();
        }
        if (inFlight < depth || std::chrono::steady_clock::now() - s_recordStart > std::chrono::microseconds(ageUs))
            SubmitRecorded(4);
    }

    void Flush()
    {
        if (!s_vk)
            return;
        SubmitRecorded(5);
        CpWaitTimer wait(CP_WAIT_FLUSH);
        std::unique_lock lock(s_submitMutex);
        s_submitCv.wait(lock, [] { return s_inFlight.empty(); });
    }

    void QueueWrite(std::function<void()> write)
    {
        if (!s_vk || g_eagerWrites)
        {
            write();
            return;
        }
        // Attach to the newest submission: the one being recorded (it then
        // becomes visible when that completes, a little later than strictly
        // needed, which is always safe for fences), else the last in flight.
        std::unique_lock lock(s_submitMutex);
        if (s_recording)
        {
            s_submissions[s_slot].writes.push_back({ std::move(write) });
            bool inPass = s_passActive;
            // With the GPU idle nothing would carry this write until the next
            // submit: send the work now so whoever waits on it isn't stalled.
            bool idle = s_inFlight.empty();
            lock.unlock();
            if (idle)
                SubmitRecorded(6);
            else if (!inPass)
                SignalWrites();  // in a pass, the pass end signals it
            return;
        }
        if (s_inFlight.empty())
        {
            lock.unlock();
            write();
            return;
        }
        s_submissions[s_inFlight.back()].writes.push_back({ std::move(write) });
    }

    // Occlusion queries, counted (NFSMW_OCCLUSION=1). The game brackets its
    // sun quads with them and turns the share of samples that passed into
    // the glare's strength (gpu/zpd_report.h has the reports). A bracket
    // runs from its BEGIN ZPD to its END ZPD; the draws in it are counted
    // in segments, each a precise occlusion query begun right before a
    // draw inside its pass (no pass is split for a query) and ended at the
    // pass's end, at the next ZPD or when the weight changes, so a segment
    // belongs to every bracket open while it ran (they may overlap; Vulkan
    // allows one active query). The END's report is stored when the newest
    // submission completes (every segment's count is in by then: the
    // completion thread harvests in submission order, never waiting), in
    // stream order, so a predicated-tiling query's last tile, which makes
    // it ready, lands last. Tile 0 holds the whole count (untiled
    // rendering: later tiles' draws are skipped, their brackets count 0),
    // as GetData sums the tiles. A result lands a frame or two after its
    // draw; the game polls without waiting and keeps its value meanwhile,
    // as on the console. Nothing here waits.
    bool ZpassDone(uint32_t address)
    {
        if (!s_zpdOn)
            return false;
        address &= 0x1FFFFFFF;
        if (address > kSharedSize - 32)
            return false;
        // Every ZPD ends the segment (inside the open pass: legal), so no
        // segment spans a bracket's edge.
        ZpdEndSegment();
        s_zpdCounts.zpds++;
        switch (zpd::Classify(address))
        {
        case zpd::Kind::Begin:
        {
            zpd::Bracket old;
            auto opened = s_zpdOpen.Open(address, s_zpdSegNext, s_frame, &old);
            s_zpdCounts.begins++;
            if (opened == zpd::BracketBook::Opened::Replaced)
            {
                s_zpdCounts.replaced++;
                LogOnce((0x2Cull << 56) | address, "occlusion query: BEGIN %08X again before its END (open since frame %llu): replaced",
                    address, (unsigned long long)old.frame);
            }
            else if (opened == zpd::BracketBook::Opened::DroppedOldest)
            {
                s_zpdCounts.dropped++;
                LogOnce((0x2Aull << 56) | 3, "occlusion query: more than %u brackets open: the oldest (BEGIN %08X, frame %llu) dropped",
                    zpd::BracketBook::kMax, old.beginAddress, (unsigned long long)old.frame);
            }
            ZpdLog("f%llu BEGIN %08X: segments from %llu, %u open", (unsigned long long)s_frame, address,
                (unsigned long long)s_zpdSegNext, s_zpdOpen.Size());
            return false;  // its zeros are stored as without counting
        }
        case zpd::Kind::End:
        {
            ZpdEnd end{ address, s_zpdSegNext, s_zpdSegNext, s_frame, std::chrono::steady_clock::now() };
            zpd::Bracket closed;
            if (s_zpdOpen.Close(address, &closed))
                end.s0 = closed.s0;
            else
            {
                s_zpdCounts.unmatched++;
                LogOnce((0x2Dull << 56) | address, "occlusion query: END %08X without its BEGIN: zero samples", address);
            }
            s_zpdCounts.ends++;
            ZpdLog("f%llu END %08X queued: segments %llu..%llu", (unsigned long long)s_frame, address,
                (unsigned long long)end.s0, (unsigned long long)end.s1);
            // With the newest submission, as QueueWrite: the one being
            // recorded, else the last in flight, else now (nothing in
            // flight: every count is in, harvested before its submission
            // left s_inFlight under this lock).
            std::lock_guard lock(s_submitMutex);
            if (s_recording)
                s_submissions[s_slot].zpdEnds.push_back(end);
            else if (!s_inFlight.empty())
                s_submissions[s_inFlight.back()].zpdEnds.push_back(end);
            else
                ZpdStore(end);
            return true;
        }
        default:
            s_zpdCounts.unknown++;
            LogOnce((0x2Eull << 56) | address, "occlusion query: report at %08X is neither a BEGIN (+0x20) nor an END (+0) of a 64-byte block: zero samples",
                address);
            return false;
        }
    }

    void LogOcclusionCounts(const char* what)
    {
        if (!s_zpdOn)  // (also when the device counts but the pools couldn't be made)
            return;
        std::string line;
        {
            std::lock_guard lock(s_zpdLatestMutex);
            for (const auto& [address, latest] : s_zpdLatest)
                line += std::format(" {:08X} {} (frame {})", address, latest.first, latest.second);
        }
        fprintf(stderr, "[zpd] %s: latest END counts:%s\n", what, line.empty() ? " none yet" : line.c_str());
    }

    bool PresentFrontBuffer(const uint32_t fetch[6], uint32_t width, uint32_t height)
    {
        if (!s_vk || width == 0 || height == 0)
            return false;
        // NFSMW_SHOW_TEXTURE=<hex base>,<width>,<height>: present that tiled
        // 32-bit guest texture instead of the front buffer (debugging).
        static const std::array<uint32_t, 3> showTexture = [] {
            std::array<uint32_t, 3> t{};
            if (const char* v = std::getenv("NFSMW_SHOW_TEXTURE"))
                sscanf(v, "%x,%u,%u", &t[0], &t[1], &t[2]);
            return t;
        }();
        uint32_t debugFetch[6];
        if (showTexture[0])
        {
            memcpy(debugFetch, fetch, sizeof(debugFetch));
            debugFetch[0] = (debugFetch[0] & ~(0x1FFu << 22)) | (((showTexture[1] + 31) / 32) << 22) | 0x80000000u;
            debugFetch[1] = (debugFetch[1] & 0xFFFu) | (showTexture[0] & 0x1FFFF000u);
            fetch = debugFetch;
            width = showTexture[1];
            height = showTexture[2];
        }
        uint32_t pitch = ((fetch[0] >> 22) & 0x1FF) << 5;
        struct { uint32_t base, pitch, tiled, endian, format, swizzle, width, height, sx, sy, slotWords, scaled, denX, denY; } pc = {
            fetch[1] & 0x1FFFF000u, pitch ? pitch : (width + 31) & ~31u, (fetch[0] >> 31) & 1, (fetch[1] >> 6) & 3,
            fetch[1] & 0x3F, (fetch[3] >> 1) & 0xFFF, width, height, 1, 1, 0, 0, 1, 1 };
        uint32_t frontBytes = pc.pitch * ((height + 31) & ~31u) * 4;
        // At host resolution when a scaled resolve wrote it (the game's frames;
        // movies and loading screens drawn at 1x use the 1x image). At 1.5x
        // only a front buffer of whole blocks (even; 1280x720 is).
        bool scaled = pc.tiled && !showTexture[0] && s_scaled.enabled && s_scaled.scale.Exact(width, height) &&
            ScaledRangeValid(pc.base, frontBytes);
        uint32_t outW = scaled ? s_scaled.scale.x.Size(width) : width, outH = scaled ? s_scaled.scale.y.Size(height) : height;
        Image& front = scaled ? s_frontScaled : s_front;
        uint32_t& frontW = scaled ? s_frontScaledWidth : s_frontWidth;
        uint32_t& frontH = scaled ? s_frontScaledHeight : s_frontHeight;
        if (outW != frontW || outH != frontH)
        {
            // A new size: wait for any use of the old image before replacing it.
            DestroyFrontImage(front, frontW, frontH);
            front = CreateImage(VK_IMAGE_TYPE_2D, VK_FORMAT_R8G8B8A8_UNORM, { outW, outH, 1 }, 1,
                VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT,  // sampled: presenter post-processing
                VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_ASPECT_COLOR_BIT);
            Begin();
            ToGeneral(front.image, VK_IMAGE_ASPECT_COLOR_BIT, 1);
            frontW = outW;
            frontH = outH;
            fprintf(stderr, "[renderer] front image %ux%u (%s)\n", outW, outH, scaled ? "host resolution" : "1x");
        }
        if (scaled)
        {
            pc.width = outW;
            pc.height = outH;
            pc.sx = s_scaled.scale.x.num;
            pc.sy = s_scaled.scale.y.num;
            pc.denX = s_scaled.scale.x.den;
            pc.denY = s_scaled.scale.y.den;
            pc.slotWords = s_scaled.slotWords;
            pc.scaled = 1;
            s_scaled.untiledBytes += uint64_t(outW) * outH * 4;
        }
        else
            EnsureShadow(pc.base, frontBytes);
        EndPass();
        Begin();
        FullBarrier();  // after earlier work, including the presenter's last read
        VkDescriptorImageInfo ii{ VK_NULL_HANDLE, front.view, VK_IMAGE_LAYOUT_GENERAL };
        VkDescriptorBufferInfo pool = DummyStorage(s_scaled.pool);
        VkDescriptorBufferInfo table = DummyStorage(s_scaled.table);
        VkWriteDescriptorSet w[7] = {
            { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 0, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &s_sharedInfo[0], nullptr },
            { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 1, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &ii, nullptr, nullptr },
            { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 2, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &pool, nullptr },
            { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, VK_NULL_HANDLE, 3, 0, 1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &table, nullptr },
        };
        s_stateEpoch++;
        vkCmdBindPipeline(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_frontPipeline);
        s_pushDescriptorSet(s_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, s_frontLayout, 0, AddSharedParts(w, 4), w);
        vkCmdPushConstants(s_cmd, s_frontLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
        vkCmdDispatch(s_cmd, (outW + 7) / 8, (outH + 7) / 8, 1);
        GuardGpu(pc.base, frontBytes, false, "front buffer");
        FullBarrier();
        SubmitRecorded(7);  // the presenter's blit comes later in queue order
        video::SubmitFrontImage(front.image, outW, outH, width, height);
        // Back at 1x (a live change): the host-resolution image is no longer
        // needed. The presenter has this frame to show instead.
        if (!scaled && s_frontScaled.image && s_renderScale.IsOne())
            DestroyFrontImage(s_frontScaled, s_frontScaledWidth, s_frontScaledHeight);
        if (g_eagerWrites)
        {
            // With fences published early nothing else keeps the GPU from
            // falling behind: allow one frame in flight, as the console's
            // GPU never lags more. The ring then fills and the game waits.
            static uint64_t previousSwap = 0;
            uint64_t thisSwap = s_submissionCounter - 1;
            if (previousSwap)
            {
                CpWaitTimer wait(CP_WAIT_THROTTLE);
                std::unique_lock lock(s_submitMutex);
                s_submitCv.wait(lock, [] { return s_inFlight.empty() || s_submissions[s_inFlight.front()].number > previousSwap; });
            }
            previousSwap = thisSwap;
        }

        // NFSMW_CHECK_FRONT=<n>: at the n-th GPU present, read the image back
        // into build/front_check.ppm (verifies this path without a screen).
        // NFSMW_CHECK_FRONT_SEC=<s>: from s seconds after launch, 6 captures
        // 20 presents apart, into build/front_check_<k>.ppm.
        static const int checkAt = [] { const char* v = std::getenv("NFSMW_CHECK_FRONT"); return v ? std::atoi(v) : -1; }();
        static const double checkSec = [] { const char* v = std::getenv("NFSMW_CHECK_FRONT_SEC"); return v ? std::atof(v) : -1.0; }();
        static const auto launch = std::chrono::steady_clock::now();
        static int presents = 0, seriesStart = -1, seriesTaken = 0;
        presents++;
        std::string checkPath;
        if (presents == checkAt)
            checkPath = "build/front_check.ppm";
        else if (checkSec >= 0 && seriesTaken < 6)
        {
            if (seriesStart < 0 && std::chrono::duration<double>(std::chrono::steady_clock::now() - launch).count() >= checkSec)
                seriesStart = presents;
            if (seriesStart >= 0 && (presents - seriesStart) % 20 == 0)
                checkPath = "build/front_check_" + std::to_string(seriesTaken++) + ".ppm";
        }
        if (!checkPath.empty())
            width = outW, height = outH;  // what was presented
        VkDeviceSize at = checkPath.empty() ? 0 : RingAllocate(VkDeviceSize(width) * height * 4);
        if (at == ~VkDeviceSize(0))
        {
            LogOnce(0x2400, "front buffer capture %ux%u doesn't fit a staging slot", width, height);
            checkPath.clear();
        }
        if (!checkPath.empty())
        {
            Begin();
            FullBarrier();
            VkBufferImageCopy region{};
            region.bufferOffset = at;
            region.imageSubresource = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1 };
            region.imageExtent = { width, height, 1 };
            vkCmdCopyImageToBuffer(s_cmd, front.image, VK_IMAGE_LAYOUT_GENERAL, s_ring, 1, &region);
            Flush();
            if (FILE* f = fopen(checkPath.c_str(), "wb"))
            {
                // One bulk copy out of the ring first: on a discrete GPU it
                // is uncached VRAM, where per-pixel reads took ~20 s a frame.
                std::vector<uint8_t> rgba(s_ringHost + at, s_ringHost + at + size_t(width) * height * 4), rgb(size_t(width) * height * 3);
                for (size_t i = 0; i < size_t(width) * height; i++)
                    memcpy(&rgb[i * 3], &rgba[i * 4], 3);
                fprintf(f, "P6\n%u %u\n255\n", width, height);
                fwrite(rgb.data(), 1, rgb.size(), f);
                fclose(f);
                fprintf(stderr, "[renderer] front buffer check written\n");
                LogOcclusionCounts(checkPath.c_str());  // NFSMW_OCCLUSION=1: the counts behind it
            }
        }
        return true;
    }

    void OnSwap(uint64_t frame)
    {
        if (s_vk)
            ApplyLiveSettings();
        if (s_uploadLog)
        {
            UploadLog& log = *s_uploadLog;
            if (log.now)  // s_frame just ended
            {
                std::string paths, writers;
                for (auto& [k, n] : log.byPath)
                    paths += std::format(" {} {}", k, n);
                for (auto& [k, n] : log.byWriters)
                    writers += std::format(" {} {},", k, n);
                fprintf(stderr, "[upload] frame %llu summary: %u uploads (%llu KB, %u ending a pass) | paths:%s | writers:%s | resolves %u (%u fused)\n",
                    (unsigned long long)s_frame, log.uploads, (unsigned long long)(log.bytes / 1024), log.endedPasses, paths.c_str(),
                    writers.c_str(), log.resolvesNow, log.fusedNow);
                log.uploads = log.endedPasses = log.resolvesNow = log.fusedNow = 0;
                log.bytes = 0;
                log.byPath.clear();
                log.byWriters.clear();
            }
            log.now = frame >= log.first && frame <= log.last;
            writewatch::SetWriterFrame(frame);
        }
        s_frame = frame;
        if (s_zpdOn)
            ZpdOnSwap(frame);
        if (s_logShadowMap)
            ShadowMapProbeOnSwap();
        // NFSMW_PASS_BARRIER: per frame of the window ending here, with the
        // level it ran at, before its [perf] line. Vertex barriers cost a
        // pass's overlap (a split: a pass too).
        if (s_passBarrierLevel && frame % 120 == 0)
        {
            static BarrierCounts last{};
            const BarrierCounts& c = s_barrierCounts;
            fprintf(stderr, "[barrier] level %u per frame: %.1f pass ends and %.1f resolves narrowed, %.2f full at the cap | vertex barriers %.2f"
                " (vertex data %.2f, indices %.2f, a texture %.2f), %.2f splitting a pass\n", s_passBarrier,
                double(c.passEnds - last.passEnds) / 120, double(c.resolves - last.resolves) / 120, double(c.capped - last.capped) / 120,
                double(c.vertex[0] + c.vertex[1] + c.vertex[2] - last.vertex[0] - last.vertex[1] - last.vertex[2]) / 120,
                double(c.vertex[0] - last.vertex[0]) / 120, double(c.vertex[1] - last.vertex[1]) / 120,
                double(c.vertex[2] - last.vertex[2]) / 120, double(c.splits - last.splits) / 120);
            last = c;
        }
        // NFSMW_PASS_BARRIER_ALT: the next window's level. The swap's front
        // buffer has just put a full barrier behind everything recorded; a
        // CPU present's Flush (frame dumps) only idles the GPU, so what a
        // resolve left for the vertex stages gets one here before level 0.
        if (s_passBarrierAlt && frame % 120 == 0)
        {
            s_passBarrier = (frame / 120) % 2 ? s_passBarrierLevel : 0u;
            if (!s_passBarrier && (s_vertexPendingLo < s_vertexPendingHi || !s_vertexPendingTextures.empty()))
            {
                Begin();
                FullBarrier();
            }
            fprintf(stderr, "[barrier] alt: frames %llu-%llu run with level %u\n", (unsigned long long)(frame + 1),
                (unsigned long long)(frame + 120), s_passBarrier);
        }
        // NFSMW_LOG_UNTILED_AREA=1: with each [perf] line, whether the toggle
        // was on in its window (=alt: every other one) and what it did.
        if (s_logUntiledArea && frame % 120 == 0)
        {
            static UntiledAreaCounts last;
            const UntiledAreaCounts& n = s_ua;
            auto per = [](uint64_t now, uint64_t before) { return double(now - before) / 120.0; };
            const bool on = s_untiledArea == 1 || (s_untiledArea == 2 && (frame - 1) / 120 % 2 == 1);
            fprintf(stderr, "[untiled-area] %s per frame: %.1f untiled draws cut to their viewport, %.1f whole target (%.1f clipping off, "
                "%.1f no viewport transform, %.1f read past), %.1f nothing inside; %.2f pass restarts | %.1f passes, %.2f Mpx, %.1f MB loaded+stored; "
                "untiled %.1f passes, %.2f Mpx, %.1f MB | %llu resolves read past (total)\n",
                on ? "on" : "off", per(n.cut, last.cut), per(n.whole, last.whole), per(n.clipOff, last.clipOff), per(n.noViewport, last.noViewport),
                per(n.wholeArea, last.wholeArea), per(n.empty, last.empty), per(n.restarts, last.restarts), per(n.passes, last.passes),
                per(n.passPixels, last.passPixels) / 1e6, per(n.passBytes, last.passBytes) / 1e6, per(n.untiledPasses, last.untiledPasses),
                per(n.untiledPixels, last.untiledPixels) / 1e6, per(n.untiledBytes, last.untiledBytes) / 1e6, (unsigned long long)n.resolvesPast);
            last = n;
        }
        // NFSMW_LOG_CP_CACHES=1 (or NFSMW_CP_CACHE_VERIFY): the command
        // processor's lookup caches, with each [perf] line. (A separate
        // check from NFSMW_CP_OPT_VERIFY, whose index check reads the ring
        // back: uncached VRAM on the Deck and the Steam Machine, seconds a
        // frame.)
        static const bool logCaches = std::getenv("NFSMW_LOG_CP_CACHES") != nullptr || s_textureCache->verify;
        if (logCaches && frame % 120 == 0)
        {
            const TextureCache& t = *s_textureCache;
            const ShaderCache& sh = *s_shaderCache;
            const uint64_t now[9] = { t.hits, t.misses, t.revalidated, t.stale, t.uncached, t.recentHits, sh.loads, sh.hits,
                sh.comparedBytes };
            static uint64_t last[9] = {};
            uint64_t d[9];
            for (int i = 0; i < 9; i++)
                d[i] = now[i] - last[i];
            std::copy(std::begin(now), std::end(now), last);
            auto pct = [](uint64_t part, uint64_t whole) { return whole ? 100.0 * double(part) / double(whole) : 0.0; };
            fprintf(stderr, "[cpcache] textures: %.0f lookups/frame, %.1f%% kept (%.1f%% of them checked after a write, %.1f%% by hash), "
                "%.1f%% stale, %.1f%% not keepable | shaders: %.0f loads/frame, %.1f%% found by address (%.0f KB compared) | "
                "mismatches: %llu textures, %llu shaders (%llu texture checks raced a write)\n",
                double(d[0] + d[1]) / 120, pct(d[0], d[0] + d[1]), pct(d[2], d[0]), pct(d[5], d[0]), pct(d[3], d[0] + d[1]),
                pct(d[4], d[0] + d[1]), double(d[6]) / 120, pct(d[7], d[6]), double(d[8]) / 1024 / 120,
                (unsigned long long)t.mismatches, (unsigned long long)sh.mismatches, (unsigned long long)t.raced);
        }
        s_pipelineWaitLeftUs = s_pipelineWaitUs;
        // A loading screen (or movie): wait for pipelines in full for 2 s.
        if (s_stats.draws - s_drawsAtSwap < 64)
            s_waitPipelinesUntil = frame + 120;
        s_drawsAtSwap = s_stats.draws;
        if (s_pipelineSkips)
        {
            static uint32_t logged = 0;
            if (logged++ < 300)
                fprintf(stderr, "[pipelines] frame %llu: %u draws skipped, %zu pipelines compiling\n",
                    (unsigned long long)frame, s_pipelineSkips, s_pendingPipelines.size());
            s_stats.pipelineSkips += s_pipelineSkips;
            s_pipelineSkips = 0;
        }
        // New pipelines are saved every 10 s, on a thread of its own: getting a
        // large cache's data and writing it took ~70 ms, a visible stutter
        // when it ran here (the pipeline cache is internally synchronized).
        if (s_vk && frame % 600 == 0 && s_stats.pipelines != s_pipelinesAtSave)
        {
            s_pipelinesAtSave = uint32_t(s_stats.pipelines);
            static std::atomic<bool> saving{ false };
            if (!saving.exchange(true))
                std::thread([] {
                    SetHostThreadName("nfsmw-cachesave");
                    hostcpu::LeaveReservedCore();
                    SavePipelineCache();
                    saving = false;
                }).detach();
        }
        // NFSMW_LOG_DRAWS=<seconds>: the first frame after that time.
        static const double logDrawsSec = [] { const char* v = std::getenv("NFSMW_LOG_DRAWS"); return v ? std::atof(v) : -1.0; }();
        static const auto launch = std::chrono::steady_clock::now();
        static bool loggedDraws = false;
        s_logDrawsNow = false;
        if (logDrawsSec >= 0 && !loggedDraws &&
            std::chrono::duration<double>(std::chrono::steady_clock::now() - launch).count() >= logDrawsSec)
        {
            loggedDraws = true;
            s_logDrawsNow = true;
            fprintf(stderr, "[draws] frame %llu\n", (unsigned long long)frame);
        }
        if (!s_vk || s_profileFrame < 0)
            return;
        if (frame == uint64_t(s_profileFrame))
        {
            s_profiling = true;
            s_passInfos.clear();
            s_gapLast = { s_stats.resolves, s_stats.uploads, s_stats.submits, writewatch::GetGuardStats().uploadBytes };
            s_passQueryNext = 0;
            return;
        }
        if (!s_profiling)
            return;
        s_profiling = false;
        Flush();  // presentation no longer waits: make the results ready
        // Per pass, also the GPU time since the previous pass ended: the
        // resolves, untiles, copies and barriers between them (and idle time
        // where a submission waited).
        double total = 0, between = 0;
        uint64_t lastEnd = 0;
        fprintf(stderr, "[profile] frame %d: %zu passes\n", s_profileFrame, s_passInfos.size());
        for (const PassInfo& pi : s_passInfos)
        {
            uint64_t ts[2] = {};
            vkGetQueryPoolResults(s_dev, s_passQueries, pi.query, 2, sizeof(ts), ts, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT);
            double us = ts[1] > ts[0] ? double(ts[1] - ts[0]) * s_timestampPeriod / 1000.0 : 0.0;
            double gap = lastEnd && ts[0] > lastEnd ? double(ts[0] - lastEnd) * s_timestampPeriod / 1000.0 : 0.0;
            total += us;
            between += gap;
            if (ts[1] > ts[0])
                lastEnd = ts[1];
            fprintf(stderr, "[profile] %8.1f us  %4u draws  (+%7.1f us before: %u resolves, %u uploads, %u submits, %llu KB synced)  %s\n",
                us, pi.draws, gap, pi.resolves, pi.uploads, pi.submits, (unsigned long long)pi.shadowKB, pi.desc.c_str());
        }
        fprintf(stderr, "[profile] total %.1f us in passes, %.1f us between them\n", total, between);
    }

    std::vector<uint32_t> CompileShader(const std::string& glsl, int stage, std::string& log)
    {
        static std::once_flag initialized;
        std::call_once(initialized, [] { glslang_initialize_process(); });
        glslang_stage_t stages[3] = { GLSLANG_STAGE_VERTEX, GLSLANG_STAGE_FRAGMENT, GLSLANG_STAGE_COMPUTE };
        return CompileGlsl(glsl, stages[std::clamp(stage, 0, 2)], log);
    }

    void Shutdown()
    {
        if (!s_vk)
            return;
        {
            // Let running compiles finish (their pipelines go into the
            // cache being saved); drop the queued ones.
            std::unique_lock lock(s_compileMutex);
            s_compileQueue.clear();
            s_compileCv.wait_for(lock, std::chrono::seconds(2), [] { return s_compileActive == 0; });
            s_compileStopping = true;
        }
        s_compileCv.notify_all();
        SavePipelineCache();
    }

    void HitchReport(uint64_t frame, double frameMs, uint32_t vblanks)
    {
        // A frame is late when it took NFSMW_HITCH_LATE_MS (default 3; 0 off)
        // longer than the median of the last 63, so a 20 ms frame among 16.7
        // ms ones is, unless it continues a cadence (as long as the frame
        // before, a whole number of vblanks: 30 fps in movies and parts of
        // the front end); or when it took NFSMW_HITCH_MS (default 50) or
        // more. At most 8 are logged per 120-frame [perf] window (its "late"
        // count has them all), with where the command processor's time went
        // and what the game's threads were doing meanwhile. A fixed 50 ms
        // threshold hid the 19-25 ms frames that stutter while driving.
        static const double lateMs = [] { const char* v = std::getenv("NFSMW_HITCH_LATE_MS"); return v ? std::atof(v) : 3.0; }();
        static const double threshold = [] { const char* v = std::getenv("NFSMW_HITCH_MS"); return v ? std::atof(v) : 50.0; }();
        static std::array<double, 63> recent{};
        static uint32_t recentCount = 0;
        double median = 0, previous = recentCount ? recent[(recentCount - 1) % recent.size()] : 0;
        if (recentCount >= 15)
        {
            std::array<double, 63> sorted = recent;
            uint32_t n = std::min<uint32_t>(recentCount, uint32_t(sorted.size()));
            std::nth_element(sorted.begin(), sorted.begin() + n / 2, sorted.begin() + n);
            median = sorted[n / 2];
        }
        recent[recentCount++ % recent.size()] = frameMs;

        struct Totals
        {
            uint64_t busyNs, waitNs[CP_WAIT_KINDS], draws, tileSkips, resolves, submits, snapshotBytes, gpuExecNs, gameWaitNs, gamePresentNs,
                allocations, allocationNs;
            writewatch::GuardStats guard;
            uint64_t cpuNs = 0;
            long preemptions = 0;
        };
        static Totals last{};
        Totals now{ g_cpBusyNs, {}, s_stats.draws, s_stats.tileSkips, s_stats.resolves, s_stats.submits, s_stats.snapshotBytes,
            s_gpuExecNs.load(), timeline::g_gameWaitNs.load(), timeline::g_gamePresentNs.load(), s_imageTotals.allocations,
            s_imageTotals.allocationNs, writewatch::GetGuardStats() };
        std::copy(std::begin(g_cpWaitKindNs), std::end(g_cpWaitKindNs), now.waitNs);
        // This (the command processor's) thread's CPU time, and on Linux the
        // times the scheduler took its core away: busy time the thread wasn't
        // on a core is time it was descheduled, not work.
        struct timespec cpu{};
        clock_gettime(CLOCK_THREAD_CPUTIME_ID, &cpu);
        now.cpuNs = uint64_t(cpu.tv_sec) * 1000000000ull + uint64_t(cpu.tv_nsec);
#ifdef RUSAGE_THREAD
        struct rusage usage{};
        getrusage(RUSAGE_THREAD, &usage);
        now.preemptions = usage.ru_nivcsw;
#endif
        double gameFrameMs = double(timeline::g_gameLongestFrameNs.exchange(0)) / 1e6;
        double vblanksLong = std::round(frameMs / 16.667);
        bool cadence = std::abs(frameMs - previous) < 1 && vblanksLong >= 2 && std::abs(frameMs - vblanksLong * 16.667) < 1;
        bool late = frameMs >= threshold || (lateMs > 0 && median > 0 && frameMs > median + lateMs && !cadence);
        static uint64_t window = 0;
        static uint32_t logged = 0;
        if ((frame + 119) / 120 != window)  // the [perf] line at frame % 120 == 0 closes a window
        {
            window = (frame + 119) / 120;
            logged = 0;
        }
        if (late)
            s_stats.lateFrames++;
        if (late && logged++ < 8)
        {
            auto ms = [](uint64_t ns) { return double(ns) / 1e6; };
            double waits[CP_WAIT_KINDS], waitMs = 0;
            for (uint32_t k = 0; k < CP_WAIT_KINDS; k++)
                waitMs += waits[k] = ms(now.waitNs[k] - last.waitNs[k]);
            // The frame's time is guest time, which stood still while the
            // game was suspended: that wait is shown apart, outside the
            // frame's waits and idle time.
            waitMs -= waits[CP_WAIT_SUSPENDED];
            char suspended[48] = "";
            if (waits[CP_WAIT_SUSPENDED] > 0)
                snprintf(suspended, sizeof(suspended), ", suspended %.1f ms", waits[CP_WAIT_SUSPENDED]);
            double busyMs = ms(now.busyNs - last.busyNs);
            const writewatch::GuardStats& g = now.guard;
            const writewatch::GuardStats& lg = last.guard;
            // idle: the ring was empty (the command processor waited for the game).
            char preempted[32] = "";
#ifdef RUSAGE_THREAD
            snprintf(preempted, sizeof(preempted), ", preempted %ld times", now.preemptions - last.preemptions);
#endif
            // Textures' time includes their shadow sync; mprotect counts all
            // threads (the guard's fault handler too).
            fprintf(stderr, "[hitch] frame %llu took %.1f ms (median %.1f, %u vblanks) | command processor: busy %.1f ms, "
                "waits %.1f ms (throttle %.1f, regmem %.1f, shadow %.1f, flush %.1f, pipeline %.1f, slot %.1f)%s, idle %.1f ms, "
                "on a core %.1f ms%s | "
                "shadow sync %.1f ms (%.0f KB, %.0f KB of it new; copying %.1f ms), textures %u (%.1f ms, %.0f KB, %u scaled), images created %u (%.1f ms), "
                "image memory allocations %llu (%.1f ms), shaders %u (%.1f ms), pipelines %u (%.1f ms), submits %llu (%.1f ms), draws %llu (%llu later tiles), resolves %llu, "
                "snapshots %.0f KB | guard: %llu faults, %llu stalls (%.1f ms), %llu mprotect (%.1f ms), %.0f KB read back, "
                "%.0f KB file reads | "
                "game: longest frame %.1f ms, present %.1f ms, D3D waits %.1f ms | GPU executing %.1f ms\n",
                (unsigned long long)frame, frameMs, median, vblanks, busyMs, waitMs, waits[CP_WAIT_THROTTLE], waits[CP_WAIT_REGMEM],
                waits[CP_WAIT_SHADOW], waits[CP_WAIT_FLUSH], waits[CP_WAIT_PIPELINE], waits[CP_WAIT_SLOT], suspended,
                std::max(0.0, frameMs - busyMs - waitMs), ms(now.cpuNs - last.cpuNs), preempted, s_hitch.shadowMs, double(g.uploadBytes - lg.uploadBytes) / 1024,
                double(g.firstUploadBytes - lg.firstUploadBytes) / 1024, ms(g.uploadNs - lg.uploadNs), s_hitch.textures, s_hitch.textureMs, double(s_hitch.textureBytes) / 1024,
                s_hitch.scaledTextures, s_hitch.images, s_hitch.imageMs, (unsigned long long)(now.allocations - last.allocations),
                ms(now.allocationNs - last.allocationNs), s_hitch.shaders, s_hitch.shaderMs, s_hitch.pipelines,
                s_hitch.pipelineMs, (unsigned long long)(now.submits - last.submits), s_hitch.submitMs,
                (unsigned long long)(now.draws - last.draws), (unsigned long long)(now.tileSkips - last.tileSkips),
                (unsigned long long)(now.resolves - last.resolves), double(now.snapshotBytes - last.snapshotBytes) / 1024,
                (unsigned long long)(g.faults - lg.faults), (unsigned long long)(g.stalls - lg.stalls), double(g.stallUs - lg.stallUs) / 1000,
                (unsigned long long)(g.protects - lg.protects), ms(g.protectNs - lg.protectNs), double(g.readbackBytes - lg.readbackBytes) / 1024, double(g.hostWriteBytes - lg.hostWriteBytes) / 1024,
                gameFrameMs, ms(now.gamePresentNs - last.gamePresentNs), ms(now.gameWaitNs - last.gameWaitNs),
                ms(now.gpuExecNs - last.gpuExecNs));
        }
        last = now;
        s_hitch = {};
    }

    namespace
    {
        std::mutex s_perfMutex;
        PerfNumbers s_perfNumbers;
    }

    PerfNumbers GetPerfNumbers()
    {
        std::lock_guard lock(s_perfMutex);
        return s_perfNumbers;
    }

    void PublishPerfNumbers(const PerfNumbers& numbers)
    {
        std::lock_guard lock(s_perfMutex);
        s_perfNumbers = numbers;
    }

    MemoryStats GetMemoryStats()
    {
        MemoryStats m{};
        for (const auto& [key, rt] : s_targets)
        {
            m.targets += rt->image.bytes;
            m.targetCount++;
        }
        for (const auto& [key, t] : s_textures)
        {
            m.textures += t->image.bytes;
            m.textureCount++;
        }
        m.scaled = s_scaled.poolBytes + s_scaled.tableBytes + s_scaled.scratchBytes;
        m.front = s_front.bytes + s_frontScaled.bytes;
        auto own = [&m](const Image& image) {
            if (image.memory && !image.block)
            {
                m.own++;
                m.ownBytes += image.bytes;
            }
        };
        for (const auto& [key, rt] : s_targets)
            own(rt->image);
        for (const auto& [key, t] : s_textures)
            own(t->image);
        for (const Image* image : { &s_front, &s_frontScaled, &s_dummy[0], &s_dummy[1], &s_dummy[2], &s_dummyStorage })
            own(*image);
        for (const auto& block : s_blocks)
        {
            m.blocks++;
            m.blockBytes += block->ranges.Size();
            m.rangeBytes += block->ranges.Used();
            m.ranges += block->ranges.Ranges();
        }
        {
            std::lock_guard lock(s_spareMutex);
            for (const std::vector<Spare>& spares : s_spares)
                m.spares += uint32_t(spares.size());
            m.spareBlocks = s_imageTotals.spareBlocks;
            m.spareNs = s_imageTotals.spareNs;
            m.blocksFreed = s_imageTotals.blocksFreed;
        }
        m.spareBytes = m.spares * s_blockSize;
        m.images = s_imageTotals.images;
        m.imageNs = s_imageTotals.imageNs;
        m.allocations = s_imageTotals.allocations;
        m.allocationNs = s_imageTotals.allocationNs;
        m.fromSpare = s_imageTotals.fromSpare;
        m.blocksHere = s_imageTotals.blocksHere;
        if (s_vk && s_memoryBudget)
        {
            VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT };
            VkPhysicalDeviceMemoryProperties2 props{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &budget };
            vkGetPhysicalDeviceMemoryProperties2(s_vk->physical, &props);
            for (uint32_t h = 0; h < props.memoryProperties.memoryHeapCount; h++)
                if (props.memoryProperties.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
                {
                    m.deviceUsed += budget.heapUsage[h];
                    m.deviceBudget += budget.heapBudget[h];
                }
        }
        return m;
    }

    ScaledStats GetScaledStats()
    {
        return { s_scaled.loads, s_scaled.untiledBytes, s_scaled.resolvedBytes, s_scaled.allocations, s_scaled.evictions,
            s_scaled.fallbacks };
    }

    Stats GetStats()
    {
        s_stats.shaderCacheHits = s_shaderCacheHits.load(std::memory_order_relaxed);
        s_stats.gpuBusyUs = s_gpuBusyUs.load();
        s_stats.gpuExecUs = s_gpuExecNs.load() / 1000;
        return s_stats;
    }
}