From 579cedbf1dfa7da8460720582e759a80203abfea Mon Sep 17 00:00:00 2001 From: vii Date: Wed, 16 Sep 2026 21:22:31 -0500 Subject: [PATCH] perf(textures): pool ETC2 staging buffers and speed up resampling Staging buffers stay mapped and are reused instead of allocated per upload. Box resampling precomputes column spans and copies repeated rows, with identical output. --- src/graphics/texture_override.cc | 42 +++++++++-- src/graphics/vulkan_etc2_emulation.cc | 104 +++++++++++++++++++++----- tests/texture_override_test.cc | 50 +++++++++++++ tests/vulkan_etc2_emulation_test.cc | 68 ++++++++++++++++- 4 files changed, 238 insertions(+), 26 deletions(-) diff --git a/src/graphics/texture_override.cc b/src/graphics/texture_override.cc index 6a6ec71..c00c54b 100644 --- a/src/graphics/texture_override.cc +++ b/src/graphics/texture_override.cc @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -82,6 +83,23 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, height == 0 || source_width == 0 || source_height == 0) { return; } + // Source box of destination column x is [columns[2x], columns[2x + 1]). + thread_local std::vector columns; + columns.resize(static_cast(width) * 2); + bool one_texel_columns = true; + for (std::uint32_t x = 0; x < width; ++x) { + const std::uint32_t x0 = static_cast( + static_cast(x) * source_width / width); + const std::uint32_t x1 = std::max( + x0 + 1, static_cast( + (static_cast(x) + 1) * source_width / width)); + columns[2 * static_cast(x)] = x0; + columns[2 * static_cast(x) + 1] = x1; + one_texel_columns = one_texel_columns && x1 == x0 + 1; + } + const std::size_t row_bytes = static_cast(width) * 4; + std::uint64_t previous_y0 = 0; + std::uint64_t previous_y1 = 0; for (std::uint32_t y = 0; y < height; ++y) { const std::uint64_t y0 = static_cast(y) * source_height / height; std::uint64_t y1 = @@ -89,13 +107,25 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, if (y1 <= y0) { y1 = y0 + 1; } - for (std::uint32_t x = 0; x < width; ++x) { - const std::uint64_t x0 = static_cast(x) * source_width / width; - std::uint64_t x1 = - (static_cast(x) + 1) * source_width / width; - if (x1 <= x0) { - x1 = x0 + 1; + std::uint8_t* out_row = destination + y * row_bytes; + if (y > 0 && y0 == previous_y0 && y1 == previous_y1) { + std::memcpy(out_row, out_row - row_bytes, row_bytes); + continue; + } + previous_y0 = y0; + previous_y1 = y1; + if (one_texel_columns && y1 == y0 + 1) { + const std::uint8_t* source_row = source + y0 * source_width * 4; + for (std::uint32_t x = 0; x < width; ++x) { + std::memcpy(out_row + static_cast(x) * 4, + source_row + columns[2 * static_cast(x)] * 4, + 4); } + continue; + } + for (std::uint32_t x = 0; x < width; ++x) { + const std::uint64_t x0 = columns[2 * static_cast(x)]; + const std::uint64_t x1 = columns[2 * static_cast(x) + 1]; std::uint64_t sum[4] = {0, 0, 0, 0}; for (std::uint64_t sy = y0; sy < y1; ++sy) { for (std::uint64_t sx = x0; sx < x1; ++sx) { diff --git a/src/graphics/vulkan_etc2_emulation.cc b/src/graphics/vulkan_etc2_emulation.cc index aa30ff8..2e5cb2e 100644 --- a/src/graphics/vulkan_etc2_emulation.cc +++ b/src/graphics/vulkan_etc2_emulation.cc @@ -200,12 +200,24 @@ bool IsColorFormat(EtcFormat format) { format == EtcFormat::kEtc2Rgba8; } +// A host-visible transfer buffer that stays mapped for its whole life. struct Staging { VkDevice device = VK_NULL_HANDLE; VkBuffer buffer = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize capacity = 0; + std::uint8_t* mapped = nullptr; }; +// Idle staging buffers kept for reuse. Creating, allocating, binding and +// mapping one costs hundreds of microseconds in the driver, so a burst of +// texture uploads would otherwise stall the frame that records it. +constexpr VkDeviceSize kMaxIdleStagingBytes = 64 * 1024 * 1024; +constexpr std::size_t kMaxIdleStagingBuffers = 64; +// A request may take an idle buffer up to this size, or up to 4x its own +// size when larger, so small uploads share buffers without pinning big ones. +constexpr VkDeviceSize kStagingReuseSlack = 64 * 1024; + struct PendingUpload { VkDevice device = VK_NULL_HANDLE; VkBuffer source = VK_NULL_HANDLE; @@ -252,6 +264,7 @@ void LogFailure(const char* message) { bool CreateStaging(const HostDevice& dev, VkDeviceSize size, Staging* staging, std::uint8_t** mapped) { + *mapped = nullptr; if (dev.create_buffer == nullptr || dev.allocate_memory == nullptr || dev.get_buffer_memory_requirements == nullptr || dev.bind_buffer_memory == nullptr || dev.map_memory == nullptr || @@ -301,8 +314,8 @@ bool CreateStaging(const HostDevice& dev, VkDeviceSize size, Staging* staging, dev.free_memory(dev.device, memory, nullptr); return false; } - *staging = {dev.device, buffer, memory}; *mapped = static_cast(data); + *staging = {dev.device, buffer, memory, size, *mapped}; return true; } @@ -466,6 +479,8 @@ struct VulkanEtc2Emulation::State { std::unordered_map mappings; std::unordered_map commands; std::atomic has_commands{false}; + std::vector idle_staging; + VkDeviceSize idle_staging_bytes = 0; // Heap buffers reused across submits; they only grow. std::mutex scratch_mutex; std::vector scratch_source; @@ -563,6 +578,55 @@ struct VulkanEtc2Emulation::State { } } + // Takes the smallest idle buffer that fits, or creates one. + bool AcquireStaging(const HostDevice& dev, VkDeviceSize size, + Staging* staging) { + { + std::lock_guard lock(mutex); + const VkDeviceSize limit = std::max(size * 4, kStagingReuseSlack); + auto best = idle_staging.end(); + for (auto it = idle_staging.begin(); it != idle_staging.end(); ++it) { + if (it->device == dev.device && it->capacity >= size && + it->capacity <= limit && + (best == idle_staging.end() || it->capacity < best->capacity)) { + best = it; + } + } + if (best != idle_staging.end()) { + *staging = *best; + idle_staging_bytes -= best->capacity; + *best = idle_staging.back(); + idle_staging.pop_back(); + return true; + } + } + std::uint8_t* mapped = nullptr; + return CreateStaging(dev, size, staging, &mapped); + } + + // Keeps released buffers for reuse up to the idle limits and destroys + // the rest. + void ReleaseStaging(std::vector released) { + std::vector doomed; + { + std::lock_guard lock(mutex); + for (const Staging& staging : released) { + if (idle_staging.size() < kMaxIdleStagingBuffers && + idle_staging_bytes + staging.capacity <= kMaxIdleStagingBytes) { + idle_staging.push_back(staging); + idle_staging_bytes += staging.capacity; + } else { + doomed.push_back(staging); + } + } + } + for (const Staging& staging : doomed) { + if (const HostDevice* dev = Find(staging.device); dev != nullptr) { + DestroyStaging(*dev, staging); + } + } + } + const HostDevice* Find(VkDevice device) { std::lock_guard lock(devices_mutex); for (const auto& candidate : devices) { @@ -708,6 +772,16 @@ void VulkanEtc2Emulation::DestroyDevice(VkDevice device) { ++it; } } + for (auto it = state_->idle_staging.begin(); + it != state_->idle_staging.end();) { + if (it->device == device) { + state_->idle_staging_bytes -= it->capacity; + doomed.push_back(*it); + it = state_->idle_staging.erase(it); + } else { + ++it; + } + } for (auto it = state_->images.begin(); it != state_->images.end();) { it = it->second.device == device ? state_->images.erase(it) : std::next(it); } @@ -944,18 +1018,17 @@ void VulkanEtc2Emulation::CmdCopyBufferToImage( std::vector uploads; VkDeviceSize total = 0; Staging staging; - std::uint8_t* mapped = nullptr; if (!PlanRegions(record, device, source, destination, region_count, regions, &rewritten, &uploads, &total)) { LogFailure("skipped an upload with an unsupported region"); return; } - if (!CreateStaging(*dev, total, &staging, &mapped)) { + if (!state_->AcquireStaging(*dev, total, &staging)) { LogFailure("could not allocate a host-visible staging buffer"); return; } for (PendingUpload& upload : uploads) { - upload.target = mapped + upload.target_offset; + upload.target = staging.mapped + upload.target_offset; } state_->Record(command_buffer, std::move(uploads), staging); dev->copy_buffer_to_image(command_buffer, staging.buffer, destination, @@ -979,19 +1052,18 @@ void VulkanEtc2Emulation::CmdCopyBufferToImage2( std::vector uploads; VkDeviceSize total = 0; Staging staging; - std::uint8_t* mapped = nullptr; if (!PlanRegions(record, device, info->srcBuffer, info->dstImage, info->regionCount, info->pRegions, &rewritten, &uploads, &total)) { LogFailure("skipped an upload with an unsupported region"); return; } - if (!CreateStaging(*dev, total, &staging, &mapped)) { + if (!state_->AcquireStaging(*dev, total, &staging)) { LogFailure("could not allocate a host-visible staging buffer"); return; } for (PendingUpload& upload : uploads) { - upload.target = mapped + upload.target_offset; + upload.target = staging.mapped + upload.target_offset; } state_->Record(command_buffer, std::move(uploads), staging); VkCopyBufferToImageInfo2 host_info = *info; @@ -1171,6 +1243,8 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers, } } } + TraceScope trace_scope(work.empty() ? nullptr : ActiveProfileTrace(), + "etc2 decode", "texture"); // Mapped Vulkan memory is uncached or write-combined, which makes the // decoder's scattered byte reads and writes tens of times slower than on // heap memory. Compressed bytes are streamed into a heap scratch buffer, @@ -1218,6 +1292,10 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers, compressed_total += upload.compressed; decoded_total += upload.decoded; } + trace_scope.Arg("uploads", static_cast(copies.size())); + trace_scope.Arg("compressed_bytes", + static_cast(compressed_total)); + trace_scope.Arg("decoded_bytes", static_cast(decoded_total)); std::lock_guard scratch_lock(state_->scratch_mutex); std::vector& scratch_source = state_->scratch_source; @@ -1243,8 +1321,6 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers, EtcDecodeJob job; job.format = upload.format; job.source = source + layer * compressed_layer; - TraceScope trace_scope(work.empty() ? nullptr : ActiveProfileTrace(), - "etc2 decode", "texture"); job.source_bytes = compressed_layer; job.width = upload.width; job.height = upload.height; @@ -1292,20 +1368,12 @@ void VulkanEtc2Emulation::ReleaseCommandBuffer(VkCommandBuffer command_buffer) { if (record == state_->commands.end()) { return; } - trace_scope.Arg("uploads", static_cast(copies.size())); - trace_scope.Arg("compressed_bytes", - static_cast(compressed_total)); - trace_scope.Arg("decoded_bytes", static_cast(decoded_total)); doomed = std::move(record->second.staging); state_->commands.erase(record); state_->has_commands.store(!state_->commands.empty(), std::memory_order_release); } - for (const Staging& staging : doomed) { - if (const HostDevice* dev = state_->Find(staging.device); dev != nullptr) { - DestroyStaging(*dev, staging); - } - } + state_->ReleaseStaging(std::move(doomed)); } } // namespace mocktail::graphics diff --git a/tests/texture_override_test.cc b/tests/texture_override_test.cc index 79af149..d2552e0 100644 --- a/tests/texture_override_test.cc +++ b/tests/texture_override_test.cc @@ -98,6 +98,56 @@ TEST(TextureOverrideTest, ResampleAveragesSourceBoxes) { } } +// Rounded mean of the source box [x*sw/w, (x+1)*sw/w) per axis, at least one +// texel wide. +std::vector ReferenceBoxResample(const RgbaImage& source, + std::uint32_t width, + std::uint32_t height) { + std::vector out(static_cast(width) * height * 4); + const auto span = [](std::uint32_t i, std::uint32_t from, std::uint32_t to) { + const std::uint64_t first = static_cast(i) * from / to; + const std::uint64_t last = + std::max(first + 1, (i + 1ULL) * from / to); + return std::pair{first, last}; + }; + for (std::uint32_t y = 0; y < height; ++y) { + const auto [y0, y1] = span(y, source.height, height); + for (std::uint32_t x = 0; x < width; ++x) { + const auto [x0, x1] = span(x, source.width, width); + for (int channel = 0; channel < 4; ++channel) { + std::uint64_t sum = 0; + for (std::uint64_t sy = y0; sy < y1; ++sy) { + for (std::uint64_t sx = x0; sx < x1; ++sx) { + sum += source.pixels[(sy * source.width + sx) * 4 + channel]; + } + } + const std::uint64_t count = (y1 - y0) * (x1 - x0); + out[(static_cast(y) * width + x) * 4 + channel] = + static_cast((sum + count / 2) / count); + } + } + } + return out; +} + +TEST(TextureOverrideTest, ResampleMatchesBoxFilterForMixedSizes) { + const std::pair sources[] = { + {1, 1}, {3, 5}, {48, 48}, {60, 33}, {64, 17}}; + const std::pair targets[] = { + {1, 1}, {2, 7}, {5, 3}, {48, 48}, {192, 192}, {240, 132}, {31, 200}}; + for (const auto& [source_width, source_height] : sources) { + const RgbaImage source = Gradient(source_width, source_height); + for (const auto& [width, height] : targets) { + std::vector actual( + static_cast(width) * height * 4, 0); + ResampleRgba(source, width, height, actual.data()); + ASSERT_EQ(actual, ReferenceBoxResample(source, width, height)) + << source_width << "x" << source_height << " -> " << width << "x" + << height; + } + } +} + TEST(TextureOverrideTest, LookupFindsPngNamedByHashAndCachesMisses) { TempDir dir; ASSERT_FALSE(dir.path().empty()); diff --git a/tests/vulkan_etc2_emulation_test.cc b/tests/vulkan_etc2_emulation_test.cc index 7a200e8..9eda213 100644 --- a/tests/vulkan_etc2_emulation_test.cc +++ b/tests/vulkan_etc2_emulation_test.cc @@ -127,6 +127,10 @@ VkExtent3D g_copied_extent{}; VkDeviceSize g_created_buffer_size = 0; std::uintptr_t g_next_image = 0x100; std::uint32_t g_copy_image_calls = 0; +std::uint32_t g_create_buffer_calls = 0; +std::uint32_t g_allocate_memory_calls = 0; +std::uint32_t g_destroy_buffer_calls = 0; +std::uint32_t g_free_memory_calls = 0; VkImageBlit g_blit{}; VkFilter g_blit_filter = VK_FILTER_NEAREST; @@ -159,6 +163,7 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeCreateBuffer(VkDevice, const VkAllocationCallbacks*, VkBuffer* buffer) { g_created_buffer_size = info->size; + ++g_create_buffer_calls; *buffer = FakeHandle(kStagingBuffer); return VK_SUCCESS; } @@ -173,6 +178,7 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeAllocateMemory( VkDevice, const VkMemoryAllocateInfo*, const VkAllocationCallbacks*, VkDeviceMemory* memory) { *memory = FakeHandle(kStagingMemory); + ++g_allocate_memory_calls; return VK_SUCCESS; } @@ -193,9 +199,13 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeMapMemory(VkDevice, VkDeviceMemory memory, VKAPI_ATTR void VKAPI_CALL FakeUnmapMemory(VkDevice, VkDeviceMemory) {} VKAPI_ATTR void VKAPI_CALL FakeDestroyBuffer(VkDevice, VkBuffer, - const VkAllocationCallbacks*) {} + const VkAllocationCallbacks*) { + ++g_destroy_buffer_calls; +} VKAPI_ATTR void VKAPI_CALL FakeFreeMemory(VkDevice, VkDeviceMemory, - const VkAllocationCallbacks*) {} + const VkAllocationCallbacks*) { + ++g_free_memory_calls; +} VKAPI_ATTR void VKAPI_CALL FakeCopyBufferToImage( VkCommandBuffer, VkBuffer, VkImage, VkImageLayout, std::uint32_t count, const VkBufferImageCopy* regions) { @@ -528,6 +538,60 @@ TEST(VulkanEtc2EmulationTest, FillsScaledMipsFromLevelZero) { emulation.ReleaseCommandBuffer(command_buffer); } +TEST(VulkanEtc2EmulationTest, ReusesStagingBuffersAcrossCommandBuffers) { + const VkDevice device = FakeHandle(0x17); + const VkBuffer source = FakeHandle(0x207); + const VkDeviceMemory source_memory = FakeHandle(kSourceMemory); + g_source.fill(0); + + VkPhysicalDeviceMemoryProperties memory{}; + memory.memoryTypeCount = 1; + memory.memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + VulkanEtc2Emulation emulation; + emulation.RegisterDevice(device, FakeHandle(0x27), true, + memory, FakeGetDeviceProcAddr); + + VkImageCreateInfo image_info{}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + image_info.format = VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK; + image_info.extent = {4, 4, 1}; + VkImage image = VK_NULL_HANDLE; + ASSERT_EQ(emulation.CreateImage(device, &image_info, nullptr, &image), + VK_SUCCESS); + ASSERT_EQ(emulation.BindBufferMemory(device, source, source_memory, 0), + VK_SUCCESS); + void* mapped = nullptr; + ASSERT_EQ(emulation.MapMemory(device, source_memory, 0, g_source.size(), 0, + &mapped), + VK_SUCCESS); + + g_create_buffer_calls = 0; + g_allocate_memory_calls = 0; + g_destroy_buffer_calls = 0; + g_free_memory_calls = 0; + VkBufferImageCopy region{}; + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.imageSubresource.layerCount = 1; + region.imageExtent = {4, 4, 1}; + for (std::uintptr_t frame = 0; frame < 8; ++frame) { + const VkCommandBuffer command_buffer = + FakeHandle(0x780 + frame); + emulation.CmdCopyBufferToImage(device, command_buffer, source, image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, + ®ion); + emulation.PrepareSubmit(&command_buffer, 1); + emulation.ReleaseCommandBuffer(command_buffer); + } + EXPECT_EQ(g_create_buffer_calls, 1u); + EXPECT_EQ(g_allocate_memory_calls, 1u); + EXPECT_EQ(g_destroy_buffer_calls, 0u); + + emulation.DestroyDevice(device); + EXPECT_EQ(g_destroy_buffer_calls, 1u); + EXPECT_EQ(g_free_memory_calls, 1u); +} + // Copies out of a scaled image cover its host bounds and shrink back onto // an unscaled destination, which a plain copy cannot do. TEST(VulkanEtc2EmulationTest, BlitsCopiesThatTouchScaledImages) {