diff --git a/src/graphics/texture_override.cc b/src/graphics/texture_override.cc index 6a6ec71..c00c54b 100644 --- a/src/graphics/texture_override.cc +++ b/src/graphics/texture_override.cc @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -82,6 +83,23 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, height == 0 || source_width == 0 || source_height == 0) { return; } + // Source box of destination column x is [columns[2x], columns[2x + 1]). + thread_local std::vector columns; + columns.resize(static_cast(width) * 2); + bool one_texel_columns = true; + for (std::uint32_t x = 0; x < width; ++x) { + const std::uint32_t x0 = static_cast( + static_cast(x) * source_width / width); + const std::uint32_t x1 = std::max( + x0 + 1, static_cast( + (static_cast(x) + 1) * source_width / width)); + columns[2 * static_cast(x)] = x0; + columns[2 * static_cast(x) + 1] = x1; + one_texel_columns = one_texel_columns && x1 == x0 + 1; + } + const std::size_t row_bytes = static_cast(width) * 4; + std::uint64_t previous_y0 = 0; + std::uint64_t previous_y1 = 0; for (std::uint32_t y = 0; y < height; ++y) { const std::uint64_t y0 = static_cast(y) * source_height / height; std::uint64_t y1 = @@ -89,13 +107,25 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, if (y1 <= y0) { y1 = y0 + 1; } - for (std::uint32_t x = 0; x < width; ++x) { - const std::uint64_t x0 = static_cast(x) * source_width / width; - std::uint64_t x1 = - (static_cast(x) + 1) * source_width / width; - if (x1 <= x0) { - x1 = x0 + 1; + std::uint8_t* out_row = destination + y * row_bytes; + if (y > 0 && y0 == previous_y0 && y1 == previous_y1) { + std::memcpy(out_row, out_row - row_bytes, row_bytes); + continue; + } + previous_y0 = y0; + previous_y1 = y1; + if (one_texel_columns && y1 == y0 + 1) { + const std::uint8_t* source_row = source + y0 * source_width * 4; + for (std::uint32_t x = 0; x < width; ++x) { + std::memcpy(out_row + static_cast(x) * 4, + source_row + columns[2 * static_cast(x)] * 4, + 4); } + continue; + } + for (std::uint32_t x = 0; x < width; ++x) { + const std::uint64_t x0 = columns[2 * static_cast(x)]; + const std::uint64_t x1 = columns[2 * static_cast(x) + 1]; std::uint64_t sum[4] = {0, 0, 0, 0}; for (std::uint64_t sy = y0; sy < y1; ++sy) { for (std::uint64_t sx = x0; sx < x1; ++sx) { diff --git a/src/graphics/vulkan_etc2_emulation.cc b/src/graphics/vulkan_etc2_emulation.cc index aa30ff8..2e5cb2e 100644 --- a/src/graphics/vulkan_etc2_emulation.cc +++ b/src/graphics/vulkan_etc2_emulation.cc @@ -200,12 +200,24 @@ bool IsColorFormat(EtcFormat format) { format == EtcFormat::kEtc2Rgba8; } +// A host-visible transfer buffer that stays mapped for its whole life. struct Staging { VkDevice device = VK_NULL_HANDLE; VkBuffer buffer = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize capacity = 0; + std::uint8_t* mapped = nullptr; }; +// Idle staging buffers kept for reuse. Creating, allocating, binding and +// mapping one costs hundreds of microseconds in the driver, so a burst of +// texture uploads would otherwise stall the frame that records it. +constexpr VkDeviceSize kMaxIdleStagingBytes = 64 * 1024 * 1024; +constexpr std::size_t kMaxIdleStagingBuffers = 64; +// A request may take an idle buffer up to this size, or up to 4x its own +// size when larger, so small uploads share buffers without pinning big ones. +constexpr VkDeviceSize kStagingReuseSlack = 64 * 1024; + struct PendingUpload { VkDevice device = VK_NULL_HANDLE; VkBuffer source = VK_NULL_HANDLE; @@ -252,6 +264,7 @@ void LogFailure(const char* message) { bool CreateStaging(const HostDevice& dev, VkDeviceSize size, Staging* staging, std::uint8_t** mapped) { + *mapped = nullptr; if (dev.create_buffer == nullptr || dev.allocate_memory == nullptr || dev.get_buffer_memory_requirements == nullptr || dev.bind_buffer_memory == nullptr || dev.map_memory == nullptr || @@ -301,8 +314,8 @@ bool CreateStaging(const HostDevice& dev, VkDeviceSize size, Staging* staging, dev.free_memory(dev.device, memory, nullptr); return false; } - *staging = {dev.device, buffer, memory}; *mapped = static_cast(data); + *staging = {dev.device, buffer, memory, size, *mapped}; return true; } @@ -466,6 +479,8 @@ struct VulkanEtc2Emulation::State { std::unordered_map mappings; std::unordered_map commands; std::atomic has_commands{false}; + std::vector idle_staging; + VkDeviceSize idle_staging_bytes = 0; // Heap buffers reused across submits; they only grow. std::mutex scratch_mutex; std::vector scratch_source; @@ -563,6 +578,55 @@ struct VulkanEtc2Emulation::State { } } + // Takes the smallest idle buffer that fits, or creates one. + bool AcquireStaging(const HostDevice& dev, VkDeviceSize size, + Staging* staging) { + { + std::lock_guard lock(mutex); + const VkDeviceSize limit = std::max(size * 4, kStagingReuseSlack); + auto best = idle_staging.end(); + for (auto it = idle_staging.begin(); it != idle_staging.end(); ++it) { + if (it->device == dev.device && it->capacity >= size && + it->capacity <= limit && + (best == idle_staging.end() || it->capacity < best->capacity)) { + best = it; + } + } + if (best != idle_staging.end()) { + *staging = *best; + idle_staging_bytes -= best->capacity; + *best = idle_staging.back(); + idle_staging.pop_back(); + return true; + } + } + std::uint8_t* mapped = nullptr; + return CreateStaging(dev, size, staging, &mapped); + } + + // Keeps released buffers for reuse up to the idle limits and destroys + // the rest. + void ReleaseStaging(std::vector released) { + std::vector doomed; + { + std::lock_guard lock(mutex); + for (const Staging& staging : released) { + if (idle_staging.size() < kMaxIdleStagingBuffers && + idle_staging_bytes + staging.capacity <= kMaxIdleStagingBytes) { + idle_staging.push_back(staging); + idle_staging_bytes += staging.capacity; + } else { + doomed.push_back(staging); + } + } + } + for (const Staging& staging : doomed) { + if (const HostDevice* dev = Find(staging.device); dev != nullptr) { + DestroyStaging(*dev, staging); + } + } + } + const HostDevice* Find(VkDevice device) { std::lock_guard lock(devices_mutex); for (const auto& candidate : devices) { @@ -708,6 +772,16 @@ void VulkanEtc2Emulation::DestroyDevice(VkDevice device) { ++it; } } + for (auto it = state_->idle_staging.begin(); + it != state_->idle_staging.end();) { + if (it->device == device) { + state_->idle_staging_bytes -= it->capacity; + doomed.push_back(*it); + it = state_->idle_staging.erase(it); + } else { + ++it; + } + } for (auto it = state_->images.begin(); it != state_->images.end();) { it = it->second.device == device ? state_->images.erase(it) : std::next(it); } @@ -944,18 +1018,17 @@ void VulkanEtc2Emulation::CmdCopyBufferToImage( std::vector uploads; VkDeviceSize total = 0; Staging staging; - std::uint8_t* mapped = nullptr; if (!PlanRegions(record, device, source, destination, region_count, regions, &rewritten, &uploads, &total)) { LogFailure("skipped an upload with an unsupported region"); return; } - if (!CreateStaging(*dev, total, &staging, &mapped)) { + if (!state_->AcquireStaging(*dev, total, &staging)) { LogFailure("could not allocate a host-visible staging buffer"); return; } for (PendingUpload& upload : uploads) { - upload.target = mapped + upload.target_offset; + upload.target = staging.mapped + upload.target_offset; } state_->Record(command_buffer, std::move(uploads), staging); dev->copy_buffer_to_image(command_buffer, staging.buffer, destination, @@ -979,19 +1052,18 @@ void VulkanEtc2Emulation::CmdCopyBufferToImage2( std::vector uploads; VkDeviceSize total = 0; Staging staging; - std::uint8_t* mapped = nullptr; if (!PlanRegions(record, device, info->srcBuffer, info->dstImage, info->regionCount, info->pRegions, &rewritten, &uploads, &total)) { LogFailure("skipped an upload with an unsupported region"); return; } - if (!CreateStaging(*dev, total, &staging, &mapped)) { + if (!state_->AcquireStaging(*dev, total, &staging)) { LogFailure("could not allocate a host-visible staging buffer"); return; } for (PendingUpload& upload : uploads) { - upload.target = mapped + upload.target_offset; + upload.target = staging.mapped + upload.target_offset; } state_->Record(command_buffer, std::move(uploads), staging); VkCopyBufferToImageInfo2 host_info = *info; @@ -1171,6 +1243,8 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers, } } } + TraceScope trace_scope(work.empty() ? nullptr : ActiveProfileTrace(), + "etc2 decode", "texture"); // Mapped Vulkan memory is uncached or write-combined, which makes the // decoder's scattered byte reads and writes tens of times slower than on // heap memory. Compressed bytes are streamed into a heap scratch buffer, @@ -1218,6 +1292,10 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers, compressed_total += upload.compressed; decoded_total += upload.decoded; } + trace_scope.Arg("uploads", static_cast(copies.size())); + trace_scope.Arg("compressed_bytes", + static_cast(compressed_total)); + trace_scope.Arg("decoded_bytes", static_cast(decoded_total)); std::lock_guard scratch_lock(state_->scratch_mutex); std::vector& scratch_source = state_->scratch_source; @@ -1243,8 +1321,6 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers, EtcDecodeJob job; job.format = upload.format; job.source = source + layer * compressed_layer; - TraceScope trace_scope(work.empty() ? nullptr : ActiveProfileTrace(), - "etc2 decode", "texture"); job.source_bytes = compressed_layer; job.width = upload.width; job.height = upload.height; @@ -1292,20 +1368,12 @@ void VulkanEtc2Emulation::ReleaseCommandBuffer(VkCommandBuffer command_buffer) { if (record == state_->commands.end()) { return; } - trace_scope.Arg("uploads", static_cast(copies.size())); - trace_scope.Arg("compressed_bytes", - static_cast(compressed_total)); - trace_scope.Arg("decoded_bytes", static_cast(decoded_total)); doomed = std::move(record->second.staging); state_->commands.erase(record); state_->has_commands.store(!state_->commands.empty(), std::memory_order_release); } - for (const Staging& staging : doomed) { - if (const HostDevice* dev = state_->Find(staging.device); dev != nullptr) { - DestroyStaging(*dev, staging); - } - } + state_->ReleaseStaging(std::move(doomed)); } } // namespace mocktail::graphics diff --git a/tests/texture_override_test.cc b/tests/texture_override_test.cc index 79af149..d2552e0 100644 --- a/tests/texture_override_test.cc +++ b/tests/texture_override_test.cc @@ -98,6 +98,56 @@ TEST(TextureOverrideTest, ResampleAveragesSourceBoxes) { } } +// Rounded mean of the source box [x*sw/w, (x+1)*sw/w) per axis, at least one +// texel wide. +std::vector ReferenceBoxResample(const RgbaImage& source, + std::uint32_t width, + std::uint32_t height) { + std::vector out(static_cast(width) * height * 4); + const auto span = [](std::uint32_t i, std::uint32_t from, std::uint32_t to) { + const std::uint64_t first = static_cast(i) * from / to; + const std::uint64_t last = + std::max(first + 1, (i + 1ULL) * from / to); + return std::pair{first, last}; + }; + for (std::uint32_t y = 0; y < height; ++y) { + const auto [y0, y1] = span(y, source.height, height); + for (std::uint32_t x = 0; x < width; ++x) { + const auto [x0, x1] = span(x, source.width, width); + for (int channel = 0; channel < 4; ++channel) { + std::uint64_t sum = 0; + for (std::uint64_t sy = y0; sy < y1; ++sy) { + for (std::uint64_t sx = x0; sx < x1; ++sx) { + sum += source.pixels[(sy * source.width + sx) * 4 + channel]; + } + } + const std::uint64_t count = (y1 - y0) * (x1 - x0); + out[(static_cast(y) * width + x) * 4 + channel] = + static_cast((sum + count / 2) / count); + } + } + } + return out; +} + +TEST(TextureOverrideTest, ResampleMatchesBoxFilterForMixedSizes) { + const std::pair sources[] = { + {1, 1}, {3, 5}, {48, 48}, {60, 33}, {64, 17}}; + const std::pair targets[] = { + {1, 1}, {2, 7}, {5, 3}, {48, 48}, {192, 192}, {240, 132}, {31, 200}}; + for (const auto& [source_width, source_height] : sources) { + const RgbaImage source = Gradient(source_width, source_height); + for (const auto& [width, height] : targets) { + std::vector actual( + static_cast(width) * height * 4, 0); + ResampleRgba(source, width, height, actual.data()); + ASSERT_EQ(actual, ReferenceBoxResample(source, width, height)) + << source_width << "x" << source_height << " -> " << width << "x" + << height; + } + } +} + TEST(TextureOverrideTest, LookupFindsPngNamedByHashAndCachesMisses) { TempDir dir; ASSERT_FALSE(dir.path().empty()); diff --git a/tests/vulkan_etc2_emulation_test.cc b/tests/vulkan_etc2_emulation_test.cc index 7a200e8..9eda213 100644 --- a/tests/vulkan_etc2_emulation_test.cc +++ b/tests/vulkan_etc2_emulation_test.cc @@ -127,6 +127,10 @@ VkExtent3D g_copied_extent{}; VkDeviceSize g_created_buffer_size = 0; std::uintptr_t g_next_image = 0x100; std::uint32_t g_copy_image_calls = 0; +std::uint32_t g_create_buffer_calls = 0; +std::uint32_t g_allocate_memory_calls = 0; +std::uint32_t g_destroy_buffer_calls = 0; +std::uint32_t g_free_memory_calls = 0; VkImageBlit g_blit{}; VkFilter g_blit_filter = VK_FILTER_NEAREST; @@ -159,6 +163,7 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeCreateBuffer(VkDevice, const VkAllocationCallbacks*, VkBuffer* buffer) { g_created_buffer_size = info->size; + ++g_create_buffer_calls; *buffer = FakeHandle(kStagingBuffer); return VK_SUCCESS; } @@ -173,6 +178,7 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeAllocateMemory( VkDevice, const VkMemoryAllocateInfo*, const VkAllocationCallbacks*, VkDeviceMemory* memory) { *memory = FakeHandle(kStagingMemory); + ++g_allocate_memory_calls; return VK_SUCCESS; } @@ -193,9 +199,13 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeMapMemory(VkDevice, VkDeviceMemory memory, VKAPI_ATTR void VKAPI_CALL FakeUnmapMemory(VkDevice, VkDeviceMemory) {} VKAPI_ATTR void VKAPI_CALL FakeDestroyBuffer(VkDevice, VkBuffer, - const VkAllocationCallbacks*) {} + const VkAllocationCallbacks*) { + ++g_destroy_buffer_calls; +} VKAPI_ATTR void VKAPI_CALL FakeFreeMemory(VkDevice, VkDeviceMemory, - const VkAllocationCallbacks*) {} + const VkAllocationCallbacks*) { + ++g_free_memory_calls; +} VKAPI_ATTR void VKAPI_CALL FakeCopyBufferToImage( VkCommandBuffer, VkBuffer, VkImage, VkImageLayout, std::uint32_t count, const VkBufferImageCopy* regions) { @@ -528,6 +538,60 @@ TEST(VulkanEtc2EmulationTest, FillsScaledMipsFromLevelZero) { emulation.ReleaseCommandBuffer(command_buffer); } +TEST(VulkanEtc2EmulationTest, ReusesStagingBuffersAcrossCommandBuffers) { + const VkDevice device = FakeHandle(0x17); + const VkBuffer source = FakeHandle(0x207); + const VkDeviceMemory source_memory = FakeHandle(kSourceMemory); + g_source.fill(0); + + VkPhysicalDeviceMemoryProperties memory{}; + memory.memoryTypeCount = 1; + memory.memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + VulkanEtc2Emulation emulation; + emulation.RegisterDevice(device, FakeHandle(0x27), true, + memory, FakeGetDeviceProcAddr); + + VkImageCreateInfo image_info{}; + image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + image_info.format = VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK; + image_info.extent = {4, 4, 1}; + VkImage image = VK_NULL_HANDLE; + ASSERT_EQ(emulation.CreateImage(device, &image_info, nullptr, &image), + VK_SUCCESS); + ASSERT_EQ(emulation.BindBufferMemory(device, source, source_memory, 0), + VK_SUCCESS); + void* mapped = nullptr; + ASSERT_EQ(emulation.MapMemory(device, source_memory, 0, g_source.size(), 0, + &mapped), + VK_SUCCESS); + + g_create_buffer_calls = 0; + g_allocate_memory_calls = 0; + g_destroy_buffer_calls = 0; + g_free_memory_calls = 0; + VkBufferImageCopy region{}; + region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + region.imageSubresource.layerCount = 1; + region.imageExtent = {4, 4, 1}; + for (std::uintptr_t frame = 0; frame < 8; ++frame) { + const VkCommandBuffer command_buffer = + FakeHandle(0x780 + frame); + emulation.CmdCopyBufferToImage(device, command_buffer, source, image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, + ®ion); + emulation.PrepareSubmit(&command_buffer, 1); + emulation.ReleaseCommandBuffer(command_buffer); + } + EXPECT_EQ(g_create_buffer_calls, 1u); + EXPECT_EQ(g_allocate_memory_calls, 1u); + EXPECT_EQ(g_destroy_buffer_calls, 0u); + + emulation.DestroyDevice(device); + EXPECT_EQ(g_destroy_buffer_calls, 1u); + EXPECT_EQ(g_free_memory_calls, 1u); +} + // Copies out of a scaled image cover its host bounds and shrink back onto // an unscaled destination, which a plain copy cannot do. TEST(VulkanEtc2EmulationTest, BlitsCopiesThatTouchScaledImages) {