perf(textures): pool ETC2 staging buffers and speed up resampling

Staging buffers stay mapped and are reused instead of allocated per upload. Box resampling precomputes column spans and copies repeated rows, with identical output.

src/graphics/texture_override.cc

@@ -2,6 +2,7 @@
 
 #include <png.h>
 
+#include <algorithm>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
@@ -82,6 +83,23 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width,
       height == 0 || source_width == 0 || source_height == 0) {
     return;
   }
+  // Source box of destination column x is [columns[2x], columns[2x + 1]).
+  thread_local std::vector<std::uint32_t> columns;
+  columns.resize(static_cast<std::size_t>(width) * 2);
+  bool one_texel_columns = true;
+  for (std::uint32_t x = 0; x < width; ++x) {
+    const std::uint32_t x0 = static_cast<std::uint32_t>(
+        static_cast<std::uint64_t>(x) * source_width / width);
+    const std::uint32_t x1 = std::max<std::uint32_t>(
+        x0 + 1, static_cast<std::uint32_t>(
+                    (static_cast<std::uint64_t>(x) + 1) * source_width / width));
+    columns[2 * static_cast<std::size_t>(x)] = x0;
+    columns[2 * static_cast<std::size_t>(x) + 1] = x1;
+    one_texel_columns = one_texel_columns && x1 == x0 + 1;
+  }
+  const std::size_t row_bytes = static_cast<std::size_t>(width) * 4;
+  std::uint64_t previous_y0 = 0;
+  std::uint64_t previous_y1 = 0;
   for (std::uint32_t y = 0; y < height; ++y) {
     const std::uint64_t y0 = static_cast<std::uint64_t>(y) * source_height / height;
     std::uint64_t y1 =
@@ -89,13 +107,25 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width,
     if (y1 <= y0) {
       y1 = y0 + 1;
     }
-    for (std::uint32_t x = 0; x < width; ++x) {
-      const std::uint64_t x0 = static_cast<std::uint64_t>(x) * source_width / width;
-      std::uint64_t x1 =
-          (static_cast<std::uint64_t>(x) + 1) * source_width / width;
-      if (x1 <= x0) {
-        x1 = x0 + 1;
+    std::uint8_t* out_row = destination + y * row_bytes;
+    if (y > 0 && y0 == previous_y0 && y1 == previous_y1) {
+      std::memcpy(out_row, out_row - row_bytes, row_bytes);
+      continue;
+    }
+    previous_y0 = y0;
+    previous_y1 = y1;
+    if (one_texel_columns && y1 == y0 + 1) {
+      const std::uint8_t* source_row = source + y0 * source_width * 4;
+      for (std::uint32_t x = 0; x < width; ++x) {
+        std::memcpy(out_row + static_cast<std::size_t>(x) * 4,
+                    source_row + columns[2 * static_cast<std::size_t>(x)] * 4,
+                    4);
       }
+      continue;
+    }
+    for (std::uint32_t x = 0; x < width; ++x) {
+      const std::uint64_t x0 = columns[2 * static_cast<std::size_t>(x)];
+      const std::uint64_t x1 = columns[2 * static_cast<std::size_t>(x) + 1];
       std::uint64_t sum[4] = {0, 0, 0, 0};
       for (std::uint64_t sy = y0; sy < y1; ++sy) {
         for (std::uint64_t sx = x0; sx < x1; ++sx) {

src/graphics/vulkan_etc2_emulation.cc

@@ -200,12 +200,24 @@ bool IsColorFormat(EtcFormat format) {
          format == EtcFormat::kEtc2Rgba8;
 }
 
+// A host-visible transfer buffer that stays mapped for its whole life.
 struct Staging {
   VkDevice device = VK_NULL_HANDLE;
   VkBuffer buffer = VK_NULL_HANDLE;
   VkDeviceMemory memory = VK_NULL_HANDLE;
+  VkDeviceSize capacity = 0;
+  std::uint8_t* mapped = nullptr;
 };
 
+// Idle staging buffers kept for reuse. Creating, allocating, binding and
+// mapping one costs hundreds of microseconds in the driver, so a burst of
+// texture uploads would otherwise stall the frame that records it.
+constexpr VkDeviceSize kMaxIdleStagingBytes = 64 * 1024 * 1024;
+constexpr std::size_t kMaxIdleStagingBuffers = 64;
+// A request may take an idle buffer up to this size, or up to 4x its own
+// size when larger, so small uploads share buffers without pinning big ones.
+constexpr VkDeviceSize kStagingReuseSlack = 64 * 1024;
+
 struct PendingUpload {
   VkDevice device = VK_NULL_HANDLE;
   VkBuffer source = VK_NULL_HANDLE;
@@ -252,6 +264,7 @@ void LogFailure(const char* message) {
 
 bool CreateStaging(const HostDevice& dev, VkDeviceSize size, Staging* staging,
                    std::uint8_t** mapped) {
+  *mapped = nullptr;
   if (dev.create_buffer == nullptr || dev.allocate_memory == nullptr ||
       dev.get_buffer_memory_requirements == nullptr ||
       dev.bind_buffer_memory == nullptr || dev.map_memory == nullptr ||
@@ -301,8 +314,8 @@ bool CreateStaging(const HostDevice& dev, VkDeviceSize size, Staging* staging,
     dev.free_memory(dev.device, memory, nullptr);
     return false;
   }
-  *staging = {dev.device, buffer, memory};
   *mapped = static_cast<std::uint8_t*>(data);
+  *staging = {dev.device, buffer, memory, size, *mapped};
   return true;
 }
 
@@ -466,6 +479,8 @@ struct VulkanEtc2Emulation::State {
   std::unordered_map<VkDeviceMemory, Mapping> mappings;
   std::unordered_map<VkCommandBuffer, CommandRecord> commands;
   std::atomic<bool> has_commands{false};
+  std::vector<Staging> idle_staging;
+  VkDeviceSize idle_staging_bytes = 0;
   // Heap buffers reused across submits; they only grow.
   std::mutex scratch_mutex;
   std::vector<std::uint8_t> scratch_source;
@@ -563,6 +578,55 @@ struct VulkanEtc2Emulation::State {
     }
   }
 
+  // Takes the smallest idle buffer that fits, or creates one.
+  bool AcquireStaging(const HostDevice& dev, VkDeviceSize size,
+                      Staging* staging) {
+    {
+      std::lock_guard<std::mutex> lock(mutex);
+      const VkDeviceSize limit = std::max(size * 4, kStagingReuseSlack);
+      auto best = idle_staging.end();
+      for (auto it = idle_staging.begin(); it != idle_staging.end(); ++it) {
+        if (it->device == dev.device && it->capacity >= size &&
+            it->capacity <= limit &&
+            (best == idle_staging.end() || it->capacity < best->capacity)) {
+          best = it;
+        }
+      }
+      if (best != idle_staging.end()) {
+        *staging = *best;
+        idle_staging_bytes -= best->capacity;
+        *best = idle_staging.back();
+        idle_staging.pop_back();
+        return true;
+      }
+    }
+    std::uint8_t* mapped = nullptr;
+    return CreateStaging(dev, size, staging, &mapped);
+  }
+
+  // Keeps released buffers for reuse up to the idle limits and destroys
+  // the rest.
+  void ReleaseStaging(std::vector<Staging> released) {
+    std::vector<Staging> doomed;
+    {
+      std::lock_guard<std::mutex> lock(mutex);
+      for (const Staging& staging : released) {
+        if (idle_staging.size() < kMaxIdleStagingBuffers &&
+            idle_staging_bytes + staging.capacity <= kMaxIdleStagingBytes) {
+          idle_staging.push_back(staging);
+          idle_staging_bytes += staging.capacity;
+        } else {
+          doomed.push_back(staging);
+        }
+      }
+    }
+    for (const Staging& staging : doomed) {
+      if (const HostDevice* dev = Find(staging.device); dev != nullptr) {
+        DestroyStaging(*dev, staging);
+      }
+    }
+  }
+
   const HostDevice* Find(VkDevice device) {
     std::lock_guard<std::mutex> lock(devices_mutex);
     for (const auto& candidate : devices) {
@@ -708,6 +772,16 @@ void VulkanEtc2Emulation::DestroyDevice(VkDevice device) {
         ++it;
       }
     }
+    for (auto it = state_->idle_staging.begin();
+         it != state_->idle_staging.end();) {
+      if (it->device == device) {
+        state_->idle_staging_bytes -= it->capacity;
+        doomed.push_back(*it);
+        it = state_->idle_staging.erase(it);
+      } else {
+        ++it;
+      }
+    }
     for (auto it = state_->images.begin(); it != state_->images.end();) {
       it = it->second.device == device ? state_->images.erase(it) : std::next(it);
     }
@@ -944,18 +1018,17 @@ void VulkanEtc2Emulation::CmdCopyBufferToImage(
   std::vector<PendingUpload> uploads;
   VkDeviceSize total = 0;
   Staging staging;
-  std::uint8_t* mapped = nullptr;
   if (!PlanRegions(record, device, source, destination, region_count, regions,
                    &rewritten, &uploads, &total)) {
     LogFailure("skipped an upload with an unsupported region");
     return;
   }
-  if (!CreateStaging(*dev, total, &staging, &mapped)) {
+  if (!state_->AcquireStaging(*dev, total, &staging)) {
     LogFailure("could not allocate a host-visible staging buffer");
     return;
   }
   for (PendingUpload& upload : uploads) {
-    upload.target = mapped + upload.target_offset;
+    upload.target = staging.mapped + upload.target_offset;
   }
   state_->Record(command_buffer, std::move(uploads), staging);
   dev->copy_buffer_to_image(command_buffer, staging.buffer, destination,
@@ -979,19 +1052,18 @@ void VulkanEtc2Emulation::CmdCopyBufferToImage2(
   std::vector<PendingUpload> uploads;
   VkDeviceSize total = 0;
   Staging staging;
-  std::uint8_t* mapped = nullptr;
   if (!PlanRegions(record, device, info->srcBuffer, info->dstImage,
                    info->regionCount, info->pRegions, &rewritten, &uploads,
                    &total)) {
     LogFailure("skipped an upload with an unsupported region");
     return;
   }
-  if (!CreateStaging(*dev, total, &staging, &mapped)) {
+  if (!state_->AcquireStaging(*dev, total, &staging)) {
     LogFailure("could not allocate a host-visible staging buffer");
     return;
   }
   for (PendingUpload& upload : uploads) {
-    upload.target = mapped + upload.target_offset;
+    upload.target = staging.mapped + upload.target_offset;
   }
   state_->Record(command_buffer, std::move(uploads), staging);
   VkCopyBufferToImageInfo2 host_info = *info;
@@ -1171,6 +1243,8 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers,
       }
     }
   }
+  TraceScope trace_scope(work.empty() ? nullptr : ActiveProfileTrace(),
+                         "etc2 decode", "texture");
   // Mapped Vulkan memory is uncached or write-combined, which makes the
   // decoder's scattered byte reads and writes tens of times slower than on
   // heap memory. Compressed bytes are streamed into a heap scratch buffer,
@@ -1218,6 +1292,10 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers,
     compressed_total += upload.compressed;
     decoded_total += upload.decoded;
   }
+  trace_scope.Arg("uploads", static_cast<std::int64_t>(copies.size()));
+  trace_scope.Arg("compressed_bytes",
+                  static_cast<std::int64_t>(compressed_total));
+  trace_scope.Arg("decoded_bytes", static_cast<std::int64_t>(decoded_total));
 
   std::lock_guard<std::mutex> scratch_lock(state_->scratch_mutex);
   std::vector<std::uint8_t>& scratch_source = state_->scratch_source;
@@ -1243,8 +1321,6 @@ void VulkanEtc2Emulation::PrepareSubmit(const VkCommandBuffer* command_buffers,
       EtcDecodeJob job;
       job.format = upload.format;
       job.source = source + layer * compressed_layer;
-  TraceScope trace_scope(work.empty() ? nullptr : ActiveProfileTrace(),
-                         "etc2 decode", "texture");
       job.source_bytes = compressed_layer;
       job.width = upload.width;
       job.height = upload.height;
@@ -1292,20 +1368,12 @@ void VulkanEtc2Emulation::ReleaseCommandBuffer(VkCommandBuffer command_buffer) {
     if (record == state_->commands.end()) {
       return;
     }
-  trace_scope.Arg("uploads", static_cast<std::int64_t>(copies.size()));
-  trace_scope.Arg("compressed_bytes",
-                  static_cast<std::int64_t>(compressed_total));
-  trace_scope.Arg("decoded_bytes", static_cast<std::int64_t>(decoded_total));
     doomed = std::move(record->second.staging);
     state_->commands.erase(record);
     state_->has_commands.store(!state_->commands.empty(),
                                std::memory_order_release);
   }
-  for (const Staging& staging : doomed) {
-    if (const HostDevice* dev = state_->Find(staging.device); dev != nullptr) {
-      DestroyStaging(*dev, staging);
-    }
-  }
+  state_->ReleaseStaging(std::move(doomed));
 }
 
 }  // namespace mocktail::graphics

tests/texture_override_test.cc

@@ -98,6 +98,56 @@ TEST(TextureOverrideTest, ResampleAveragesSourceBoxes) {
   }
 }
 
+// Rounded mean of the source box [x*sw/w, (x+1)*sw/w) per axis, at least one
+// texel wide.
+std::vector<std::uint8_t> ReferenceBoxResample(const RgbaImage& source,
+                                               std::uint32_t width,
+                                               std::uint32_t height) {
+  std::vector<std::uint8_t> out(static_cast<std::size_t>(width) * height * 4);
+  const auto span = [](std::uint32_t i, std::uint32_t from, std::uint32_t to) {
+    const std::uint64_t first = static_cast<std::uint64_t>(i) * from / to;
+    const std::uint64_t last =
+        std::max<std::uint64_t>(first + 1, (i + 1ULL) * from / to);
+    return std::pair{first, last};
+  };
+  for (std::uint32_t y = 0; y < height; ++y) {
+    const auto [y0, y1] = span(y, source.height, height);
+    for (std::uint32_t x = 0; x < width; ++x) {
+      const auto [x0, x1] = span(x, source.width, width);
+      for (int channel = 0; channel < 4; ++channel) {
+        std::uint64_t sum = 0;
+        for (std::uint64_t sy = y0; sy < y1; ++sy) {
+          for (std::uint64_t sx = x0; sx < x1; ++sx) {
+            sum += source.pixels[(sy * source.width + sx) * 4 + channel];
+          }
+        }
+        const std::uint64_t count = (y1 - y0) * (x1 - x0);
+        out[(static_cast<std::size_t>(y) * width + x) * 4 + channel] =
+            static_cast<std::uint8_t>((sum + count / 2) / count);
+      }
+    }
+  }
+  return out;
+}
+
+TEST(TextureOverrideTest, ResampleMatchesBoxFilterForMixedSizes) {
+  const std::pair<std::uint32_t, std::uint32_t> sources[] = {
+      {1, 1}, {3, 5}, {48, 48}, {60, 33}, {64, 17}};
+  const std::pair<std::uint32_t, std::uint32_t> targets[] = {
+      {1, 1}, {2, 7}, {5, 3}, {48, 48}, {192, 192}, {240, 132}, {31, 200}};
+  for (const auto& [source_width, source_height] : sources) {
+    const RgbaImage source = Gradient(source_width, source_height);
+    for (const auto& [width, height] : targets) {
+      std::vector<std::uint8_t> actual(
+          static_cast<std::size_t>(width) * height * 4, 0);
+      ResampleRgba(source, width, height, actual.data());
+      ASSERT_EQ(actual, ReferenceBoxResample(source, width, height))
+          << source_width << "x" << source_height << " -> " << width << "x"
+          << height;
+    }
+  }
+}
+
 TEST(TextureOverrideTest, LookupFindsPngNamedByHashAndCachesMisses) {
   TempDir dir;
   ASSERT_FALSE(dir.path().empty());

tests/vulkan_etc2_emulation_test.cc

@@ -127,6 +127,10 @@ VkExtent3D g_copied_extent{};
 VkDeviceSize g_created_buffer_size = 0;
 std::uintptr_t g_next_image = 0x100;
 std::uint32_t g_copy_image_calls = 0;
+std::uint32_t g_create_buffer_calls = 0;
+std::uint32_t g_allocate_memory_calls = 0;
+std::uint32_t g_destroy_buffer_calls = 0;
+std::uint32_t g_free_memory_calls = 0;
 VkImageBlit g_blit{};
 VkFilter g_blit_filter = VK_FILTER_NEAREST;
 
@@ -159,6 +163,7 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeCreateBuffer(VkDevice,
                                                 const VkAllocationCallbacks*,
                                                 VkBuffer* buffer) {
   g_created_buffer_size = info->size;
+  ++g_create_buffer_calls;
   *buffer = FakeHandle<VkBuffer>(kStagingBuffer);
   return VK_SUCCESS;
 }
@@ -173,6 +178,7 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeAllocateMemory(
     VkDevice, const VkMemoryAllocateInfo*, const VkAllocationCallbacks*,
     VkDeviceMemory* memory) {
   *memory = FakeHandle<VkDeviceMemory>(kStagingMemory);
+  ++g_allocate_memory_calls;
   return VK_SUCCESS;
 }
 
@@ -193,9 +199,13 @@ VKAPI_ATTR VkResult VKAPI_CALL FakeMapMemory(VkDevice, VkDeviceMemory memory,
 
 VKAPI_ATTR void VKAPI_CALL FakeUnmapMemory(VkDevice, VkDeviceMemory) {}
 VKAPI_ATTR void VKAPI_CALL FakeDestroyBuffer(VkDevice, VkBuffer,
-                                             const VkAllocationCallbacks*) {}
+                                             const VkAllocationCallbacks*) {
+  ++g_destroy_buffer_calls;
+}
 VKAPI_ATTR void VKAPI_CALL FakeFreeMemory(VkDevice, VkDeviceMemory,
-                                          const VkAllocationCallbacks*) {}
+                                          const VkAllocationCallbacks*) {
+  ++g_free_memory_calls;
+}
 VKAPI_ATTR void VKAPI_CALL FakeCopyBufferToImage(
     VkCommandBuffer, VkBuffer, VkImage, VkImageLayout, std::uint32_t count,
     const VkBufferImageCopy* regions) {
@@ -528,6 +538,60 @@ TEST(VulkanEtc2EmulationTest, FillsScaledMipsFromLevelZero) {
   emulation.ReleaseCommandBuffer(command_buffer);
 }
 
+TEST(VulkanEtc2EmulationTest, ReusesStagingBuffersAcrossCommandBuffers) {
+  const VkDevice device = FakeHandle<VkDevice>(0x17);
+  const VkBuffer source = FakeHandle<VkBuffer>(0x207);
+  const VkDeviceMemory source_memory = FakeHandle<VkDeviceMemory>(kSourceMemory);
+  g_source.fill(0);
+
+  VkPhysicalDeviceMemoryProperties memory{};
+  memory.memoryTypeCount = 1;
+  memory.memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
+                                        VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
+  VulkanEtc2Emulation emulation;
+  emulation.RegisterDevice(device, FakeHandle<VkPhysicalDevice>(0x27), true,
+                           memory, FakeGetDeviceProcAddr);
+
+  VkImageCreateInfo image_info{};
+  image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
+  image_info.format = VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK;
+  image_info.extent = {4, 4, 1};
+  VkImage image = VK_NULL_HANDLE;
+  ASSERT_EQ(emulation.CreateImage(device, &image_info, nullptr, &image),
+            VK_SUCCESS);
+  ASSERT_EQ(emulation.BindBufferMemory(device, source, source_memory, 0),
+            VK_SUCCESS);
+  void* mapped = nullptr;
+  ASSERT_EQ(emulation.MapMemory(device, source_memory, 0, g_source.size(), 0,
+                                &mapped),
+            VK_SUCCESS);
+
+  g_create_buffer_calls = 0;
+  g_allocate_memory_calls = 0;
+  g_destroy_buffer_calls = 0;
+  g_free_memory_calls = 0;
+  VkBufferImageCopy region{};
+  region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+  region.imageSubresource.layerCount = 1;
+  region.imageExtent = {4, 4, 1};
+  for (std::uintptr_t frame = 0; frame < 8; ++frame) {
+    const VkCommandBuffer command_buffer =
+        FakeHandle<VkCommandBuffer>(0x780 + frame);
+    emulation.CmdCopyBufferToImage(device, command_buffer, source, image,
+                                   VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1,
+                                   &region);
+    emulation.PrepareSubmit(&command_buffer, 1);
+    emulation.ReleaseCommandBuffer(command_buffer);
+  }
+  EXPECT_EQ(g_create_buffer_calls, 1u);
+  EXPECT_EQ(g_allocate_memory_calls, 1u);
+  EXPECT_EQ(g_destroy_buffer_calls, 0u);
+
+  emulation.DestroyDevice(device);
+  EXPECT_EQ(g_destroy_buffer_calls, 1u);
+  EXPECT_EQ(g_free_memory_calls, 1u);
+}
+
 // Copies out of a scaled image cover its host bounds and shrink back onto
 // an unscaled destination, which a plain copy cannot do.
 TEST(VulkanEtc2EmulationTest, BlitsCopiesThatTouchScaledImages) {