From 74e7455ee170ee74266c009c2c1141c16ea783e3 Mon Sep 17 00:00:00 2001 From: glook9001 Date: Thu, 1 Oct 2026 12:08:35 +0300 Subject: [PATCH 1/2] perf(graphics): SIMD PSHUFB ETC2/EAC decode with bit-exact scalar verification --- src/graphics/etc2_decoder.cc | 352 ++++++++++++++++++++++++++--------- tests/etc2_decoder_test.cc | 269 ++++++++++++++++++++++++++ 2 files changed, 535 insertions(+), 86 deletions(-) diff --git a/src/graphics/etc2_decoder.cc b/src/graphics/etc2_decoder.cc index 361ec53..60ba5a8 100644 --- a/src/graphics/etc2_decoder.cc +++ b/src/graphics/etc2_decoder.cc @@ -2,7 +2,11 @@ #include #include +#include #include +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) +#include +#endif #include #include @@ -42,9 +46,46 @@ int Expand5(int value) { return (value << 3) | (value >> 2); } int Expand6(int value) { return (value << 2) | (value >> 4); } int Expand7(int value) { return (value << 1) | (value >> 6); } +inline std::uint32_t MakeRgba32(int r, int g, int bl, int a) { +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + __m128i v = _mm_set_epi32(a, bl, g, r); + __m128i p16 = _mm_packs_epi32(v, v); + __m128i p8 = _mm_packus_epi16(p16, p16); + return static_cast(_mm_cvtsi128_si32(p8)); +#else + return static_cast(Clamp255(r)) | + (static_cast(Clamp255(g)) << 8) | + (static_cast(Clamp255(bl)) << 16) | + (static_cast(static_cast(a)) << 24); +#endif +} + +#if defined(__SSSE3__) || defined(__x86_64__) || defined(_M_X64) +struct ColumnShuffleTable { + alignas(16) std::uint8_t masks[256][16]; + constexpr ColumnShuffleTable() : masks{} { + for (int m = 0; m < 16; ++m) { + for (int l = 0; l < 16; ++l) { + const int idx = (m << 4) | l; + for (int p = 0; p < 4; ++p) { + const int color_idx = (((m >> p) & 1) << 1) | ((l >> p) & 1); + for (int b = 0; b < 4; ++b) { + masks[idx][p * 4 + b] = static_cast(color_idx * 4 + b); + } + } + } + } + } +}; +inline constexpr ColumnShuffleTable kColShuffleTable{}; +#endif + // Decodes one ETC2 RGB block into 16 column-major RGBA texels (x * 4 + y). +#if (defined(__GNUC__) || defined(__clang__)) && (defined(__x86_64__) || defined(_M_X64)) +__attribute__((target("ssse3"))) +#endif void DecodeRgbBlock(const std::uint8_t* b, bool punchthrough, - std::array* out) { + std::uint32_t* out32) { static constexpr int kDelta[8] = {0, 1, 2, 3, -4, -3, -2, -1}; const bool diff_or_opaque = (b[3] & 2) != 0; const bool opaque = !punchthrough || diff_or_opaque; @@ -53,13 +94,6 @@ void DecodeRgbBlock(const std::uint8_t* b, bool punchthrough, auto index_of = [msb, lsb](int texel) { return (((msb >> texel) & 1) << 1) | ((lsb >> texel) & 1); }; - auto write = [out](int texel, int r, int g, int bl, int a) { - const std::size_t o = static_cast(texel) * 4; - (*out)[o] = Clamp255(r); - (*out)[o + 1] = Clamp255(g); - (*out)[o + 2] = Clamp255(bl); - (*out)[o + 3] = static_cast(a); - }; const int r_sum = (b[0] >> 3) + kDelta[b[0] & 7]; const int g_sum = (b[1] >> 3) + kDelta[b[1] & 7]; @@ -100,13 +134,13 @@ void DecodeRgbBlock(const std::uint8_t* b, bool punchthrough, paint[3][c] = c2[c] - d; } } + std::uint32_t paint32[4]; + for (int i = 0; i < 4; ++i) { + paint32[i] = MakeRgba32(paint[i][0], paint[i][1], paint[i][2], 255); + } for (int texel = 0; texel < 16; ++texel) { const int index = index_of(texel); - if (!opaque && index == 2) { - write(texel, 0, 0, 0, 0); - } else { - write(texel, paint[index][0], paint[index][1], paint[index][2], 255); - } + out32[texel] = (!opaque && index == 2) ? 0 : paint32[index]; } return; } @@ -124,9 +158,10 @@ void DecodeRgbBlock(const std::uint8_t* b, bool punchthrough, const int bv = Expand6(b[7] & 63); for (int x = 0; x < 4; ++x) { for (int y = 0; y < 4; ++y) { - write(x * 4 + y, (x * (rh - ro) + y * (rv - ro) + 4 * ro + 2) >> 2, - (x * (gh - go) + y * (gv - go) + 4 * go + 2) >> 2, - (x * (bh - bo) + y * (bv - bo) + 4 * bo + 2) >> 2, 255); + out32[x * 4 + y] = MakeRgba32( + (x * (rh - ro) + y * (rv - ro) + 4 * ro + 2) >> 2, + (x * (gh - go) + y * (gv - go) + 4 * go + 2) >> 2, + (x * (bh - bo) + y * (bv - bo) + 4 * bo + 2) >> 2, 255); } } return; @@ -144,39 +179,108 @@ void DecodeRgbBlock(const std::uint8_t* b, bool punchthrough, } const int tables[2] = {b[3] >> 5, (b[3] >> 2) & 7}; const bool flipped = (b[3] & 1) != 0; - for (int x = 0; x < 4; ++x) { - for (int y = 0; y < 4; ++y) { - const int texel = x * 4 + y; - const int index = index_of(texel); - if (!opaque && index == 2) { - write(texel, 0, 0, 0, 0); - continue; - } - const int sub = flipped ? (y >= 2 ? 1 : 0) : (x >= 2 ? 1 : 0); + + // Basis Universal palette precalculation: 8 colors per block instead of 48 additions/clamps. + std::uint32_t palette[2][4]; + for (int sub = 0; sub < 2; ++sub) { + for (int idx = 0; idx < 4; ++idx) { const int modifier = - (!opaque && index == 0) ? 0 : kEtc1Modifiers[tables[sub]][index]; - write(texel, base[sub][0] + modifier, base[sub][1] + modifier, - base[sub][2] + modifier, 255); + (!opaque && idx == 0) ? 0 : kEtc1Modifiers[tables[sub]][idx]; + palette[sub][idx] = MakeRgba32(base[sub][0] + modifier, + base[sub][1] + modifier, + base[sub][2] + modifier, 255); + } + } + + if (opaque) { +#if defined(__SSSE3__) || defined(__x86_64__) || defined(_M_X64) + const __m128i pal0 = _mm_loadu_si128(reinterpret_cast(palette[0])); + const __m128i pal1 = _mm_loadu_si128(reinterpret_cast(palette[1])); + if (!flipped) { + for (int x = 0; x < 2; ++x) { + const int combo = (((msb >> (x * 4)) & 15) << 4) | ((lsb >> (x * 4)) & 15); + const __m128i mask = _mm_load_si128( + reinterpret_cast(kColShuffleTable.masks[combo])); + _mm_storeu_si128(reinterpret_cast<__m128i*>(out32 + x * 4), + _mm_shuffle_epi8(pal0, mask)); + } + for (int x = 2; x < 4; ++x) { + const int combo = (((msb >> (x * 4)) & 15) << 4) | ((lsb >> (x * 4)) & 15); + const __m128i mask = _mm_load_si128( + reinterpret_cast(kColShuffleTable.masks[combo])); + _mm_storeu_si128(reinterpret_cast<__m128i*>(out32 + x * 4), + _mm_shuffle_epi8(pal1, mask)); + } + } else { + for (int x = 0; x < 4; ++x) { + const int combo = (((msb >> (x * 4)) & 15) << 4) | ((lsb >> (x * 4)) & 15); + const __m128i mask = _mm_load_si128( + reinterpret_cast(kColShuffleTable.masks[combo])); + const __m128i col0 = _mm_shuffle_epi8(pal0, mask); + const __m128i col1 = _mm_shuffle_epi8(pal1, mask); + const __m128i blended = + _mm_unpacklo_epi64(col0, _mm_srli_si128(col1, 8)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(out32 + x * 4), blended); + } + } +#else + if (!flipped) { + for (int x = 0; x < 2; ++x) { + const std::uint32_t* pal = palette[0]; + out32[x * 4 + 0] = pal[index_of(x * 4 + 0)]; + out32[x * 4 + 1] = pal[index_of(x * 4 + 1)]; + out32[x * 4 + 2] = pal[index_of(x * 4 + 2)]; + out32[x * 4 + 3] = pal[index_of(x * 4 + 3)]; + } + for (int x = 2; x < 4; ++x) { + const std::uint32_t* pal = palette[1]; + out32[x * 4 + 0] = pal[index_of(x * 4 + 0)]; + out32[x * 4 + 1] = pal[index_of(x * 4 + 1)]; + out32[x * 4 + 2] = pal[index_of(x * 4 + 2)]; + out32[x * 4 + 3] = pal[index_of(x * 4 + 3)]; + } + } else { + for (int x = 0; x < 4; ++x) { + out32[x * 4 + 0] = palette[0][index_of(x * 4 + 0)]; + out32[x * 4 + 1] = palette[0][index_of(x * 4 + 1)]; + out32[x * 4 + 2] = palette[1][index_of(x * 4 + 2)]; + out32[x * 4 + 3] = palette[1][index_of(x * 4 + 3)]; + } + } +#endif + } else { + for (int x = 0; x < 4; ++x) { + for (int y = 0; y < 4; ++y) { + const int texel = x * 4 + y; + const int index = index_of(texel); + if (index == 2) { + out32[texel] = 0; + } else { + const int sub = flipped ? (y >= 2 ? 1 : 0) : (x >= 2 ? 1 : 0); + out32[texel] = palette[sub][index]; + } + } } } } // EAC 3-bit index for column-major texel (x * 4 + y). -int EacIndex(const std::uint8_t* b, int texel) { +inline std::uint64_t ReadEacBits(const std::uint8_t* b) { std::uint64_t bits = 0; for (int i = 2; i < 8; ++i) { bits = (bits << 8) | b[i]; } + return bits; +} + +inline int EacIndexFromBits(std::uint64_t bits, int texel) { return static_cast((bits >> (45 - texel * 3)) & 7); } -int EacModifier(const std::uint8_t* b, int texel) { - return kEacModifiers[b[1] & 15][EacIndex(b, texel)]; -} - -std::uint16_t DecodeR11(const std::uint8_t* b, int texel, bool is_signed) { - const int multiplier = b[1] >> 4; - const int modifier = EacModifier(b, texel); +inline std::uint16_t DecodeR11Modifier(const std::uint8_t* b, int multiplier, + const int* modifier_table, + int modifier_idx, bool is_signed) { + const int modifier = modifier_table[modifier_idx]; const int scaled = multiplier != 0 ? modifier * multiplier * 8 : modifier; if (!is_signed) { const int value = std::clamp(b[0] * 8 + 4 + scaled, 0, 2047); @@ -308,12 +412,18 @@ class DecodePool { run_ = run; context_ = context; unclaimed_ = helpers; - running_ = helpers; + running_.store(helpers, std::memory_order_relaxed); + } + for (std::size_t i = 0; i < helpers; ++i) { + ready_.notify_one(); } - ready_.notify_all(); run(context); - std::unique_lock lock(mutex_); - done_.wait(lock, [this] { return running_ == 0; }); + if (running_.load(std::memory_order_acquire) > 0) { + std::unique_lock lock(done_mutex_); + done_.wait(lock, [this] { + return running_.load(std::memory_order_acquire) == 0; + }); + } run_ = nullptr; context_ = nullptr; return true; @@ -343,29 +453,33 @@ class DecodePool { } void WorkerLoop() { - std::unique_lock lock(mutex_); for (;;) { - ready_.wait(lock, [this] { return unclaimed_ > 0; }); - --unclaimed_; - void (*run)(void*) = run_; - void* context = context_; - lock.unlock(); + void (*run)(void*) = nullptr; + void* context = nullptr; + { + std::unique_lock lock(mutex_); + ready_.wait(lock, [this] { return unclaimed_ > 0; }); + --unclaimed_; + run = run_; + context = context_; + } run(context); - lock.lock(); - if (--running_ == 0) { - done_.notify_all(); + if (running_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + std::lock_guard lock(done_mutex_); + done_.notify_one(); } } } std::mutex batch_mutex_; std::mutex mutex_; + std::mutex done_mutex_; std::condition_variable ready_; std::condition_variable done_; void (*run_)(void*) = nullptr; void* context_ = nullptr; std::size_t unclaimed_ = 0; - std::size_t running_ = 0; + std::atomic running_{0}; std::size_t threads_ = 0; }; @@ -378,6 +492,16 @@ void DecodeEtcJobs(EtcDecodeJob* jobs, std::size_t count, } DecodeBandQueue queue; std::vector& bands = queue.bands; + std::size_t estimated_bands = 0; + for (std::size_t index = 0; index < count; ++index) { + const EtcDecodeJob& job = jobs[index]; + const std::uint32_t blocks_wide = (job.width + 3) / 4; + const std::uint32_t blocks_high = (job.height + 3) / 4; + const std::uint32_t rows_per_band = static_cast( + std::max(1, kBlocksPerBand / (blocks_wide ? blocks_wide : 1))); + estimated_bands += (blocks_high + rows_per_band - 1) / (rows_per_band ? rows_per_band : 1); + } + bands.reserve(estimated_bands); std::uint64_t total_blocks = 0; for (std::size_t index = 0; index < count; ++index) { EtcDecodeJob& job = jobs[index]; @@ -439,48 +563,104 @@ bool DecodeEtcImageBlockRows(EtcFormat format, const std::uint8_t* source, const std::size_t texel_bytes = EtcDecodedTexelBytes(format); const std::size_t block_bytes = EtcBlockBytes(format); - std::array rgba{}; + alignas(16) std::uint32_t cols[16]; const std::uint64_t end_block_row = static_cast(first_block_row) + block_row_count; + const bool is_rgba_family = (format == EtcFormat::kEtc2Rgb8 || + format == EtcFormat::kEtc2Rgb8A1 || + format == EtcFormat::kEtc2Rgba8); + for (std::uint64_t by = first_block_row; by < end_block_row; ++by) { for (std::uint64_t bx = 0; bx < blocks_wide; ++bx) { const std::uint8_t* block = source + (by * blocks_wide + bx) * block_bytes; - if (format == EtcFormat::kEtc2Rgb8 || format == EtcFormat::kEtc2Rgb8A1) { - DecodeRgbBlock(block, format == EtcFormat::kEtc2Rgb8A1, &rgba); - } else if (format == EtcFormat::kEtc2Rgba8) { - DecodeRgbBlock(block + 8, false, &rgba); - } - for (std::uint32_t y = 0; y < 4 && by * 4 + y < height; ++y) { - for (std::uint32_t x = 0; x < 4 && bx * 4 + x < width; ++x) { - const int texel = static_cast(x * 4 + y); - std::uint8_t* out = - destination + ((by * 4 + y) * width + bx * 4 + x) * texel_bytes; - switch (format) { - case EtcFormat::kEtc2Rgb8: - case EtcFormat::kEtc2Rgb8A1: - case EtcFormat::kEtc2Rgba8: - std::copy_n(rgba.data() + texel * 4, 4, out); - if (format == EtcFormat::kEtc2Rgba8) { - const int alpha = - block[0] + EacModifier(block, texel) * (block[1] >> 4); - out[3] = Clamp255(alpha); - } - break; - case EtcFormat::kEacR11: - case EtcFormat::kEacR11Signed: - case EtcFormat::kEacRg11: - case EtcFormat::kEacRg11Signed: { - const bool is_signed = format == EtcFormat::kEacR11Signed || - format == EtcFormat::kEacRg11Signed; - const int channels = texel_bytes / 2; - for (int channel = 0; channel < channels; ++channel) { - const std::uint16_t value = - DecodeR11(block + channel * 8, texel, is_signed); - out[channel * 2] = static_cast(value); - out[channel * 2 + 1] = static_cast(value >> 8); - } - break; + const bool interior = (by * 4 + 4 <= height) && (bx * 4 + 4 <= width); + + if (is_rgba_family) { + if (format == EtcFormat::kEtc2Rgb8 || format == EtcFormat::kEtc2Rgb8A1) { + DecodeRgbBlock(block, format == EtcFormat::kEtc2Rgb8A1, cols); + } else { + DecodeRgbBlock(block + 8, false, cols); + const std::uint64_t alpha_bits = ReadEacBits(block); + const int alpha_base = block[0]; + const int alpha_multiplier = block[1] >> 4; + const auto& alpha_table = kEacModifiers[block[1] & 15]; + std::uint8_t alpha_palette[8]; + for (int i = 0; i < 8; ++i) { + alpha_palette[i] = + Clamp255(alpha_base + alpha_table[i] * alpha_multiplier); + } + for (int texel = 0; texel < 16; ++texel) { + cols[texel] = + (cols[texel] & 0x00FFFFFF) | + (static_cast( + alpha_palette[EacIndexFromBits(alpha_bits, texel)]) + << 24); + } + } + + if (interior) { + std::uint32_t* dst_row = reinterpret_cast( + destination + (by * 4 * width + bx * 4) * 4); +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + __m128 r0 = _mm_load_ps(reinterpret_cast(cols)); + __m128 r1 = _mm_load_ps(reinterpret_cast(cols + 4)); + __m128 r2 = _mm_load_ps(reinterpret_cast(cols + 8)); + __m128 r3 = _mm_load_ps(reinterpret_cast(cols + 12)); + _MM_TRANSPOSE4_PS(r0, r1, r2, r3); + _mm_storeu_ps(reinterpret_cast(dst_row), r0); + dst_row += width; + _mm_storeu_ps(reinterpret_cast(dst_row), r1); + dst_row += width; + _mm_storeu_ps(reinterpret_cast(dst_row), r2); + dst_row += width; + _mm_storeu_ps(reinterpret_cast(dst_row), r3); +#else + for (int y = 0; y < 4; ++y) { + dst_row[0] = cols[y]; + dst_row[1] = cols[4 + y]; + dst_row[2] = cols[8 + y]; + dst_row[3] = cols[12 + y]; + dst_row += width; + } +#endif + } else { + for (std::uint32_t y = 0; y < 4 && by * 4 + y < height; ++y) { + for (std::uint32_t x = 0; x < 4 && bx * 4 + x < width; ++x) { + const int texel = static_cast(x * 4 + y); + std::uint32_t* out = reinterpret_cast( + destination + ((by * 4 + y) * width + bx * 4 + x) * 4); + *out = cols[texel]; + } + } + } + } else { + const bool is_signed = format == EtcFormat::kEacR11Signed || + format == EtcFormat::kEacRg11Signed; + const int channels = static_cast(texel_bytes / 2); + std::uint64_t channel_bits[2] = {0, 0}; + std::uint16_t channel_palette[2][8]; + for (int c = 0; c < channels; ++c) { + const std::uint8_t* cb = block + c * 8; + channel_bits[c] = ReadEacBits(cb); + const int mult = cb[1] >> 4; + const int* table = kEacModifiers[cb[1] & 15]; + for (int i = 0; i < 8; ++i) { + channel_palette[c][i] = + DecodeR11Modifier(cb, mult, table, i, is_signed); + } + } + for (std::uint32_t y = 0; y < 4 && by * 4 + y < height; ++y) { + for (std::uint32_t x = 0; x < 4 && bx * 4 + x < width; ++x) { + const int texel = static_cast(x * 4 + y); + std::uint8_t* out = + destination + ((by * 4 + y) * width + bx * 4 + x) * texel_bytes; + for (int channel = 0; channel < channels; ++channel) { + const std::uint16_t value = + channel_palette[channel][EacIndexFromBits(channel_bits[channel], + texel)]; + out[channel * 2] = static_cast(value); + out[channel * 2 + 1] = static_cast(value >> 8); } } } diff --git a/tests/etc2_decoder_test.cc b/tests/etc2_decoder_test.cc index ccf5d2e..1937521 100644 --- a/tests/etc2_decoder_test.cc +++ b/tests/etc2_decoder_test.cc @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -360,5 +361,273 @@ TEST(Etc2DecoderTest, RejectsShortBuffers) { output.data(), output.size())); } +namespace reference { + +constexpr std::array kDistance = {3, 6, 11, 16, 23, 32, 41, 64}; + +constexpr int kEtc1Modifiers[8][4] = { + {2, 8, -2, -8}, {5, 17, -5, -17}, {9, 29, -9, -29}, + {13, 42, -13, -42}, {18, 60, -18, -60}, {24, 80, -24, -80}, + {33, 106, -33, -106}, {47, 183, -47, -183}, +}; + +constexpr int kEacModifiers[16][8] = { + {-3, -6, -9, -15, 2, 5, 8, 14}, {-3, -7, -10, -13, 2, 6, 9, 12}, + {-2, -5, -8, -13, 1, 4, 7, 12}, {-2, -4, -6, -13, 1, 3, 5, 12}, + {-3, -6, -8, -12, 2, 5, 7, 11}, {-3, -7, -9, -11, 2, 6, 8, 10}, + {-4, -7, -8, -11, 3, 6, 7, 10}, {-3, -5, -8, -11, 2, 4, 7, 10}, + {-2, -6, -8, -10, 1, 5, 7, 9}, {-2, -5, -8, -10, 1, 4, 7, 9}, + {-2, -4, -8, -10, 1, 3, 7, 9}, {-2, -5, -7, -10, 1, 4, 6, 9}, + {-3, -4, -7, -10, 2, 3, 6, 9}, {-1, -2, -3, -10, 0, 1, 2, 9}, + {-4, -6, -8, -9, 3, 5, 7, 8}, {-3, -5, -7, -9, 2, 4, 6, 8}, +}; + +inline std::uint8_t Clamp255(int value) { + return static_cast(std::clamp(value, 0, 255)); +} + +inline int Expand4(int value) { return (value << 4) | value; } +inline int Expand5(int value) { return (value << 3) | (value >> 2); } +inline int Expand6(int value) { return (value << 2) | (value >> 4); } +inline int Expand7(int value) { return (value << 1) | (value >> 6); } + +inline void DecodeRgbBlock(const std::uint8_t* b, bool punchthrough, + std::array* out) { + static constexpr int kDelta[8] = {0, 1, 2, 3, -4, -3, -2, -1}; + const bool diff_or_opaque = (b[3] & 2) != 0; + const bool opaque = !punchthrough || diff_or_opaque; + const int msb = (b[4] << 8) | b[5]; + const int lsb = (b[6] << 8) | b[7]; + auto index_of = [msb, lsb](int texel) { + return (((msb >> texel) & 1) << 1) | ((lsb >> texel) & 1); + }; + auto write = [&](int texel, int r, int g, int bl, int a) { + (*out)[texel * 4] = Clamp255(r); + (*out)[texel * 4 + 1] = Clamp255(g); + (*out)[texel * 4 + 2] = Clamp255(bl); + (*out)[texel * 4 + 3] = static_cast(a); + }; + + const int r_sum = (b[0] >> 3) + kDelta[b[0] & 7]; + const int g_sum = (b[1] >> 3) + kDelta[b[1] & 7]; + const int b_sum = (b[2] >> 3) + kDelta[b[2] & 7]; + const bool individual = !punchthrough && !diff_or_opaque; + + if (!individual && (r_sum < 0 || r_sum > 31 || g_sum < 0 || g_sum > 31)) { + std::array, 4> paint{}; + if (r_sum < 0 || r_sum > 31) { + const std::array c1 = { + Expand4((((b[0] >> 3) & 3) << 2) | (b[0] & 3)), + Expand4(b[1] >> 4), Expand4(b[1] & 15)}; + const std::array c2 = {Expand4(b[2] >> 4), Expand4(b[2] & 15), + Expand4(b[3] >> 4)}; + const int d = kDistance[(((b[3] >> 2) & 3) << 1) | (b[3] & 1)]; + for (int c = 0; c < 3; ++c) { + paint[0][c] = c1[c]; + paint[1][c] = c2[c] + d; + paint[2][c] = c2[c]; + paint[3][c] = c2[c] - d; + } + } else { + const std::array c1 = { + Expand4((b[0] >> 3) & 15), + Expand4(((b[0] & 7) << 1) | ((b[1] >> 4) & 1)), + Expand4((b[1] & 8) | ((b[1] & 3) << 1) | (b[2] >> 7))}; + const std::array c2 = { + Expand4((b[2] >> 3) & 15), Expand4(((b[2] & 7) << 1) | (b[3] >> 7)), + Expand4((b[3] >> 3) & 15)}; + const int v1 = (c1[0] << 16) | (c1[1] << 8) | c1[2]; + const int v2 = (c2[0] << 16) | (c2[1] << 8) | c2[2]; + const int d = + kDistance[(b[3] & 4) | ((b[3] & 1) << 1) | (v1 >= v2 ? 1 : 0)]; + for (int c = 0; c < 3; ++c) { + paint[0][c] = c1[c] + d; + paint[1][c] = c1[c] - d; + paint[2][c] = c2[c] + d; + paint[3][c] = c2[c] - d; + } + } + for (int texel = 0; texel < 16; ++texel) { + const int index = index_of(texel); + if (!opaque && index == 2) { + write(texel, 0, 0, 0, 0); + } else { + write(texel, paint[index][0], paint[index][1], paint[index][2], 255); + } + } + return; + } + + if (!individual && (b_sum < 0 || b_sum > 31)) { + const int ro = Expand6((b[0] >> 1) & 63); + const int go = Expand7(((b[0] & 1) << 6) | ((b[1] >> 1) & 63)); + const int bo = Expand6(((b[1] & 1) << 5) | (b[2] & 0x18) | + ((b[2] & 3) << 1) | (b[3] >> 7)); + const int rh = Expand6(((b[3] & 0x7c) >> 1) | (b[3] & 1)); + const int gh = Expand7(b[4] >> 1); + const int bh = Expand6(((b[4] & 1) << 5) | (b[5] >> 3)); + const int rv = Expand6(((b[5] & 7) << 3) | (b[6] >> 5)); + const int gv = Expand7(((b[6] & 31) << 2) | (b[7] >> 6)); + const int bv = Expand6(b[7] & 63); + for (int x = 0; x < 4; ++x) { + for (int y = 0; y < 4; ++y) { + write(x * 4 + y, (x * (rh - ro) + y * (rv - ro) + 4 * ro + 2) >> 2, + (x * (gh - go) + y * (gv - go) + 4 * go + 2) >> 2, + (x * (bh - bo) + y * (bv - bo) + 4 * bo + 2) >> 2, 255); + } + } + return; + } + + std::array, 2> base{}; + for (int c = 0; c < 3; ++c) { + if (individual) { + base[0][c] = Expand4(b[c] >> 4); + base[1][c] = Expand4(b[c] & 15); + } else { + base[0][c] = Expand5(b[c] >> 3); + base[1][c] = Expand5((b[c] >> 3) + kDelta[b[c] & 7]); + } + } + const int tables[2] = {b[3] >> 5, (b[3] >> 2) & 7}; + const bool flipped = (b[3] & 1) != 0; + for (int x = 0; x < 4; ++x) { + for (int y = 0; y < 4; ++y) { + const int texel = x * 4 + y; + const int index = index_of(texel); + if (!opaque && index == 2) { + write(texel, 0, 0, 0, 0); + continue; + } + const int sub = flipped ? (y >= 2 ? 1 : 0) : (x >= 2 ? 1 : 0); + const int modifier = + (!opaque && index == 0) ? 0 : kEtc1Modifiers[tables[sub]][index]; + write(texel, base[sub][0] + modifier, base[sub][1] + modifier, + base[sub][2] + modifier, 255); + } + } +} + +inline int EacIndex(const std::uint8_t* b, int texel) { + std::uint64_t bits = 0; + for (int i = 2; i < 8; ++i) { + bits = (bits << 8) | b[i]; + } + return static_cast((bits >> (45 - texel * 3)) & 7); +} + +inline int EacModifier(const std::uint8_t* b, int texel) { + return kEacModifiers[b[1] & 15][EacIndex(b, texel)]; +} + +inline std::uint16_t DecodeR11(const std::uint8_t* b, int texel, bool is_signed) { + const int multiplier = b[1] >> 4; + const int modifier = EacModifier(b, texel); + const int scaled = multiplier != 0 ? modifier * multiplier * 8 : modifier; + if (!is_signed) { + const int value = std::clamp(b[0] * 8 + 4 + scaled, 0, 2047); + return static_cast((value << 5) | (value >> 6)); + } + const int base = std::max(-127, static_cast(static_cast(b[0]))); + const int value = std::clamp(base * 8 + scaled, -1023, 1023); + const int magnitude = value < 0 ? -value : value; + const int expanded = (magnitude << 5) | (magnitude >> 5); + return static_cast( + static_cast(value < 0 ? -expanded : expanded)); +} + +inline void DecodeReference(EtcFormat format, const std::uint8_t* source, + std::uint32_t width, std::uint32_t height, + std::uint8_t* destination) { + const std::size_t texel_bytes = EtcDecodedTexelBytes(format); + const std::size_t block_bytes = EtcBlockBytes(format); + const std::uint64_t blocks_wide = (static_cast(width) + 3) / 4; + const std::uint64_t blocks_high = (static_cast(height) + 3) / 4; + std::array rgba{}; + + for (std::uint64_t by = 0; by < blocks_high; ++by) { + for (std::uint64_t bx = 0; bx < blocks_wide; ++bx) { + const std::uint8_t* block = source + (by * blocks_wide + bx) * block_bytes; + if (format == EtcFormat::kEtc2Rgb8 || format == EtcFormat::kEtc2Rgb8A1) { + DecodeRgbBlock(block, format == EtcFormat::kEtc2Rgb8A1, &rgba); + } else if (format == EtcFormat::kEtc2Rgba8) { + DecodeRgbBlock(block + 8, false, &rgba); + } + for (std::uint32_t y = 0; y < 4 && by * 4 + y < height; ++y) { + for (std::uint32_t x = 0; x < 4 && bx * 4 + x < width; ++x) { + const int texel = static_cast(x * 4 + y); + std::uint8_t* out = destination + ((by * 4 + y) * width + bx * 4 + x) * texel_bytes; + switch (format) { + case EtcFormat::kEtc2Rgb8: + case EtcFormat::kEtc2Rgb8A1: + case EtcFormat::kEtc2Rgba8: + std::copy_n(rgba.data() + texel * 4, 4, out); + if (format == EtcFormat::kEtc2Rgba8) { + const int alpha = block[0] + EacModifier(block, texel) * (block[1] >> 4); + out[3] = Clamp255(alpha); + } + break; + case EtcFormat::kEacR11: + case EtcFormat::kEacR11Signed: + case EtcFormat::kEacRg11: + case EtcFormat::kEacRg11Signed: { + const bool is_signed = format == EtcFormat::kEacR11Signed || format == EtcFormat::kEacRg11Signed; + const int channels = static_cast(texel_bytes / 2); + for (int channel = 0; channel < channels; ++channel) { + const std::uint16_t value = DecodeR11(block + channel * 8, texel, is_signed); + out[channel * 2] = static_cast(value); + out[channel * 2 + 1] = static_cast(value >> 8); + } + break; + } + } + } + } + } + } +} + +} // namespace reference + +TEST(Etc2DecoderTest, ComparesBitExactAgainstScalarReferenceAcrossAllFormats) { + const EtcFormat formats[] = { + EtcFormat::kEtc2Rgb8, + EtcFormat::kEtc2Rgb8A1, + EtcFormat::kEtc2Rgba8, + EtcFormat::kEacR11, + EtcFormat::kEacR11Signed, + EtcFormat::kEacRg11, + EtcFormat::kEacRg11Signed, + }; + const struct { + std::uint32_t width; + std::uint32_t height; + } sizes[] = { + {1, 1}, + {4, 4}, + {7, 5}, + {16, 16}, + {67, 59}, + {128, 128}, + }; + for (std::size_t f = 0; f < std::size(formats); ++f) { + const EtcFormat format = formats[f]; + for (std::size_t s = 0; s < std::size(sizes); ++s) { + const std::uint32_t width = sizes[s].width; + const std::uint32_t height = sizes[s].height; + const auto source = RandomBlocks( + format, width, height, + static_cast(0xABCD1234 + f * 100 + s)); + const auto actual = Decode(format, source, width, height); + std::vector expected(actual.size(), 0); + reference::DecodeReference(format, source.data(), width, height, + expected.data()); + EXPECT_EQ(actual, expected) + << "Mismatch in format=" << static_cast(format) + << " width=" << width << " height=" << height; + } + } +} + } // namespace } // namespace mocktail::graphics From c6afc5af242ae8626bc3b34376dc071bb5fe1389 Mon Sep 17 00:00:00 2001 From: glook9001 Date: Thu, 1 Oct 2026 12:08:54 +0300 Subject: [PATCH 2/2] perf(graphics): eliminate crop page faults with strided resampling and streaming WC loads --- include/mocktail/graphics/texture_override.h | 4 + src/graphics/texture_override.cc | 45 ++++++++--- src/graphics/vulkan_etc2_emulation.cc | 78 ++++++++++++++------ tests/texture_override_test.cc | 15 ++++ 4 files changed, 107 insertions(+), 35 deletions(-) diff --git a/include/mocktail/graphics/texture_override.h b/include/mocktail/graphics/texture_override.h index 8bd3b83..9b12ee2 100644 --- a/include/mocktail/graphics/texture_override.h +++ b/include/mocktail/graphics/texture_override.h @@ -35,6 +35,10 @@ void ResampleRgba(const RgbaImage& source, std::uint32_t width, void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, std::uint32_t source_height, std::uint32_t width, std::uint32_t height, std::uint8_t* destination); +void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, + std::uint32_t source_height, std::size_t source_stride_bytes, + std::uint32_t width, std::uint32_t height, + std::uint8_t* destination); // Replacement textures keyed by the hash of a texture's level-0 compressed // bytes. An override lives at /.png. With a dump diff --git a/src/graphics/texture_override.cc b/src/graphics/texture_override.cc index c00c54b..5271d57 100644 --- a/src/graphics/texture_override.cc +++ b/src/graphics/texture_override.cc @@ -72,20 +72,33 @@ void ResampleRgba(const RgbaImage& source, std::uint32_t width, static_cast(source.width) * source.height * 4) { return; } - ResampleRgba(source.pixels.data(), source.width, source.height, width, - height, destination); + ResampleRgba(source.pixels.data(), source.width, source.height, + static_cast(source.width) * 4, width, height, + destination); } void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, std::uint32_t source_height, std::uint32_t width, std::uint32_t height, std::uint8_t* destination) { + ResampleRgba(source, source_width, source_height, + static_cast(source_width) * 4, width, height, + destination); +} + +void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, + std::uint32_t source_height, std::size_t source_stride_bytes, + std::uint32_t width, std::uint32_t height, + std::uint8_t* destination) { if (source == nullptr || destination == nullptr || width == 0 || - height == 0 || source_width == 0 || source_height == 0) { + height == 0 || source_width == 0 || source_height == 0 || + source_stride_bytes < static_cast(source_width) * 4) { return; } // Source box of destination column x is [columns[2x], columns[2x + 1]). thread_local std::vector columns; - columns.resize(static_cast(width) * 2); + if (columns.size() < static_cast(width) * 2) { + columns.resize(static_cast(width) * 2); + } bool one_texel_columns = true; for (std::uint32_t x = 0; x < width; ++x) { const std::uint32_t x0 = static_cast( @@ -115,11 +128,11 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, previous_y0 = y0; previous_y1 = y1; if (one_texel_columns && y1 == y0 + 1) { - const std::uint8_t* source_row = source + y0 * source_width * 4; + const std::uint8_t* source_row = source + y0 * source_stride_bytes; for (std::uint32_t x = 0; x < width; ++x) { - std::memcpy(out_row + static_cast(x) * 4, - source_row + columns[2 * static_cast(x)] * 4, - 4); + std::uint32_t pixel; + std::memcpy(&pixel, source_row + columns[2 * static_cast(x)] * 4, 4); + std::memcpy(out_row + static_cast(x) * 4, &pixel, 4); } continue; } @@ -128,8 +141,9 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, const std::uint64_t x1 = columns[2 * static_cast(x) + 1]; std::uint64_t sum[4] = {0, 0, 0, 0}; for (std::uint64_t sy = y0; sy < y1; ++sy) { + const std::uint8_t* row_ptr = source + sy * source_stride_bytes; for (std::uint64_t sx = x0; sx < x1; ++sx) { - const std::uint8_t* pixel = source + (sy * source_width + sx) * 4; + const std::uint8_t* pixel = row_ptr + sx * 4; for (int channel = 0; channel < 4; ++channel) { sum[channel] += pixel[channel]; } @@ -137,9 +151,16 @@ void ResampleRgba(const std::uint8_t* source, std::uint32_t source_width, } const std::uint64_t count = (y1 - y0) * (x1 - x0); std::uint8_t* out = destination + (static_cast(y) * width + x) * 4; - for (int channel = 0; channel < 4; ++channel) { - out[channel] = - static_cast((sum[channel] + count / 2) / count); + if (count == 1) { + for (int channel = 0; channel < 4; ++channel) { + out[channel] = static_cast(sum[channel]); + } + } else { + const std::uint64_t half = count / 2; + for (int channel = 0; channel < 4; ++channel) { + out[channel] = + static_cast((sum[channel] + half) / count); + } } } } diff --git a/src/graphics/vulkan_etc2_emulation.cc b/src/graphics/vulkan_etc2_emulation.cc index a1bff07..b989958 100644 --- a/src/graphics/vulkan_etc2_emulation.cc +++ b/src/graphics/vulkan_etc2_emulation.cc @@ -18,6 +18,10 @@ #include #include +#if defined(__x86_64__) +#include +#endif + namespace mocktail::graphics { bool LookupEmulatedEtc2Format(VkFormat format, Etc2EmulatedFormat* out) { @@ -398,6 +402,45 @@ bool CreateStagingBlock(const HostDevice& dev, VkDeviceSize size, return true; } +#if defined(__x86_64__) +__attribute__((target("sse4.1"))) +inline void CopyHostVisibleMemory(std::uint8_t* dst, const std::uint8_t* src, + std::size_t bytes) { + if ((reinterpret_cast(src) & 15) != 0 || bytes < 16) { + std::memcpy(dst, src, bytes); + return; + } + std::size_t i = 0; + for (; i + 64 <= bytes; i += 64) { + __m128i c0 = _mm_stream_load_si128( + reinterpret_cast<__m128i*>(const_cast(src + i))); + __m128i c1 = _mm_stream_load_si128( + reinterpret_cast<__m128i*>(const_cast(src + i + 16))); + __m128i c2 = _mm_stream_load_si128( + reinterpret_cast<__m128i*>(const_cast(src + i + 32))); + __m128i c3 = _mm_stream_load_si128( + reinterpret_cast<__m128i*>(const_cast(src + i + 48))); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), c0); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), c1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 32), c2); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 48), c3); + } + for (; i + 16 <= bytes; i += 16) { + __m128i c = _mm_stream_load_si128( + reinterpret_cast<__m128i*>(const_cast(src + i))); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), c); + } + if (i < bytes) { + std::memcpy(dst + i, src + i, bytes - i); + } +} +#else +inline void CopyHostVisibleMemory(std::uint8_t* dst, const std::uint8_t* src, + std::size_t bytes) { + std::memcpy(dst, src, bytes); +} +#endif + // Packs each upload's compressed bytes into `destination`, dropping the row // and layer padding the application may have asked for. Sources are parallel // to `uploads`. @@ -411,17 +454,18 @@ void GatherCompressed(const std::vector& uploads, std::uint8_t* target = destination + offset; const VkDeviceSize compressed_layer = upload.compressed / upload.layers; if (upload.source_span == upload.compressed) { - std::memcpy(target, source, upload.compressed); + CopyHostVisibleMemory(target, source, upload.compressed); } else { const VkDeviceSize block_rows = (static_cast(upload.height) + 3) / 4; const VkDeviceSize packed_row = compressed_layer / block_rows; for (std::uint32_t layer = 0; layer < upload.layers; ++layer) { for (VkDeviceSize row = 0; row < block_rows; ++row) { - std::memcpy(target + layer * compressed_layer + row * packed_row, - source + layer * upload.source_layer_stride + - row * upload.source_row_stride, - packed_row); + CopyHostVisibleMemory( + target + layer * compressed_layer + row * packed_row, + source + layer * upload.source_layer_stride + + row * upload.source_row_stride, + packed_row); } } } @@ -710,21 +754,11 @@ struct VulkanEtc2Emulation::State { upload.target); return; } - const std::uint8_t* source = decoded; - std::vector cropped; - if (upload.source_width != upload.width || - upload.source_height != upload.height) { - const std::size_t row = static_cast(upload.source_width) * 4; - cropped.resize(row * upload.source_height); - for (std::uint32_t y = 0; y < upload.source_height; ++y) { - std::memcpy(cropped.data() + y * row, - decoded + static_cast(y) * upload.width * 4, - row); - } - source = cropped.data(); - } - ResampleRgba(source, upload.source_width, upload.source_height, - upload.target_width, upload.target_height, upload.target); + const std::size_t source_stride = + static_cast(upload.width) * 4; + ResampleRgba(decoded, upload.source_width, upload.source_height, + source_stride, upload.target_width, upload.target_height, + upload.target); } else { std::memcpy(upload.target, decoded, upload.decoded); } @@ -1700,9 +1734,7 @@ Etc2SubmitWait VulkanEtc2Emulation::PrepareSubmit( collect(command_buffers[index]); const auto record = state_->commands.find(command_buffers[index]); if (record != state_->commands.end()) { - const std::vector secondaries = - record->second.secondaries; - for (VkCommandBuffer secondary : secondaries) { + for (VkCommandBuffer secondary : record->second.secondaries) { collect(secondary); } } diff --git a/tests/texture_override_test.cc b/tests/texture_override_test.cc index d2552e0..6036376 100644 --- a/tests/texture_override_test.cc +++ b/tests/texture_override_test.cc @@ -96,6 +96,21 @@ TEST(TextureOverrideTest, ResampleAveragesSourceBoxes) { EXPECT_EQ(grown[pixel * 4 + 0], 9); EXPECT_EQ(grown[pixel * 4 + 3], 6); } + + // 4x4 image with row stride = 16 bytes, but we only resample the top-left 2x2 region + std::vector padded(4 * 16, 0); + for (std::uint32_t y = 0; y < 2; ++y) { + for (std::uint32_t x = 0; x < 2; ++x) { + padded[y * 16 + x * 4 + 0] = source.pixels[(y * 2 + x) * 4 + 0]; + padded[y * 16 + x * 4 + 1] = source.pixels[(y * 2 + x) * 4 + 1]; + padded[y * 16 + x * 4 + 2] = source.pixels[(y * 2 + x) * 4 + 2]; + padded[y * 16 + x * 4 + 3] = source.pixels[(y * 2 + x) * 4 + 3]; + } + } + std::uint8_t strided_four[16] = {}; + ResampleRgba(padded.data(), 2, 2, 16, 2, 2, strided_four); + EXPECT_EQ(std::vector(strided_four, strided_four + 16), + source.pixels); } // Rounded mean of the source box [x*sw/w, (x+1)*sw/w) per axis, at least one