From dce293e25d2756581405f0ff7b5ecc83ff96bce2 Mon Sep 17 00:00:00 2001 From: AyeZeeBB Date: Wed, 21 Oct 2026 17:16:56 -0500 Subject: [PATCH] Fit Check as a plugin: screen capture, the menu's preview turned and read, and what the skater wears Pull request 326 split into general plugin APIs and contrib/plugins/fit-check: - reskate.capture (start, stop, discard, state, compare, kept, save): the screen recorded before the overlay draws, cut out by the game's depth or by colour, saved as GIF, APNG, PNG or WebM with an optional card; the "capture" permission. - reskate.room.turn: a stand-in pad's stick held over to turn the menu's preview. - reskate.menu.game and reskate.menu.grid: what the game's menu and its item grid show. - reskate.outfit.worn, board and title: what the skater wears, by name. - reskate.files.reveal: a data folder opened in Explorer. The plugin finds where the turn comes round, eases the stick off, picks evenly spaced frames and saves them with the board's parts or the outfit on the card. --- CMakeLists.txt | 9 + Extension/Console/commands.cpp | 1 + Extension/Console/commands.h | 2 + Extension/Customization/item_browser.cpp | 25 + Extension/Customization/item_browser.h | 6 +- Extension/Customization/worn_items.cpp | 144 +++++ Extension/Customization/worn_items.h | 19 + Extension/Plugins/Test/plugin_tests.cpp | 151 ++++- Extension/Plugins/plugin_api.cpp | 1 + Extension/Plugins/plugin_api.h | 3 + Extension/Plugins/plugin_api_capture.cpp | 314 +++++++++++ Extension/Plugins/plugin_game.h | 65 +++ Extension/Plugins/plugin_game_runtime.cpp | 124 +++++ Extension/Plugins/plugin_manifest.cpp | 4 +- Extension/Plugins/plugin_vm.h | 5 +- .../Rendering/Capture/Test/capture_tests.cpp | 438 +++++++++++++++ .../Rendering/Capture/capture_commands.cpp | 48 ++ Extension/Rendering/Capture/capture_encode.h | 38 ++ Extension/Rendering/Capture/capture_gif.cpp | 229 ++++++++ Extension/Rendering/Capture/capture_image.cpp | 171 ++++++ Extension/Rendering/Capture/capture_image.h | 75 +++ Extension/Rendering/Capture/capture_matte.cpp | 226 ++++++++ Extension/Rendering/Capture/capture_matte.h | 29 + Extension/Rendering/Capture/capture_panel.cpp | 206 +++++++ Extension/Rendering/Capture/capture_panel.h | 23 + Extension/Rendering/Capture/capture_png.cpp | 150 +++++ .../Rendering/Capture/capture_recorder.cpp | 465 ++++++++++++++++ .../Rendering/Capture/capture_recorder.h | 117 ++++ .../Rendering/Capture/capture_runtime.cpp | 34 ++ Extension/Rendering/Capture/capture_runtime.h | 13 + Extension/Rendering/Capture/capture_vp8.cpp | 458 +++++++++++++++ Extension/Rendering/Capture/capture_vp8.h | 16 + .../Rendering/Capture/capture_vp8_tables.h | 226 ++++++++ Extension/Rendering/Capture/capture_webm.cpp | 183 ++++++ Extension/UI/Overlay/depth_probe.cpp | 520 ++++++++++++++++++ Extension/UI/Overlay/frame_grab.cpp | 185 +++++++ Extension/UI/Overlay/overlay.h | 24 + Extension/UI/Overlay/overlay_internal.h | 17 + Extension/UI/Overlay/overlay_render.cpp | 16 +- Extension/UI/Overlay/scripted_pad.cpp | 23 +- Extension/UI/Overlay/scripted_pad.h | 4 + Extension/World/character_room.cpp | 2 + Extension/World/character_room.h | 4 + Runtime/runtime_init.cpp | 2 + cmake/Core.cmake | 1 + cmake/Dependencies.cmake | 3 +- cmake/Features.cmake | 16 + cmake/Runtime.cmake | 4 + cmake/UI.cmake | 2 + contrib/plugins/fit-check/README.md | 67 +++ contrib/plugins/fit-check/plugin/main.lua | 357 ++++++++++++ contrib/plugins/fit-check/plugin/tests.lua | 102 ++++ contrib/plugins/fit-check/plugin/turn.lua | 216 ++++++++ contrib/plugins/fit-check/reskate-plugin.json | 6 + 54 files changed, 5579 insertions(+), 10 deletions(-) create mode 100644 Extension/Customization/worn_items.cpp create mode 100644 Extension/Customization/worn_items.h create mode 100644 Extension/Plugins/plugin_api_capture.cpp create mode 100644 Extension/Rendering/Capture/Test/capture_tests.cpp create mode 100644 Extension/Rendering/Capture/capture_commands.cpp create mode 100644 Extension/Rendering/Capture/capture_encode.h create mode 100644 Extension/Rendering/Capture/capture_gif.cpp create mode 100644 Extension/Rendering/Capture/capture_image.cpp create mode 100644 Extension/Rendering/Capture/capture_image.h create mode 100644 Extension/Rendering/Capture/capture_matte.cpp create mode 100644 Extension/Rendering/Capture/capture_matte.h create mode 100644 Extension/Rendering/Capture/capture_panel.cpp create mode 100644 Extension/Rendering/Capture/capture_panel.h create mode 100644 Extension/Rendering/Capture/capture_png.cpp create mode 100644 Extension/Rendering/Capture/capture_recorder.cpp create mode 100644 Extension/Rendering/Capture/capture_recorder.h create mode 100644 Extension/Rendering/Capture/capture_runtime.cpp create mode 100644 Extension/Rendering/Capture/capture_runtime.h create mode 100644 Extension/Rendering/Capture/capture_vp8.cpp create mode 100644 Extension/Rendering/Capture/capture_vp8.h create mode 100644 Extension/Rendering/Capture/capture_vp8_tables.h create mode 100644 Extension/Rendering/Capture/capture_webm.cpp create mode 100644 Extension/UI/Overlay/depth_probe.cpp create mode 100644 Extension/UI/Overlay/frame_grab.cpp create mode 100644 contrib/plugins/fit-check/README.md create mode 100644 contrib/plugins/fit-check/plugin/main.lua create mode 100644 contrib/plugins/fit-check/plugin/tests.lua create mode 100644 contrib/plugins/fit-check/plugin/turn.lua create mode 100644 contrib/plugins/fit-check/reskate-plugin.json diff --git a/CMakeLists.txt b/CMakeLists.txt index f284285c..8926f15e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -401,6 +401,15 @@ if(DINGOSDK_BUILD_XRAY_TESTS) target_link_libraries(dingosdk_xray_skeleton_tests PRIVATE dingosdk_game_archives dingosdk_logging) add_test(NAME xray_skeleton COMMAND dingosdk_xray_skeleton_tests "${DINGOSDK_TEST_GAME_ROOT}") endif() +option(DINGOSDK_BUILD_CAPTURE_TESTS "Build screen capture regression tests" OFF) +if(DINGOSDK_BUILD_CAPTURE_TESTS AND TARGET dingosdk_capture) + enable_testing() + # Matting, resizing, the card, GIF / APNG / WebM files that decode back to what went in, and a + # recording of a turning board found, cut out and saved (reskate.capture). + add_executable(dingosdk_capture_tests Extension/Rendering/Capture/Test/capture_tests.cpp) + target_link_libraries(dingosdk_capture_tests PRIVATE dingosdk_capture) + add_test(NAME capture COMMAND dingosdk_capture_tests "${PROJECT_SOURCE_DIR}/External/fonts") +endif() option(DINGOSDK_BUILD_PLUGIN_TESTS "Build plugin host regression tests" OFF) if(DINGOSDK_BUILD_PLUGIN_TESTS AND WIN32) enable_testing() diff --git a/Extension/Console/commands.cpp b/Extension/Console/commands.cpp index f6906e04..53fa0532 100644 --- a/Extension/Console/commands.cpp +++ b/Extension/Console/commands.cpp @@ -227,6 +227,7 @@ const Commands &game_commands() { register_multiplayer_commands(*result); register_perf_commands(*result); register_item_commands(*result); + register_capture_commands(*result); register_trainer_commands(*result); register_replay_keyframes_commands(*result); register_plugin_commands(*result); diff --git a/Extension/Console/commands.h b/Extension/Console/commands.h index 27c1f7ab..aa0826fd 100644 --- a/Extension/Console/commands.h +++ b/Extension/Console/commands.h @@ -25,6 +25,8 @@ void register_perf_commands(Commands &); // perf_commands.cpp: starts `perf memory`'s trace with the game when RESKATE_MEMORY_TRACE asks for it. void start_memory_trace_from_environment() noexcept; void register_item_commands(Commands &); +// Extension/Rendering/Capture/capture_commands.cpp: `capture depth`, research for reskate.capture. +void register_capture_commands(Commands &); void register_trainer_commands(Commands &); void register_replay_keyframes_commands(Commands &); void register_plugin_commands(Commands &); diff --git a/Extension/Customization/item_browser.cpp b/Extension/Customization/item_browser.cpp index e723c8dd..8501a610 100644 --- a/Extension/Customization/item_browser.cpp +++ b/Extension/Customization/item_browser.cpp @@ -221,6 +221,20 @@ Rows read_rows(const md::Context& context, md::Value items, const Layout& layout return rows; } +// Every item's title as the grids showed it this session (a mod's item has none in the game's own +// catalog), for plugins (reskate.outfit.title). +struct Titles { + std::mutex mutex; + std::map> by_key; +}; +Titles& titles() { static auto* value = new Titles; return *value; } +void remember_title(const std::string& key, const std::string& title) { + if (key.empty() || title.empty()) return; + auto& t = titles(); + std::lock_guard lock(t.mutex); + if (t.by_key.size() < 65536) t.by_key.insert_or_assign(key, title); +} + // The catalog item behind a tile's item model. Empty key: the tile does not show a cosmetic. Item resolve(const md::Context& context, std::uint64_t content) { Item item; @@ -236,6 +250,7 @@ Item resolve(const md::Context& context, std::uint64_t content) { item.key = md::string(md::read(at + context.member(data.type, data_key).offset), 255); item.title = md::string(md::read(at + context.member(data.type, data_title).offset), 255); item.category = md::string(md::read(at + context.member(data.type, data_category).offset), 96); + remember_title(item.key, item.title); const auto& catalogs = content_cache::catalogs(); item.modded = catalogs.available && !catalogs.items.contains(folded(item.key)); return item; @@ -551,6 +566,12 @@ void set_search(std::string text) { sh.search_pending = true; } void set_filter(Filter filter) noexcept { shared().filter.store(static_cast(filter)); } +std::string known_title(std::string_view key) { + auto& t = titles(); + std::lock_guard lock(t.mutex); + const auto found = t.by_key.find(key); + return found == t.by_key.end() ? std::string() : found->second; +} const overlay::ItemBrowserHost* overlay_host() noexcept { static const overlay::ItemBrowserHost host{ [] { @@ -675,6 +696,7 @@ void follow_focus(State& s, Shared& sh, std::uintptr_t base) { if (item == grid->native.end() || item->key == next.focused_key) return; next.focused_title = item->title; next.focused_key = item->key; + next.focused_category = item->category; next.focused_favorite = s.favorites.keys.contains(item->key); } std::lock_guard shared_lock(sh.mutex); @@ -786,10 +808,13 @@ void update(std::uintptr_t base) noexcept { next.shown = static_cast(active->shown.size()) - (std::min)(blanks, static_cast(active->shown.size())); next.total = static_cast(active->native.size()) - blanks; for (const auto& item : active->native) next.favorites += s.favorites.keys.contains(item.key); + for (const auto& item : active->native) + if (!item.key.empty()) next.grid_keys.push_back(item.key); } if (focused) { next.focused_title = focused->title; next.focused_key = focused->key; + next.focused_category = focused->category; next.focused_favorite = s.favorites.keys.contains(focused->key); } std::lock_guard shared_lock(sh.mutex); diff --git a/Extension/Customization/item_browser.h b/Extension/Customization/item_browser.h index e0c085e3..c2946295 100644 --- a/Extension/Customization/item_browser.h +++ b/Extension/Customization/item_browser.h @@ -3,6 +3,7 @@ #include #include #include +#include // Favorites, search and filters for the skater's item grids (Apparel, Accessories, Board, Tattoos, // Gestures, Appearance). The grids stay the game's own: their tile rows are reordered and filtered in @@ -15,7 +16,8 @@ struct View { std::string search; Filter filter{}; unsigned shown{}, total{}, favorites{}; // of the grid that has the focus - std::string focused_title, focused_key; + std::string focused_title, focused_key, focused_category; + std::vector grid_keys; // every item of the grid that has the focus bool focused_favorite{}; }; View view(); @@ -24,6 +26,8 @@ void toggle_focused_favorite() noexcept; void set_search(std::string text); void set_filter(Filter filter) noexcept; void cycle_filter() noexcept; +// An item's title as an item grid showed it this session (mods' items too), or empty. Any thread. +std::string known_title(std::string_view key); // The table the overlay's search bar and keys use (process lifetime). const overlay::ItemBrowserHost* overlay_host() noexcept; // Console research aids; their answers go to the log. diff --git a/Extension/Customization/worn_items.cpp b/Extension/Customization/worn_items.cpp new file mode 100644 index 00000000..4b45229a --- /dev/null +++ b/Extension/Customization/worn_items.cpp @@ -0,0 +1,144 @@ +#include "worn_items.h" +#include "item_browser.h" +#include "local_customization.h" +#include "local_customization_runtime.h" +#include "Engine/Core/Platform/memory.h" +#include "Engine/Game/Build/addresses.h" +#include "Engine/Game/Build/20260929/engine.h" +#include "Engine/Game/Build/20260929/skater_slot_override.h" +#include +#include +#include +#include +#include + +// What the skater wears, read from the game's own presets: ported from Fit Check (pull request 326, +// turnaround.cpp), for plugins (reskate.outfit.worn, board, title). +namespace dingosdk::worn_items { +namespace { +// The board's own recipe in a skater preset (its template), and its slots, in the card's order. Read +// from the game's presets with Fit Check's `turnaround parts`, 2026-10-20. +constexpr std::uint32_t board_template = 0x5e61aaef; +struct BoardSlot { std::uint32_t slot; const char* label; }; +constexpr BoardSlot board_slots[] = { + {0x66071628, "Deck"}, {0x755d5ee0, "Grip tape"}, {0x976f18c4, "Grip cutout"}, {0x0add98be, "Trucks"}, {0xc83bb94e, "Wheels"}}; + +// An item's name as the game shows it: its catalog title, the title an item grid showed (a mod's +// item has no catalog entry), or its key made readable the way the game does for a mod's item. +} // namespace +std::string item_title(const std::string& asset) { + auto title = profile::item_display_metadata(asset, Json::object()).value("title", ""); + if (title.empty()) title = item_browser::known_title(asset); + if (title.empty()) { + title = asset.starts_with("Own_") || asset.starts_with("own_") ? asset.substr(4) : asset; + std::replace(title.begin(), title.end(), '_', ' '); + } + return title; +} + +namespace { +// The selected skater preset (the native getter skater_slot_override.cpp uses), or -1. Game thread. +std::int32_t selected_preset(std::uintptr_t base, std::uintptr_t manager) { + namespace slot = addr::skater_slot_override; + std::array bytes{}; + if (!memory::read_bytes(base + slot::native_selected, bytes.data(), bytes.size()) || bytes != slot::native_selected_prefix) return -1; + return reinterpret_cast(base + slot::native_selected)(manager); +} + +// The board the skater rides now: the board recipe of the selected preset (the first when none is +// selected), its parts in the card's order, empty slots left out. Game thread. +} // namespace +std::vector worn_board() { + std::vector parts; + const auto base = reinterpret_cast(GetModuleHandleW(nullptr)); + std::uintptr_t manager{}, begin{}, end{}; + if (!memory::read(base + addr::engine::loadout_manager, manager) || !manager || !memory::read(manager + 0x18, begin) || + !memory::read(manager + 0x20, end) || begin >= end || (end - begin) % 0x48 || (end - begin) / 0x48 > 10) + return parts; + const auto count = static_cast((end - begin) / 0x48); + auto preset = selected_preset(base, manager); + if (preset < 0 || preset >= count) preset = 0; + profile::CosmeticLoadout loadout; + std::vector native; + if (!profile_runtime::read_cosmetic_recipes(reinterpret_cast(begin + static_cast(preset) * 0x48), loadout, native)) + return parts; + for (const auto& recipe : loadout.recipes) { + if (recipe.template_key != board_template) continue; + for (const auto& wanted : board_slots) + for (const auto& item : recipe.items) { + if (item.slot != wanted.slot || item.asset.empty()) continue; + parts.push_back({wanted.label, item.asset, item_title(item.asset)}); + } + break; + } + return parts; +} + +// What the skater wears now: every item of the selected preset +// but the board's, labelled by its catalog category (the game's names for its grids: "tops", +// "headgear"...), clothes and accessories only (not the body, hair, tattoos or gestures), in the +// order they are worn from head to foot. Game thread. +namespace { +struct OutfitKind { const char* word; const char* label; int order; }; +constexpr OutfitKind outfit_kinds[] = { + {"head", "Headwear", 0}, {"hat", "Headwear", 0}, {"cap", "Headwear", 0}, {"beanie", "Headwear", 0}, {"helmet", "Headwear", 0}, + {"eye", "Eyewear", 1}, {"glass", "Eyewear", 1}, {"mask", "Face", 1}, + {"neck", "Neckwear", 2}, {"jewel", "Jewellery", 2}, {"earring", "Earrings", 2}, + {"costume", "Costume", 3}, {"outfit", "Outfit", 3}, + {"outer", "Outerwear", 4}, {"jacket", "Outerwear", 4}, {"top", "Top", 5}, {"shirt", "Top", 5}, {"hood", "Top", 5}, + {"glove", "Gloves", 6}, {"wrist", "Wrist", 6}, {"watch", "Wrist", 6}, {"bag", "Bag", 7}, {"backpack", "Bag", 7}, + {"bottom", "Bottoms", 8}, {"pant", "Bottoms", 8}, {"short", "Bottoms", 8}, {"leg", "Bottoms", 8}, + {"sock", "Socks", 9}, {"foot", "Shoes", 10}, {"shoe", "Shoes", 10}}; +constexpr const char* not_outfit[] = {"skin", "face_", "body", "hair", "beard", "brow", "makeup", "eyecolo", "eye_colo", + "tattoo", "gesture", "emote", "voice", "bkobject", "board", "deck", "truck", "wheel", "grip", + "locomotion", "lipcolo", "mole", "stubble", "freckle", "scar", "nose", "mouth", "teeth"}; +std::string folded(std::string_view text) { + std::string out(text); + for (auto& c : out) c = static_cast(std::tolower(static_cast(c))); + return out; +} +} // namespace +std::vector worn_outfit() { + std::vector parts; + const auto base = reinterpret_cast(GetModuleHandleW(nullptr)); + std::uintptr_t manager{}, begin{}, end{}; + if (!memory::read(base + addr::engine::loadout_manager, manager) || !manager || !memory::read(manager + 0x18, begin) || + !memory::read(manager + 0x20, end) || begin >= end || (end - begin) % 0x48 || (end - begin) / 0x48 > 10) + return parts; + const auto count = static_cast((end - begin) / 0x48); + auto preset = selected_preset(base, manager); + if (preset < 0 || preset >= count) preset = 0; + profile::CosmeticLoadout loadout; + std::vector native; + if (!profile_runtime::read_cosmetic_recipes(reinterpret_cast(begin + static_cast(preset) * 0x48), loadout, native)) + return parts; + const auto& known = profile_runtime::cosmetic_runtime().items; + std::vector> found; + for (const auto& recipe : loadout.recipes) { + if (recipe.template_key == board_template) continue; + for (const auto& item : recipe.items) { + if (item.asset.empty()) continue; + const auto entry = known.find(item.asset); + const auto category = folded(entry != known.end() ? entry->second.category : std::string()); + const auto has = [&](const char* word) { return category.find(word) != std::string::npos; }; + // The body's own parts are "cust..." (custhead, custeye, custnoses, custbodies, custmoles...). + if (category.empty() || category.starts_with("cust") || std::any_of(std::begin(not_outfit), std::end(not_outfit), has)) + continue; + const auto kind = std::find_if(std::begin(outfit_kinds), std::end(outfit_kinds), [&](const OutfitKind& k) { return has(k.word); }); + std::string label; + int order = 20; + if (kind != std::end(outfit_kinds)) label = kind->label, order = kind->order; + else { + // A category this does not know: its name made readable. + label = entry->second.category; + std::replace(label.begin(), label.end(), '_', ' '); + if (!label.empty()) label[0] = static_cast(std::toupper(static_cast(label[0]))); + } + found.push_back({order, {label, item.asset, item_title(item.asset)}}); + } + } + std::stable_sort(found.begin(), found.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + for (auto& entry : found) parts.push_back(std::move(entry.second)); + return parts; +} +} // namespace dingosdk::worn_items diff --git a/Extension/Customization/worn_items.h b/Extension/Customization/worn_items.h new file mode 100644 index 00000000..bb83de86 --- /dev/null +++ b/Extension/Customization/worn_items.h @@ -0,0 +1,19 @@ +#pragma once +#include +#include + +// What the skater wears now, read from the selected skater preset (game thread): the clothes and +// accessories by the kind of item, from head to foot, and the board's parts; an item's title as the +// game shows it. For plugins (reskate.outfit.worn, board, title); from Fit Check (pull request 326). +namespace dingosdk::worn_items { +struct Worn { + std::string slot; // "Headwear", "Top", ... "Shoes"; or "Deck", "Grip tape", "Grip cutout", "Trucks", "Wheels" + std::string item; // its key + std::string title; // as the game shows it +}; +std::vector worn_outfit(); +std::vector worn_board(); +// Its catalog title, the title an item grid showed this session (a mod's item has no catalog +// entry), or its key made readable the way the game does for a mod's item. +std::string item_title(const std::string& asset); +} // namespace dingosdk::worn_items diff --git a/Extension/Plugins/Test/plugin_tests.cpp b/Extension/Plugins/Test/plugin_tests.cpp index f7907539..7e3cde78 100644 --- a/Extension/Plugins/Test/plugin_tests.cpp +++ b/Extension/Plugins/Test/plugin_tests.cpp @@ -503,14 +503,74 @@ struct FakeGame final : Game { Track track() override { return now_playing; } std::string music(std::string_view command) override { calls.push_back("music " + std::string(command)); return {}; } Progress progress() override { return {true, {{"historic", "Historic", 4}}, {{"c1", "line", "historic", 2, 1, 3}}, 1500, 9000, 2}; } - std::optional setting(std::string_view name) override { return name == "Render.Fov" ? std::optional("90") : std::nullopt; } - std::vector setting_names() override { return {"Render.Fov"}; } + std::optional setting(std::string_view name) override { + if (std::find(render_settings.begin(), render_settings.end(), name) != render_settings.end()) return "true"; + return name == "Render.Fov" ? std::optional("90") : std::nullopt; + } + std::vector setting_names() override { + auto names = render_settings; + names.insert(names.begin(), "Render.Fov"); + return names; + } std::string bind(std::string_view plugin, std::string_view action, std::uint32_t combo) override { binds[std::string(plugin) + "/" + std::string(action)] = combo; return {}; } std::vector> take_presses() override { return std::exchange(presses, {}); } void release(std::string_view plugin) override { released.emplace_back(plugin); entity_clear(plugin); } + // The game menu, its item grid, what is worn, and a screen capture of something turning once + // every `turning` frames (reskate.capture, room.turn, menu.game/grid, outfit.board). + GameMenu menu_now; + std::optional grid_now; + std::vector board_now, outfit_now; + std::vector render_settings; // more settings, each on + CaptureStatus capture_now{"idle"}; + double turning = 100; + std::vector capture_starts; + std::vector capture_saves; + float turned{}; + std::string room_turn(std::string_view, float value, int stick) override { + if (value != 0 && !menu_now.open) return "the game's menu is not open"; + turned = value; + calls.push_back(std::format("turn {:.2f} {}", value, stick)); + return {}; + } + GameMenu game_menu() override { return menu_now; } + std::optional menu_grid() override { return grid_now; } + std::vector board_parts() override { return board_now; } + std::vector outfit_worn() override { return outfit_now; } + std::string capture_start(std::string_view plugin, const CaptureStart& request) override { + capture_starts.push_back(request); + capture_now = {"recording"}; + calls.push_back(std::format("capture start {}", plugin)); + return {}; + } + void capture_stop(std::string_view) override { + if (capture_now.state == "recording") capture_now.state = "stopped"; + } + void capture_discard(std::string_view plugin) override { + capture_now = {"idle"}; + calls.push_back(std::format("capture discard {}", plugin)); + } + CaptureStatus capture_status(std::string_view) override { return capture_now; } + float capture_compare(std::string_view, std::uint32_t a, std::uint32_t b) override { + if (a >= capture_now.taken || b >= capture_now.taken) return -1; + return 100 * static_cast(std::abs(std::sin(3.14159265358979 * (double(a) - double(b)) / turning))); + } + std::vector capture_kept(std::string_view) override { + std::vector out; + for (std::uint32_t n = 0; n < capture_now.taken; ++n) out.push_back(n); + return out; + } + std::string capture_save(std::string_view, const CaptureSave& request) override { + capture_saves.push_back(request); + capture_now = {"saving", capture_now.taken, capture_now.taken}; + return {}; + } + std::string files_reveal(std::string_view plugin, std::string_view folder) override { + calls.push_back(std::format("reveal {} {}", plugin, folder)); + return {}; + } bool called(std::string_view part) const { return std::any_of(calls.begin(), calls.end(), [&](const auto& c) { return c.find(part) != std::string::npos; }); } @@ -2672,6 +2732,92 @@ void plugins_add_throwdown_modes() { // The throwdown kit: a plugin registers a mode, publishes its documents (parsed into the store the // native UI reads), hears what the native UI posts, holds the native flag, keeps data files, opens a // dialog; and everything goes when it stops. +// Fit Check as a mod (contrib/plugins/fit-check): its own checks pass, and V on the Board screen +// turns the board until it has come round, eases the stick off when it turns too fast for the frames +// wanted, and saves evenly spaced frames of one turn with the board's parts on its card, the +// browsed deck in place of the worn one; the game's bloom is off while it turns. +void fit_check_turns_the_board_and_saves_it() { + TestEnvironment environment; + std::filesystem::remove_all(scratch / "storage" / "fit-check"); + auto& game = environment.fake; + const auto directory = scratch / "mods" / "fit-check"; + std::filesystem::remove_all(directory); + std::filesystem::copy(std::filesystem::path(RESKATE_SOURCE_DIR) / "contrib" / "plugins" / "fit-check", directory, + std::filesystem::copy_options::recursive); + Host host(environment, Side::client); + host.apply({{"fit-check", directory, "1", check_package_files(directory)}}); + running(host, "fit-check", environment); + check(host.run_console("fitcheck test") && environment.printed("fitcheck: 10 passed, 0 failed") && !environment.printed("FAIL"), + "Fit Check's own checks pass:\n" + printed_lines(environment) + environment.all()); + check(game.binds.contains("fit-check/fitcheck"), "V is bound"); + + // V in the world does nothing; on the Board screen, with a deck browsed, it turns the board. + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.capture_starts.empty(), "Not out of the menu"); + game.menu_now = {true, false, {}}; + game.grid_now = Game::MenuGrid{"deck_b", "Deck B", "Decks", {"deck_a", "deck_b"}, 2, 2}; + game.board_now = {{"Deck", "deck_a", "Deck A"}, {"Wheels", "wheels_a", "Wheels A"}}; + game.render_settings = {"PostProcess.BloomEnable", "Render.ShadowsEnable"}; + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.capture_starts.size() == 1 && game.capture_starts[0].menu && game.capture_starts[0].left == 0.655f && + game.capture_starts[0].skip == 45 && game.capture_starts[0].height == 640, + "The board's area of the menu is captured\n" + environment.all()); + check(game.called("setting fit-check PostProcess.BloomEnable false") && !game.called("Render.ShadowsEnable"), + "Bloom is off while it turns, the shadow maps left\n" + environment.all()); + ticks(host, environment, 1); + check(game.called("turn 0.50 2"), "The right stick is held half over\n" + environment.all()); + // Its banner over the game, and its page saying what it is doing. + environment.panels_open = true; + ticks(host, environment, 10); + const auto banner = std::any_of(environment.layers.begin(), environment.layers.end(), [](const plugin_ui::Layer& layer) { + return std::any_of(layer.shapes.begin(), layer.shapes.end(), [](const auto& s) { return s.text.starts_with("RECORDING FIT CHECK"); }); + }); + check(banner, "The banner shows while it records\n" + environment.all()); + check(environment.panels.size() == 1 && environment.panels[0].title == "Fit Check" && environment.panels[0].widgets.size() > 8, + std::format("Its page is built ({} panels, {} widgets)\n", environment.panels.size(), + environment.panels.empty() ? 0 : environment.panels[0].widgets.size()) + environment.all()); + environment.panels_open = false; + + // Round in 100 frames, too few for 120 (4 s at 30): slower, and again. + game.capture_now.taken = 130; + ticks(host, environment, 1); + check(game.capture_starts.size() == 2 && game.capture_saves.empty(), "Too fast: started again\n" + environment.all()); + + // Round in 130: its 120 evenly spaced frames saved, the card with the browsed deck. + game.turning = 130; + game.capture_now.taken = 150; + ticks(host, environment, 1); + check(game.called("turn 0.46 2"), "The stick eased off to 0.5 x 0.92\n" + environment.all()); + check(game.capture_saves.size() == 1, "Saved once it came round\n" + environment.all()); + const auto& saved = game.capture_saves[0]; + check(saved.frames.size() == 120 && saved.frames[0] == 0 && saved.frames[60] == 65 && saved.format == "gif" && saved.size == 512 && + !saved.background && saved.solid, + "Evenly spaced frames of one turn, as a transparent GIF"); + check(saved.path.starts_with("fit-checks/Deck-B-") && saved.path.ends_with(".gif"), "named after the deck: " + saved.path); + check(saved.card_title == "Board" && saved.card.size() == 2 && saved.card[0].second == "Deck B" && saved.card[1].second == "Wheels A", + "The card has the browsed deck and the worn wheels"); + check(game.called("setting fit-check PostProcess.BloomEnable none") && game.turned == 0, "Bloom back and the stick let go\n" + environment.all()); + game.capture_now = {"saved", 150, 150, true, "cut out by depth", saved.path}; + ticks(host, environment, 1); + check(environment.logged("[fit-check] Saved Deck-B-") && environment.logged(".gif in the Fit Checks folder (cut out by depth)."), + "And it says where\n" + environment.all()); + + check(game.called("capture discard fit-check"), "Its pictures are let go once saved"); + game.calls.clear(); + // V on the Skaters screen with no grid: the skater; V again stops it. + game.grid_now.reset(); + game.menu_now = {true, true, {}}; + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.capture_starts.size() == 3 && game.capture_starts.back().left == 0.27f, "The skater's area\n" + environment.all()); + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.called("capture discard fit-check") && game.turned == 0, "V again stops it\n" + environment.all()); + check(host.run_console("fitcheck open") && game.called("reveal fit-check fit-checks"), "The folder is shown"); +} + void plugins_drive_the_throwdown_kit() { namespace kit = dingosdk::throwdowns::kit; TestEnvironment environment; @@ -3298,6 +3444,7 @@ int main() { grind_exits_pick_the_side(); smokey_throwdowns_register_and_play_one_up(); plugins_drive_the_throwdown_kit(); + fit_check_turns_the_board_and_saves_it(); the_throwdowns_page_lists_the_stock_cards_and_plugin_modes(); plugins_add_throwdown_modes(); reskate_hostile_is_stopped_or_refused_every_time(); diff --git a/Extension/Plugins/plugin_api.cpp b/Extension/Plugins/plugin_api.cpp index f0b8b216..39b465c0 100644 --- a/Extension/Plugins/plugin_api.cpp +++ b/Extension/Plugins/plugin_api.cpp @@ -710,6 +710,7 @@ void Api::install(Plugin& plugin, lua_State* L) { install_world_api(plugin, L); install_services_api(plugin, L); install_kit_api(plugin, L); + install_capture_api(plugin, L); readonly(); lua_setglobal(L, "reskate"); lua_pushcfunction(L, ApiFunctions::print, "print"); diff --git a/Extension/Plugins/plugin_api.h b/Extension/Plugins/plugin_api.h index 90e26494..d7b0aea3 100644 --- a/Extension/Plugins/plugin_api.h +++ b/Extension/Plugins/plugin_api.h @@ -23,6 +23,9 @@ void install_services_api(Plugin& plugin, lua_State* L); // The throwdown kit's documents and native flag, the plugin's data files, overlay dialogs and the // skater's push (with its wait and reset) and hold (plugin_api_throwdowns.cpp). void install_kit_api(Plugin& plugin, lua_State* L); +// Screen captures, the game menu's preview turned and read, what the skater wears, and the data +// folder shown (plugin_api_capture.cpp). +void install_capture_api(Plugin& plugin, lua_State* L); // ---- adding to the reskate table --------------------------------------------------------------- // A new part of the API is a file of `static int name(lua_State*)` functions and an install_*_api diff --git a/Extension/Plugins/plugin_api_capture.cpp b/Extension/Plugins/plugin_api_capture.cpp new file mode 100644 index 00000000..08e138cd --- /dev/null +++ b/Extension/Plugins/plugin_api_capture.cpp @@ -0,0 +1,314 @@ +#include "plugin_api.h" +#include "plugin_game.h" +#include "lua.h" +#include "lualib.h" +#include +#include +#include + +// reskate.capture (the screen as the game draws it, saved as a GIF, APNG, PNG or WebM), the game +// menu's preview turned (reskate.room.turn) and read (reskate.menu.game, reskate.menu.grid), what the +// skater wears (reskate.outfit.worn, board, title) and reskate.files.reveal: from Fit Check (pull +// request 326), whose turn finding and card are now a plugin's (contrib/plugins/fit-check). +namespace dingosdk::plugins { +namespace { +int result(lua_State* L, const std::string& why) { + lua_pushboolean(L, why.empty()); + if (why.empty()) return 1; + lua_pushlstring(L, why.data(), why.size()); + return 2; +} +void push_text(lua_State* L, const std::string& text, const char* field) { + lua_pushlstring(L, text.data(), text.size()); + lua_setfield(L, -2, field); +} +// An option of a table at `index`: a number within [least, most] (raises otherwise), or `fallback`. +double number_option(lua_State* L, int index, const char* name, double fallback, double least, double most) { + if (lua_isnoneornil(L, index)) return fallback; + lua_getfield(L, index, name); + double value = fallback; + if (!lua_isnil(L, -1)) { + value = luaL_checknumber(L, -1); + if (!(value >= least && value <= most)) luaL_error(L, "%s is %g to %g", name, least, most); + } + lua_pop(L, 1); + return value; +} +bool flag_option(lua_State* L, int index, const char* name, bool fallback) { + if (lua_isnoneornil(L, index)) return fallback; + lua_getfield(L, index, name); + const bool value = lua_isnil(L, -1) ? fallback : lua_toboolean(L, -1) != 0; + lua_pop(L, 1); + return value; +} +std::string text_option(lua_State* L, int index, const char* name, const char* fallback) { + if (lua_isnoneornil(L, index)) return fallback; + lua_getfield(L, index, name); + std::string value = fallback; + if (!lua_isnil(L, -1)) value = luaL_checkstring(L, -1); + lua_pop(L, 1); + return value; +} +// A per-second budget; false past it. +bool within(double now, double& second, int& count, int most) { + if (now - second >= 1000) { + second = now; + count = 0; + } + return ++count <= most; +} +} // namespace + +struct CaptureApi { + static Game& game(Plugin& plugin) { return plugin.services_.game(); } + static Plugin& capturing(lua_State* L, const char* what) { + auto& plugin = plugin_of(L); + need(L, plugin, "capture", what); + return plugin; + } + + // start({area = {left, top, right, bottom}, frame = "screen"|"menu", depth, height, skip, frames, seconds}) + static int start(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.start"); + Game::CaptureStart request; + if (!lua_isnoneornil(L, 1)) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_getfield(L, 1, "area"); + if (!lua_isnil(L, -1)) { + luaL_checktype(L, -1, LUA_TTABLE); + float edges[4]{}; + for (int i = 0; i < 4; ++i) { + lua_rawgeti(L, -1, i + 1); + const double v = luaL_checknumber(L, -1); + if (!(v >= 0 && v <= 1)) luaL_error(L, "an area is {left, top, right, bottom}, each 0 to 1"); + edges[i] = static_cast(v); + lua_pop(L, 1); + } + if (!(edges[2] > edges[0] && edges[3] > edges[1])) luaL_error(L, "an area's right is past its left, and its bottom past its top"); + request.left = edges[0], request.top = edges[1], request.right = edges[2], request.bottom = edges[3]; + } + lua_pop(L, 1); + const auto frame = text_option(L, 1, "frame", "screen"); + if (frame != "screen" && frame != "menu") luaL_error(L, "frame is \"screen\" or \"menu\""); + request.menu = frame == "menu"; + request.depth = flag_option(L, 1, "depth", true); + request.height = static_cast(number_option(L, 1, "height", 0, 0, 4096)); + if (request.height && request.height < 64) luaL_error(L, "height is 0 (as on screen) or 64 to 4096"); + request.skip = static_cast(number_option(L, 1, "skip", 0, 0, 600)); + request.most = static_cast(number_option(L, 1, "frames", 2400, 1, 20000)); + request.seconds = number_option(L, 1, "seconds", 60, 1, 120); + } + if (!within(plugin.services_.now_ms(), plugin.control_second_, plugin.controls_, 4)) return result(L, "asked more than 4 times in a second"); + return result(L, game(plugin).capture_start(plugin.id(), request)); + } + static int stop(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.stop"); + game(plugin).capture_stop(plugin.id()); + return 0; + } + static int discard(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.discard"); + game(plugin).capture_discard(plugin.id()); + return 0; + } + static int state(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.state"); + const auto now = game(plugin).capture_status(plugin.id()); + lua_createtable(L, 0, 6); + push_text(L, now.state, "state"); + lua_pushinteger(L, static_cast(now.taken)); + lua_setfield(L, -2, "taken"); + lua_pushinteger(L, static_cast(now.kept)); + lua_setfield(L, -2, "kept"); + lua_pushboolean(L, now.depth); + lua_setfield(L, -2, "depth"); + if (!now.message.empty()) push_text(L, now.message, "message"); + if (!now.file.empty()) push_text(L, now.file, "file"); + return 1; + } + static std::uint32_t frame_number(lua_State* L, int index) { + const double n = luaL_checknumber(L, index); + if (!(n >= 1 && n <= 20000) || n != std::floor(n)) luaL_argerror(L, index, "a frame is a whole number from 1"); + return static_cast(n) - 1; + } + // compare(a, b): how unlike two frames taken look, 0 to 255; nil when either is not known. + static int compare(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.compare"); + const float d = game(plugin).capture_compare(plugin.id(), frame_number(L, 1), frame_number(L, 2)); + if (d < 0) lua_pushnil(L); + else lua_pushnumber(L, d); + return 1; + } + static int kept(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.kept"); + const auto list = game(plugin).capture_kept(plugin.id()); + lua_createtable(L, static_cast(list.size()), 0); + for (std::size_t i = 0; i < list.size(); ++i) { + lua_pushinteger(L, static_cast(list[i]) + 1); + lua_rawseti(L, -2, static_cast(i + 1)); + } + return 1; + } + // save(path, {frames, format, fps, size, background, cutout, solid, card = {title, lines}}) + static int save(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.save"); + need(L, plugin, "files", "reskate.capture.save"); + Game::CaptureSave request; + request.path = data_path(L, 1, false); + luaL_checktype(L, 2, LUA_TTABLE); + lua_getfield(L, 2, "frames"); + luaL_checktype(L, -1, LUA_TTABLE); + const int count = lua_objlen(L, -1); + if (count < 1 || count > 360) luaL_error(L, "a capture saves 1 to 360 frames"); + for (int i = 1; i <= count; ++i) { + lua_rawgeti(L, -1, i); + request.frames.push_back(frame_number(L, lua_gettop(L))); + lua_pop(L, 1); + } + lua_pop(L, 1); + request.format = text_option(L, 2, "format", "gif"); + if (request.format != "gif" && request.format != "apng" && request.format != "png" && request.format != "webm") + luaL_error(L, "format is \"gif\", \"apng\", \"png\" or \"webm\""); + if (request.format == "png" && count != 1) luaL_error(L, "a png is one frame"); + request.fps = static_cast(number_option(L, 2, "fps", 30, 1, 60)); + request.size = static_cast(number_option(L, 2, "size", 512, 64, 2048)); + lua_getfield(L, 2, "background"); + if (lua_isnumber(L, -1)) { + const double c = lua_tonumber(L, -1); + if (!(c >= 0 && c <= 0xFFFFFF) || c != std::floor(c)) luaL_error(L, "background is 0xRRGGBB, or \"transparent\""); + request.background = static_cast(c); + } else if (!lua_isnil(L, -1) && std::string(luaL_checkstring(L, -1)) != "transparent") + luaL_error(L, "background is 0xRRGGBB, or \"transparent\""); + lua_pop(L, 1); + request.cutout = text_option(L, 2, "cutout", "auto"); + if (request.cutout != "auto" && request.cutout != "depth" && request.cutout != "colour" && request.cutout != "none") + luaL_error(L, "cutout is \"auto\", \"depth\", \"colour\" or \"none\""); + request.solid = flag_option(L, 2, "solid", true); + lua_getfield(L, 2, "card"); + if (!lua_isnil(L, -1)) { + luaL_checktype(L, -1, LUA_TTABLE); + const int card = lua_gettop(L); + request.card_title = text_option(L, card, "title", ""); + if (request.card_title.size() > 32) luaL_error(L, "a card's title is at most 32 bytes"); + lua_getfield(L, card, "lines"); + luaL_checktype(L, -1, LUA_TTABLE); + const int lines = lua_objlen(L, -1); + if (lines > 16) luaL_error(L, "a card has at most 16 lines"); + for (int i = 1; i <= lines; ++i) { + lua_rawgeti(L, -1, i); + luaL_checktype(L, -1, LUA_TTABLE); + lua_rawgeti(L, -1, 1); + lua_rawgeti(L, -2, 2); + std::string label = luaL_checkstring(L, -2), value = luaL_checkstring(L, -1); + if (label.size() > 64 || value.size() > 64) luaL_error(L, "a card's line is {label, value}, each at most 64 bytes"); + request.card.emplace_back(std::move(label), std::move(value)); + lua_pop(L, 3); + } + lua_pop(L, 1); + } + lua_pop(L, 1); + if (!within(plugin.services_.now_ms(), plugin.files_second_, plugin.file_calls_, 64)) return result(L, "more than 64 file calls in a second"); + return result(L, game(plugin).capture_save(plugin.id(), request)); + } + + // room.turn(value, stick): the game menu's preview turned by a stand-in pad's stick, held 300 ms. + static int room_turn(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "room", "reskate.room.turn"); + const double value = luaL_checknumber(L, 1); + if (!(value >= -1 && value <= 1)) luaL_argerror(L, 1, "-1 to 1"); + const std::string stick = luaL_optstring(L, 2, "right_x"); + static constexpr const char* sticks[] = {"left_x", "left_y", "right_x", "right_y"}; + const auto at = std::find_if(std::begin(sticks), std::end(sticks), [&](const char* s) { return stick == s; }); + if (at == std::end(sticks)) luaL_argerror(L, 2, "\"left_x\", \"left_y\", \"right_x\" or \"right_y\""); + return result(L, game(plugin).room_turn(plugin.id(), static_cast(value), static_cast(at - std::begin(sticks)))); + } + static int menu_game(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "menu", "reskate.menu.game"); + const auto menu = game(plugin).game_menu(); + lua_createtable(L, 0, 3); + lua_pushboolean(L, menu.open); + lua_setfield(L, -2, "open"); + lua_pushboolean(L, menu.open && menu.skaters); + lua_setfield(L, -2, "skaters"); + lua_createtable(L, static_cast(menu.buttons.size()), 0); + for (std::size_t i = 0; i < menu.buttons.size() && i < 64; ++i) { + lua_pushlstring(L, menu.buttons[i].data(), menu.buttons[i].size()); + lua_rawseti(L, -2, static_cast(i + 1)); + } + lua_setfield(L, -2, "buttons"); + return 1; + } + static int menu_grid(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "menu", "reskate.menu.grid"); + const auto grid = game(plugin).menu_grid(); + if (!grid) { + lua_pushnil(L); + return 1; + } + lua_createtable(L, 0, 6); + if (!grid->focused.empty()) push_text(L, grid->focused, "focused"); + if (!grid->title.empty()) push_text(L, grid->title, "title"); + if (!grid->category.empty()) push_text(L, grid->category, "category"); + lua_createtable(L, static_cast(std::min(grid->items.size(), 4000)), 0); + for (std::size_t i = 0; i < grid->items.size() && i < 4000; ++i) { + lua_pushlstring(L, grid->items[i].data(), grid->items[i].size()); + lua_rawseti(L, -2, static_cast(i + 1)); + } + lua_setfield(L, -2, "items"); + lua_pushinteger(L, static_cast(grid->shown)); + lua_setfield(L, -2, "shown"); + lua_pushinteger(L, static_cast(grid->total)); + lua_setfield(L, -2, "total"); + return 1; + } + static int push_worn(lua_State* L, const std::vector& list) { + lua_createtable(L, static_cast(list.size()), 0); + for (std::size_t i = 0; i < list.size(); ++i) { + lua_createtable(L, 0, 3); + push_text(L, list[i].slot, "slot"); + push_text(L, list[i].item, "item"); + push_text(L, list[i].title, "title"); + lua_rawseti(L, -2, static_cast(i + 1)); + } + return 1; + } + static int outfit_worn(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "outfit", "reskate.outfit.worn"); + return push_worn(L, game(plugin).outfit_worn()); + } + static int outfit_board(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "outfit", "reskate.outfit.board"); + return push_worn(L, game(plugin).board_parts()); + } + static int outfit_title(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "outfit", "reskate.outfit.title"); + const auto key = checked_text(L, 1, 255, "an item's key"); + const auto title = game(plugin).item_title(key); + lua_pushlstring(L, title.data(), title.size()); + return 1; + } + static int files_reveal(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "files", "reskate.files.reveal"); + const auto folder = data_path(L, 1, true); + if (!within(plugin.services_.now_ms(), plugin.reveal_second_, plugin.reveals_, 1)) return result(L, "once a second"); + return result(L, game(plugin).files_reveal(plugin.id(), folder)); + } + static void install(lua_State* L) { + add_functions(L, "capture", {{"start", start}, {"stop", stop}, {"discard", discard}, {"state", state}, {"compare", compare}, + {"kept", kept}, {"save", save}}); + add_functions(L, "room", {{"turn", room_turn}}); + add_functions(L, "menu", {{"game", menu_game}, {"grid", menu_grid}}); + add_functions(L, "outfit", {{"worn", outfit_worn}, {"board", outfit_board}, {"title", outfit_title}}); + add_functions(L, "files", {{"reveal", files_reveal}}); + } +}; + +void install_capture_api(Plugin&, lua_State* L) { CaptureApi::install(L); } +} // namespace dingosdk::plugins diff --git a/Extension/Plugins/plugin_game.h b/Extension/Plugins/plugin_game.h index 8df8e719..a76cca0e 100644 --- a/Extension/Plugins/plugin_game.h +++ b/Extension/Plugins/plugin_game.h @@ -378,6 +378,58 @@ public: std::string state, why; }; virtual RoomState room_state(std::string_view /*plugin*/) { return {"idle", {}}; } + // The game menu's preview (the skater on the Skaters page, the board on a Board tab) turned by a + // stick of a stand-in pad: `value` -1..1 of the stick held for 300 ms (call it every tick), 0 lets + // go; `stick` 0 left x, 1 left y, 2 right x, 3 right y. Only while the game's menu is open. + virtual std::string room_turn(std::string_view /*plugin*/, float /*value*/, int /*stick*/) { return unavailable; } + // The game's menu: whether it is open, whether it shows the Skaters page, and the navigations of + // the open page's buttons (how the game names where each goes: "CASPresetPurchase", ...). + struct GameMenu { + bool open{}, skaters{}; + std::vector buttons; + }; + virtual GameMenu game_menu() { return {}; } + // The customization grid that has the focus (Extension/Customization/item_browser.h), when one does. + struct MenuGrid { + std::string focused, title, category; // the highlighted item: key, title, catalog category + std::vector items; // every item's key in the grid + unsigned shown{}, total{}; + }; + virtual std::optional menu_grid() { return std::nullopt; } + // ---- capture: the screen as the game draws it (Extension/Rendering/Capture) --------------------- + // One plugin records at a time; frames are numbered from 0 as they are taken. + struct CaptureStart { + float left{}, top{}, right{1}, bottom{1}; + bool menu{}; // fractions of the game's menu frame (at most 21:9), else of the screen + bool depth{true}; + std::uint32_t height{}, skip{}, most{2400}; + double seconds{60}; + }; + virtual std::string capture_start(std::string_view /*plugin*/, const CaptureStart&) { return unavailable; } + virtual void capture_stop(std::string_view /*plugin*/) {} + virtual void capture_discard(std::string_view /*plugin*/) {} + struct CaptureStatus { + std::string state; // "idle", "recording", "stopped", "saving", "saved", "failed" + std::uint32_t taken{}, kept{}; + bool depth{}; + std::string message, file; // file: its path in the plugin's data folder + }; + // Another plugin's capture reads as idle. + virtual CaptureStatus capture_status(std::string_view /*plugin*/) { return {"idle"}; } + virtual float capture_compare(std::string_view /*plugin*/, std::uint32_t, std::uint32_t) { return -1; } + virtual std::vector capture_kept(std::string_view /*plugin*/) { return {}; } + struct CaptureSave { + std::vector frames; + std::string format{"gif"}; // "gif", "apng", "png", "webm" + std::uint32_t fps{30}, size{512}; + std::optional background; // 0xRRGGBB; none: transparent + std::string cutout{"auto"}; // "auto", "depth", "colour", "none" + bool solid{true}; + std::string card_title; + std::vector> card; // label, value + std::string path; // in the plugin's data folder + }; + virtual std::string capture_save(std::string_view /*plugin*/, const CaptureSave&) { return unavailable; } // Game buttons kept from the game while asked (overlay::hide_game_buttons/hold_game_buttons), // and all game input paused for half a second at a time. virtual void game_buttons(std::string_view /*plugin*/, std::uint16_t /*hidden*/, std::uint16_t /*held*/, bool /*paused*/) {} @@ -440,6 +492,8 @@ public: virtual std::string files_read(std::string_view /*plugin*/, std::string_view /*path*/, std::string& /*text*/) { return unavailable; } virtual std::string files_write(std::string_view /*plugin*/, std::string_view /*path*/, std::string_view /*text*/) { return unavailable; } virtual std::string files_remove(std::string_view /*plugin*/, std::string_view /*path*/) { return unavailable; } + // The data folder (or a folder in it) opened in the player's file manager. + virtual std::string files_reveal(std::string_view /*plugin*/, std::string_view /*folder*/) { return unavailable; } // The same, for a file up to `most` bytes (an animation clip's, say). virtual std::string files_read_bytes(std::string_view plugin, std::string_view path, std::string& out, std::size_t /*most*/) { return files_read(plugin, path, out); @@ -605,6 +659,17 @@ public: virtual std::string outfit_find(std::string_view) { return {}; } virtual std::vector outfit_slots() { return {}; } virtual std::string outfit_status() { return unavailable; } + // What the skater wears now (the selected preset): `slot` the kind of item ("Headwear", "Top", ..., + // "Shoes", in the order worn from head to foot; clothes and accessories only), or the board's part + // ("Deck", "Grip tape", "Grip cutout", "Trucks", "Wheels"). + struct WornItem { + std::string slot, item, title; + }; + virtual std::vector outfit_worn() { return {}; } + virtual std::vector board_parts() { return {}; } + // An item's title as the game shows it (its catalog's, the one an item grid showed, or its key + // made readable). + virtual std::string item_title(std::string_view key) { return std::string(key); } // ---- board: what is drawn of the local skateboard (Extension/Skater/board_pose.h) ------------------ struct BoardBone { Vec3 position{}; diff --git a/Extension/Plugins/plugin_game_runtime.cpp b/Extension/Plugins/plugin_game_runtime.cpp index bb93335b..186a7c83 100644 --- a/Extension/Plugins/plugin_game_runtime.cpp +++ b/Extension/Plugins/plugin_game_runtime.cpp @@ -61,6 +61,11 @@ #include "Extension/UI/Overlay/screen_look.h" #include "Extension/UI/Overlay/scripted_pad.h" #include "Extension/World/character_room.h" +#include "Extension/Rendering/Capture/capture_runtime.h" +#include "Extension/Customization/item_browser.h" +#include "Extension/Customization/worn_items.h" +#include "Extension/Multiplayer/Hud/game_ui_state.h" +#include #include #include #include @@ -1102,6 +1107,100 @@ public: skater_animation::show_in_room(false); character_room::leave(); } + std::string room_turn(std::string_view, float value, int stick) override { + if (value != 0 && !multiplayer::sample_game_ui_state(base_).in_menu) return "the game's menu is not open"; + overlay::hold_stick(stick, value, 300); + return {}; + } + GameMenu game_menu() override { + GameMenu menu; + menu.open = multiplayer::sample_game_ui_state(base_).in_menu; + if (!menu.open) return menu; + const auto buttons = character_room::menu_buttons(base_); + menu.skaters = character_room::is_skaters_page(buttons); + std::size_t start = 0; + while (start < buttons.size() && menu.buttons.size() < 64) { + const auto end = std::min(buttons.find(' ', start), buttons.size()); + if (end > start) menu.buttons.push_back(buttons.substr(start, end - start)); + start = end + 1; + } + return menu; + } + std::optional menu_grid() override { + const auto view = item_browser::view(); + if (!view.open) return std::nullopt; + return MenuGrid{view.focused_key, view.focused_title, view.focused_category, view.grid_keys, view.shown, view.total}; + } + // ---- capture: one plugin records at a time (capture_runtime.h) -------------------------------- + std::string capture_start(std::string_view plugin, const CaptureStart& request) override { + auto& recorder = capture::recorder(); + const auto now = recorder.status().state; + const bool busy = now == capture::State::recording || now == capture::State::saving; + if (busy && !capture_owner_.empty() && capture_owner_ != plugin) return capture_owner_ + " is capturing"; + capture::RecordRequest r; + r.area = {request.left, request.top, request.right, request.bottom, request.menu}; + r.depth = request.depth; + r.height = request.height; + r.skip = request.skip; + r.most = request.most; + r.seconds = request.seconds; + std::string why; + if (!recorder.start(r, why)) return why; + capture_owner_ = plugin; + capture_file_.clear(); + return {}; + } + void capture_stop(std::string_view plugin) override { + if (capture_owner_ == plugin) capture::recorder().stop(); + } + void capture_discard(std::string_view plugin) override { + // While it saves, the save goes on and is still this plugin's to read. + if (capture_owner_ != plugin || capture::recorder().status().state == capture::State::saving) return; + capture::recorder().discard(); + capture_owner_.clear(); + } + CaptureStatus capture_status(std::string_view plugin) override { + if (capture_owner_ != plugin) return {"idle"}; + static constexpr const char* names[] = {"idle", "recording", "stopped", "saving", "saved", "failed"}; + const auto s = capture::recorder().status(); + CaptureStatus out{names[static_cast(s.state)], s.taken, s.kept, s.depth, s.message, {}}; + if (s.state == capture::State::saved) out.file = capture_file_; + return out; + } + float capture_compare(std::string_view plugin, std::uint32_t a, std::uint32_t b) override { + return capture_owner_ == plugin ? capture::recorder().compare(a, b) : -1.0f; + } + std::vector capture_kept(std::string_view plugin) override { + return capture_owner_ == plugin ? capture::recorder().kept() : std::vector{}; + } + std::string capture_save(std::string_view plugin, const CaptureSave& request) override { + if (capture_owner_ != plugin) return "nothing was recorded"; + std::filesystem::path root, file; + if (auto why = data_file(plugin, request.path, root, file); !why.empty()) return why; + std::error_code error; + if (std::filesystem::is_directory(file, error)) return "a folder is there"; + capture::SaveRequest r; + r.frames = request.frames; + r.format = request.format == "apng" ? capture::Format::apng : request.format == "png" ? capture::Format::png + : request.format == "webm" ? capture::Format::webm : capture::Format::gif; + r.fps = request.fps; + r.size = request.size; + if (request.background) + r.background = capture::Rgb{static_cast(*request.background >> 16), static_cast(*request.background >> 8), + static_cast(*request.background)}; + r.cutout = request.cutout == "depth" ? capture::Cutout::depth : request.cutout == "colour" ? capture::Cutout::colour + : request.cutout == "none" ? capture::Cutout::none : capture::Cutout::automatic; + r.solid = request.solid; + r.card_title = request.card_title; + for (const auto& [label, value] : request.card) r.card.push_back({label, value}); + r.fonts = capture::menu_fonts(); + r.file = file; + std::string why; + capture::recorder().stop(); + if (!capture::recorder().save(std::move(r), why)) return why; + capture_file_ = request.path; + return {}; + } RoomState room_state(std::string_view plugin) override { using P = character_room::Phase; const auto phase = character_room::phase(); @@ -1376,7 +1475,31 @@ public: return why; } bool frozen() const noexcept { return !freezers_.empty(); } + std::vector outfit_worn() override { + std::vector out; + for (auto& w : worn_items::worn_outfit()) out.push_back({std::move(w.slot), std::move(w.item), std::move(w.title)}); + return out; + } + std::vector board_parts() override { + std::vector out; + for (auto& w : worn_items::worn_board()) out.push_back({std::move(w.slot), std::move(w.item), std::move(w.title)}); + return out; + } + std::string item_title(std::string_view key) override { return worn_items::item_title(std::string(key)); } + std::string files_reveal(std::string_view plugin, std::string_view folder) override { + std::filesystem::path root, at; + if (auto why = data_file(plugin, folder, root, at); !why.empty()) return why; + std::error_code error; + std::filesystem::create_directories(at, error); + if (error || !std::filesystem::is_directory(at, error)) return "not a folder"; + const auto done = reinterpret_cast(ShellExecuteW(nullptr, L"open", at.c_str(), nullptr, nullptr, SW_SHOWNORMAL)); + return done > 32 ? std::string{} : std::string("the folder could not be opened"); + } void release(std::string_view plugin) override { + if (capture_owner_ == plugin) { + capture::recorder().discard(); + capture_owner_.clear(); + } if (hold_owner_ == plugin) (void)skater_hold(plugin, false); plugin_feed::release_hidden(std::string(plugin)); world_entities::clear(plugin); @@ -1564,6 +1687,7 @@ private: } std::uintptr_t base_; + std::string capture_owner_, capture_file_; // the plugin whose capture it is, and where it saved to std::string camera_holder_, follower_, camera_status_; std::array camera_pose_{}; float camera_fov_{}; diff --git a/Extension/Plugins/plugin_manifest.cpp b/Extension/Plugins/plugin_manifest.cpp index 4d47b9fc..914fb012 100644 --- a/Extension/Plugins/plugin_manifest.cpp +++ b/Extension/Plugins/plugin_manifest.cpp @@ -15,13 +15,13 @@ namespace { // throwdown kit (modes, documents, the native flag); "variants" choosing which of its own mod's // variants is in the game (reskate.package.choose). A manifest's "resident" list and "variants" are // the merge's (Engine/Vfs/mod_list.h ResidentAssets, Variants), not the plugin host's. -constexpr std::array permissions{ +constexpr std::array permissions{ "console", "storage", "overlay", "menu", "ui.screen", "input", "hud", "events.tricks", "events.skater", "skater.read", "skater.teleport", "skater.control", "skater.animation", "trainer", "camera", "world.raycast", "world.objects", "world.entities", "world.environment", "world.ai", "travel", "session.read", "session.skaters", "session.nametags", "net.session", "net.http", "voice.control", "chat.read", "chat.send", "chat.local", "server", "throwdowns", "replay", "music", "progression.read", "settings.read", "audio.ui", "audio", "outfit", "board", "mods.read", - "events.physics", "game.speed", "files", "variants", "settings.write", "room"}; + "events.physics", "game.speed", "files", "variants", "settings.write", "room", "capture"}; } bool known_permission(std::string_view name) noexcept { diff --git a/Extension/Plugins/plugin_vm.h b/Extension/Plugins/plugin_vm.h index 5a52a752..12bef132 100644 --- a/Extension/Plugins/plugin_vm.h +++ b/Extension/Plugins/plugin_vm.h @@ -263,6 +263,7 @@ private: friend struct WorldApi; friend struct ServicesApi; friend struct KitApi; + friend struct CaptureApi; struct Command { std::string help; int function{}; }; struct Timer { double due{}, every{}; int function{}; }; using Clock = std::chrono::steady_clock; @@ -359,8 +360,8 @@ private: int controls_{}, edits_{}, environments_{}, voices_{}, musics_{}; double travel_at_{-1e9}, chat_at_{-1e9}, local_chat_second_{}, server_second_{}, host_at_{-1e9}, wipeout_at_{-1e9}; int local_chats_{}, server_calls_{}; - double files_second_{}, dialog_second_{}; - int file_calls_{}, dialog_calls_{}; + double files_second_{}, dialog_second_{}, reveal_second_{}; + int file_calls_{}, dialog_calls_{}, reveals_{}; std::optional> trainer_point_; std::set> binds_; std::vector screens_; diff --git a/Extension/Rendering/Capture/Test/capture_tests.cpp b/Extension/Rendering/Capture/Test/capture_tests.cpp new file mode 100644 index 00000000..171df409 --- /dev/null +++ b/Extension/Rendering/Capture/Test/capture_tests.cpp @@ -0,0 +1,438 @@ +#include "Extension/Rendering/Capture/capture_encode.h" +#include "Extension/Rendering/Capture/capture_matte.h" +#include "Extension/Rendering/Capture/capture_panel.h" +#include "Extension/Rendering/Capture/capture_recorder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Captures without the game (reskate.capture): matting, resizing, the card, the file formats, and a +// recording of a turning board or skater found, cut out and saved. +// Run with the folder of the menu fonts (External/fonts) as the first argument to test the card too, +// and a folder as the second to save a picture of it there. +namespace { +using namespace dingosdk::capture; +int failures = 0; + +void expect(bool condition, const char* what) { + if (condition) return; + ++failures; + std::printf("FAIL: %s\n", what); +} + +std::uint32_t be32(const std::uint8_t* p) { return std::uint32_t{p[0]} << 24 | std::uint32_t{p[1]} << 16 | std::uint32_t{p[2]} << 8 | p[3]; } + +// The chunk types of a PNG, in order. +std::vector chunks(const std::vector& png) { + std::vector out; + for (std::size_t at = 8; at + 12 <= png.size();) { + const auto size = be32(png.data() + at); + out.emplace_back(reinterpret_cast(png.data() + at + 4), 4); + at += 12 + size; + } + return out; +} + +// A board-ish test picture: an opaque rounded bar with a soft edge, a see-through middle band and a +// fully transparent background. +Image board(std::uint32_t size, float turn) { + Image image(size, size); + const auto n = static_cast(size); + const float c = n / 2.0f, half_w = n * 0.18f * std::abs(std::cos(turn)) + 2, half_h = n * 0.42f; + for (std::uint32_t y = 0; y < size; ++y) + for (std::uint32_t x = 0; x < size; ++x) { + const float fx = static_cast(x) + 0.5f, fy = static_cast(y) + 0.5f; + const float dx = std::abs(fx - c) - half_w, dy = std::abs(fy - c) - half_h; + const float edge = std::clamp(0.5f - std::max(dx, dy), 0.0f, 1.0f); + auto* p = image.at(x, y); + p[0] = static_cast(40 + y * 200 / size); + p[1] = 140; + p[2] = static_cast(220 - x * 100 / size); + const bool band = std::abs(fy - c) < n * 0.05f; + p[3] = static_cast(std::lround(edge * (band ? 120 : 255))); + } + return image; +} + +// `image` over a background, then "graded" the way the game's post-processing shifts both captures. +std::vector composite(const Image& image, std::array background) { + std::vector out(image.rgba.size()); + for (std::size_t i = 0; i < image.rgba.size(); i += 4) { + const float a = image.rgba[i + 3] / 255.0f; + for (int c = 0; c < 3; ++c) { + const float v = image.rgba[i + c] * a + background[c] * (1 - a); + out[i + 2 - c] = static_cast(std::lround(std::clamp(v * 0.9f + 6.0f, 0.0f, 255.0f))); // BGRA + } + out[i + 3] = 255; + } + return out; +} + +void test_matte() { + const auto truth = board(96, 0.3f); + const auto dark = composite(truth, {10, 12, 14}), light = composite(truth, {235, 236, 240}); + const PixelView d{dark.data(), 96, 96, 96 * 4, true}, l{light.data(), 96, 96, 96 * 4, true}; + const Rect all{0, 0, 96, 96}; + const auto backgrounds = measure_backgrounds(d, l, all); + expect(std::abs(backgrounds.dark[0] - (10 * 0.9f + 6)) < 1 && std::abs(backgrounds.light[2] - (240 * 0.9f + 6)) < 1, + "backgrounds are measured from the edge"); + const auto cut = matte_pair(d, l, all, backgrounds, 0.0f); + int worst_alpha = 0, worst_colour = 0; + for (std::uint32_t y = 0; y < 96; ++y) + for (std::uint32_t x = 0; x < 96; ++x) { + const auto* want = truth.at(x, y); + const auto* got = cut.at(x, y); + worst_alpha = std::max(worst_alpha, std::abs(want[3] - got[3])); + if (want[3] > 60) + for (int c = 0; c < 3; ++c) + worst_colour = std::max(worst_colour, std::abs(static_cast(want[c] * 0.9f + 6) - got[c])); + } + expect(worst_alpha <= 3, "matting recovers alpha, soft edges and the see-through band included"); + expect(worst_colour <= 6, "matting recovers the board's colour without the background in it"); + expect(content_bounds(cut).width < 96, "the cut-out has transparent space around the board"); + + const auto same = matte_pair(d, d, all, measure_backgrounds(d, d, all)); + expect(same.at(0, 0)[3] == 255, "identical captures are taken as opaque, not empty"); +} + +// A whole turn of a board with a black grip over a dark radial vignette, the Board screen's worst +// case: where the board has been, the backdrop must come back see-through, not as a dark ghost. +void test_backdrop_key(bool light_both_sides) { + constexpr std::uint32_t w = 300, h = 500, turn_frames = 120; + std::vector frames, truth; + for (std::uint32_t f = 0; f < turn_frames; ++f) { + const float angle = 6.2831853f * static_cast(f) / turn_frames; + const float half_w = 55 * std::abs(std::cos(angle)) + 3, half_h = 200; + const bool grip = std::cos(angle) < 0; + Image frame(w, h), want(w, h); + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const float fx = static_cast(x) + 0.5f, fy = static_cast(y) + 0.5f; + const float rx = (fx - 120) / 300, ry = (fy - 230) / 400; + // A vignette no simple surface fits: a soft spot, a kink and a slow ripple. + const float bg = 24 + 30 * std::exp(-(rx * rx + ry * ry) * 5) + 8 * std::max(0.0f, 0.4f - std::abs(rx)) + + 5 * std::sin(fy / 37); + const float dx = std::abs(fx - 150) - half_w, dy = std::abs(fy - 250) - half_h; + const float cover = std::clamp(0.5f - std::max(dx, dy), 0.0f, 1.0f); + // A white deck with a black grip; or one light on both sides (a white grip), which covers + // the middle for most of the turn: there the backdrop is seen least. + const float back = light_both_sides ? 200.0f : 12.0f; + const float deck[3] = {grip ? back : 226.0f, grip ? back : 220.0f, grip ? back + 1 : 208.0f}; + auto* p = frame.at(x, y); + for (int c = 0; c < 3; ++c) p[c] = static_cast(std::lround(deck[c] * cover + (bg + static_cast(c)) * (1 - cover))); + p[3] = 255; + want.at(x, y)[3] = static_cast(std::lround(cover * 255)); + } + frames.push_back(std::move(frame)); + truth.push_back(std::move(want)); + } + key_against_backdrop(frames); + std::uint64_t ghost = 0, lost = 0, background = 0, board = 0; + for (std::uint32_t f = 0; f < turn_frames; ++f) + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto want = truth[f].at(x, y)[3], got = frames[f].at(x, y)[3]; + if (want == 0) ++background, ghost += got > 64; + if (want == 255) ++board, lost += got < 192; + } + std::printf("backdrop key (%s): %.3f%% ghost, %.3f%% lost\n", light_both_sides ? "light both sides" : "black grip", 100.0 * static_cast(ghost) / static_cast(background), + 100.0 * static_cast(lost) / static_cast(board)); + expect(ghost * 1000 < background, "where the board has been comes back see-through"); + expect(lost * 100 < board, "a black grip on the dark backdrop stays board"); +} + +void test_geometry() { + const auto square = padded_square({10, 20, 30, 50}, 5, 200, 100); + expect(square.width == 60 && square.height == 60, "padded_square grows and squares"); + expect(square.x + square.width <= 200 && square.y + square.height <= 100, "padded_square stays inside"); + const auto u = united({0, 0, 10, 10}, {20, 5, 5, 20}); + expect(u.x == 0 && u.y == 0 && u.width == 25 && u.height == 25, "united covers both"); + const auto small = fitted(board(200, 0), 50); + expect(small.width == 50 && small.height == 50, "fitted keeps a square square"); + expect(small.at(25, 2)[3] == 0 && small.at(25, 15)[3] > 200, "fitted keeps the background clear and the board solid"); + // Premultiplied filtering: an opaque red pixel next to transparent green does not turn olive. + Image edge(2, 1); + edge.rgba = {255, 0, 0, 255, 0, 255, 0, 0}; + const auto halved = fitted(edge, 1); + expect(halved.at(0, 0)[0] == 255 && halved.at(0, 0)[1] == 0 && halved.at(0, 0)[3] == 128, "fitted does not bleed transparent colour"); + const auto flat = over_matte(halved, {0, 0, 255}); + expect(flat.at(0, 0)[3] == 255 && flat.at(0, 0)[0] == 128 && flat.at(0, 0)[2] == 127, "over_matte flattens"); +} + +void test_png() { + Animation animation; + animation.fps = 24; + for (int i = 0; i < 6; ++i) animation.frames.push_back(board(64, static_cast(i) * 0.5f)); + const auto apng = encode_apng(animation); + expect(apng.size() > 8 && std::memcmp(apng.data(), "\x89PNG\r\n\x1a\n", 8) == 0, "APNG signature"); + const auto types = chunks(apng); + const auto count = [&](const char* t) { return std::count(types.begin(), types.end(), std::string(t)); }; + expect(types.front() == "IHDR" && types[1] == "acTL" && types.back() == "IEND", "APNG chunk order"); + expect(count("fcTL") == 6 && count("IDAT") == 1 && count("fdAT") == 5, "APNG has one control and data chunk per frame"); + const auto still = encode_png(animation.frames.front()); + expect(chunks(still).size() == 3, "PNG is IHDR, IDAT, IEND"); +} + +// Decodes a GIF's first frame's LZW data back to indices, to check the encoder's code-size steps. +std::vector first_frame_indices(const std::vector& gif, std::size_t pixels) { + std::size_t at = 13 + 768; + while (at < gif.size() && gif[at] == 0x21) { // extensions + at += 2; + while (gif[at]) at += gif[at] + 1u; + ++at; + } + at += 10; // image descriptor + const int min_bits = gif[at++]; + std::vector data; + while (gif[at]) { + data.insert(data.end(), gif.begin() + static_cast(at + 1), gif.begin() + static_cast(at + 1 + gif[at])); + at += gif[at] + 1u; + } + const int clear = 1 << min_bits, end = clear + 1; + std::vector> table; + const auto reset = [&] { + table.assign(static_cast(end) + 1, {}); + for (int i = 0; i < clear; ++i) table[static_cast(i)] = {static_cast(i)}; + }; + reset(); + int bits = min_bits + 1, previous = -1; + std::size_t bit = 0; + std::vector out; + while (bit + static_cast(bits) <= data.size() * 8 && out.size() < pixels) { + int code = 0; + for (int k = 0; k < bits; ++k, ++bit) code |= ((data[bit / 8] >> (bit % 8)) & 1) << k; + if (code == clear) { reset(), bits = min_bits + 1, previous = -1; continue; } + if (code == end) break; + std::vector entry; + if (code < static_cast(table.size())) entry = table[static_cast(code)]; + else if (previous >= 0) entry = table[static_cast(previous)], entry.push_back(table[static_cast(previous)][0]); + else return {}; + out.insert(out.end(), entry.begin(), entry.end()); + if (previous >= 0 && table.size() < 4096) { + auto added = table[static_cast(previous)]; + added.push_back(entry[0]); + table.push_back(std::move(added)); + } + if (static_cast(table.size()) >= (1 << bits) && bits < 12) ++bits; + previous = code; + } + return out; +} + +void test_gif() { + // Noise fills the LZW table several times over, so clears and every code width get used. + Image noise(300, 200); + std::uint32_t seed = 7; + for (std::size_t i = 0; i < noise.rgba.size(); i += 4) { + seed = seed * 1664525u + 1013904223u; + noise.rgba[i] = static_cast(seed >> 24), noise.rgba[i + 1] = static_cast(seed >> 16); + noise.rgba[i + 2] = static_cast(seed >> 8), noise.rgba[i + 3] = (seed >> 7) & 1 ? 255 : 0; + } + Animation animation; + animation.frames = {noise, noise}; + const auto gif = encode_gif(animation, {.fringe = {0, 0, 0}, .dither = false}); + expect(std::memcmp(gif.data(), "GIF89a", 6) == 0 && gif.back() == 0x3b, "GIF header and trailer"); + const auto indices = first_frame_indices(gif, 300 * 200); + expect(indices.size() == 300 * 200, "GIF LZW data decodes to every pixel"); + bool transparent_right = indices.size() == 300 * 200; + for (std::size_t i = 0; transparent_right && i < indices.size(); ++i) + transparent_right = (indices[i] == 255) == (noise.rgba[i * 4 + 3] < 128); + expect(transparent_right, "GIF cuts out exactly the transparent pixels"); + + Animation timing; + timing.fps = 30; + timing.frames.assign(3, board(16, 0)); + const auto timed = encode_gif(timing); + int total = 0; + for (std::size_t i = 0; i + 5 < timed.size(); ++i) + if (timed[i] == 0x21 && timed[i + 1] == 0xf9) total += timed[i + 4] | timed[i + 5] << 8; + expect(total == 10, "GIF delays add up to the clip's length"); +} + +void test_webm() { + Animation animation; + animation.fps = 30; + for (int i = 0; i < 4; ++i) animation.frames.push_back(board(48, static_cast(i) * 0.4f)); + const auto webm = encode_webm(animation); + const auto has = [&](std::initializer_list bytes) { + return std::search(webm.begin(), webm.end(), bytes.begin(), bytes.end()) != webm.end(); + }; + expect(webm.size() > 4 && webm[0] == 0x1a && webm[1] == 0x45 && webm[2] == 0xdf && webm[3] == 0xa3, "WebM starts with an EBML header"); + expect(has({'V', '_', 'V', 'P', '8'}), "WebM track is VP8"); + expect(has({0x53, 0xc0}), "a see-through WebM marks its track AlphaMode"); + expect(has({0x9d, 0x01, 0x2a}), "WebM frames are VP8 key frames"); + Animation opaque; + opaque.frames = {over_matte(board(48, 0), {0, 0, 0})}; + const auto flat = encode_webm(opaque); + const std::array alpha_mode{0x53, 0xc0}; + expect(std::search(flat.begin(), flat.end(), alpha_mode.begin(), alpha_mode.end()) == flat.end(), + "an opaque WebM has no alpha"); +} + +void test_panel(const std::filesystem::path& fonts, const char* out) { + const auto load = [&](const char* name) { + std::ifstream in(fonts / name, std::ios::binary); + return std::vector(std::istreambuf_iterator(in), {}); + }; + const auto brush = load("PermanentMarker-Regular.ttf"), heading = load("Montserrat-ExtraBold.ttf"), body = load("Montserrat-SemiBold.ttf"); + const std::vector parts{{"Deck", "ReSkate Scrolling Pattern"}, + {"Grip tape", "A grip tape with a name long enough that it has to wrap onto a second line and then some more"}, + {"Trucks", "Silver"}, {"Wheels", ""}}; + const auto card = render_parts_panel("Board", parts, 512, {heading, body, brush}); + expect(card.height == 512 && card.width > 300, "the parts card is the height asked for"); + expect(card.at(0, 0)[3] < 40 && card.at(card.width / 2, card.height - 4)[3] > 200, "the card has rounded corners and a solid body"); + int bright = 0; + for (std::size_t i = 0; i < card.rgba.size(); i += 4) bright += card.rgba[i] > 200; + expect(bright > 500, "the card has text on it"); + const auto shown = side_by_side(fitted(board(300, 0.2f), 512), card); + expect(shown.width == 512 + card.width, "side_by_side lays the card beside the board"); + if (out) { // a folder to save the card in, to look at it + const auto png = encode_png(shown); + std::ofstream(std::filesystem::path(out) / "parts_card.png", std::ios::binary).write(reinterpret_cast(png.data()), static_cast(png.size())); + } +} +} // namespace + +// A stand-in for the game: a 1280x720 frame with a board that turns a little every frame on the right +// of the menu (or a skater in the middle, swaying as they stand), a piece of UI that never moves, and +// the game's depth when asked for. +struct FakeGame { + double turn{}, spin{1.0 / 37.3}; + bool skater{}, with_depth{}; + unsigned drawn{}; + std::vector last_; + std::vector frame() { + turn += spin; + ++drawn; + auto now = draw(); + // As the game's temporal smoothing does, a frame keeps a trace of the last one. + if (!last_.empty()) + for (std::size_t i = 0; i < now.size(); ++i) now[i] = static_cast((now[i] * 3 + last_[i] + 2) / 4); + last_ = now; + return now; + } + float centre_x() const { return 1280 * (skater ? 0.5f : 0.76f) + (skater ? 3.0f * std::sin(static_cast(drawn) * 0.21f) : 0.0f); } + float half_width() const { return 60 * std::abs(std::cos(static_cast(turn) * 6.2831853f)) + 3; } + std::vector draw() const { + constexpr std::uint32_t w = 1280, h = 720; + std::vector out(std::size_t{w} * h * 4); + const float cx = centre_x(), cy = 720 * 0.57f, half_w = half_width(), half_h = 230; + const bool back = std::cos(static_cast(turn) * 6.2831853f) < 0; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const float fx = static_cast(x) + 0.5f, fy = static_cast(y) + 0.5f; + const float bg = 28.0f + 10.0f * fy / h; + float rgb[3] = {bg, bg, bg + 4}; + if (x > 860 && x < 900 && y > 140 && y < 170) rgb[0] = rgb[1] = rgb[2] = 200; // UI that never moves + const float dx = std::abs(fx - cx) - half_w, dy = std::abs(fy - cy) - half_h; + const float cover = std::clamp(0.5f - std::max(dx, dy), 0.0f, 1.0f); + const float deck[3] = {back ? 12.0f : 40.0f, back ? 12.0f : 150.0f + 80 * fy / h, back ? 13.0f : 220.0f}; + for (int c = 0; c < 3; ++c) rgb[c] = deck[c] * cover + rgb[c] * (1 - cover); + auto* p = out.data() + (std::size_t{y} * w + x) * 4; + for (int c = 0; c < 3; ++c) p[2 - c] = static_cast(std::clamp(rgb[c] * 0.92f + 5, 0.0f, 255.0f)); + p[3] = 255; + } + return out; + } + // The game's depth for the frame drawn last: the board a quarter as far as the backdrop. + std::vector depth() const { + constexpr std::uint32_t w = 1280, h = 720; + const float cx = centre_x(), cy = 720 * 0.57f, half_w = half_width(), half_h = 230; + std::vector out(std::size_t{w} * h, 1.0f / 400); + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) + if (std::abs(static_cast(x) + 0.5f - cx) < half_w && std::abs(static_cast(y) + 0.5f - cy) < half_h) + out[std::size_t{y} * w + x] = 1.0f / 100; + return out; + } +}; + +// A recording of the board (or skater) turning, as a plugin makes one: its frames compared to find +// the turn, the frames evenly spaced through it saved. +void test_recording(const std::filesystem::path& fonts, Format format, bool skater, bool with_depth, std::uint32_t frames) { + const auto load = [&](const char* name) { + std::ifstream in(fonts / name, std::ios::binary); + return std::vector(std::istreambuf_iterator(in), {}); + }; + static const auto brush = load("PermanentMarker-Regular.ttf"), heading = load("Montserrat-ExtraBold.ttf"), + body = load("Montserrat-SemiBold.ttf"); + FakeGame game; + game.skater = skater; + game.with_depth = with_depth; + Recorder recorder; + RecordRequest request; + request.area = skater ? Area{0.27f, 0.14f, 0.735f, 0.985f, true} : Area{0.655f, 0.185f, 0.86f, 0.985f, true}; + request.depth = with_depth; + request.height = 480; + request.skip = 3; + request.most = 120; + std::string why; + expect(recorder.start(request, why), "a recording starts"); + for (int i = 0; i < 400 && recorder.status().state == State::recording; ++i) { + const bool wanted = recorder.wants_frame(); + const auto pixels = game.frame(); + if (!wanted) continue; + const auto depth = with_depth ? game.depth() : std::vector{}; + recorder.take_frame({pixels.data(), 1280, 720, 1280 * 4, true, with_depth ? reinterpret_cast(depth.data()) : nullptr, + 1280 * 4}); + } + const auto taken = recorder.status(); + expect(taken.state == State::stopped && taken.taken == 120, "it stops by itself at the frames asked for"); + expect(taken.depth == with_depth, "it says whether every frame was cut out by depth"); + // Comes round: about 37 frames on it looks as it started, half way it does not. + const float round = recorder.compare(0, 37), half = recorder.compare(0, 19); + expect(round >= 0 && half > 3 * round + 1, "a whole turn on, a frame looks like the first again; half way it does not"); + expect(recorder.compare(0, 500) < 0, "a frame not taken is not compared"); + const auto kept = recorder.kept(); + expect(kept.size() == 120 && kept.front() == 0 && kept.back() == 119, "every frame is kept while there is room"); + SaveRequest save; + for (std::uint32_t j = 0; j < frames; ++j) save.frames.push_back(static_cast(std::lround(37.3 * j / frames))); + save.format = format; + save.size = 256; + save.solid = !skater; + save.card_title = skater ? "Outfit" : "Board"; + save.card = {{"Deck", "Test Deck"}, {"Wheels", "Round"}}; + save.fonts = {heading, body, brush}; + save.file = std::filesystem::temp_directory_path() / "capture_tests" / (std::string("turn.") + extension(format)); + expect(recorder.save(save, why), "it saves"); + std::string second; + expect(!recorder.save(save, second), "one save at a time"); + for (int i = 0; i < 600 && recorder.status().state == State::saving; ++i) std::this_thread::sleep_for(std::chrono::milliseconds(50)); + const auto saved = recorder.status(); + if (saved.state != State::saved) + std::printf("recording (%s, %s, %s): %s\n", extension(format), skater ? "skater" : "board", with_depth ? "depth" : "colour", saved.message.c_str()); + expect(saved.state == State::saved, "the recording is saved"); + std::error_code error; + expect(std::filesystem::file_size(saved.file, error) > 1000, "its file is written"); + if (with_depth) expect(saved.message.find("by depth") != std::string::npos, "cut out by the game's depth when it had one"); + else expect(saved.message.find("by colour") != std::string::npos, "else by colour"); + recorder.discard(); + expect(recorder.status().state == State::idle && recorder.kept().empty(), "discarded, nothing is held"); +} + +int main(int argc, char** argv) { + test_matte(); + test_backdrop_key(false); + test_backdrop_key(true); + test_geometry(); + test_png(); + test_gif(); + test_webm(); + if (argc > 1) { + test_panel(argv[1], argc > 2 ? argv[2] : nullptr); + test_recording(argv[1], Format::apng, false, false, 16); + test_recording(argv[1], Format::gif, false, true, 24); + test_recording(argv[1], Format::webm, true, false, 16); + test_recording(argv[1], Format::png, true, true, 1); + } else std::printf("skipped: the card and recordings (pass the fonts folder)\n"); + if (!failures) std::printf("capture tests passed\n"); + return failures ? 1 : 0; +} diff --git a/Extension/Rendering/Capture/capture_commands.cpp b/Extension/Rendering/Capture/capture_commands.cpp new file mode 100644 index 00000000..c7f89cc0 --- /dev/null +++ b/Extension/Rendering/Capture/capture_commands.cpp @@ -0,0 +1,48 @@ +#include "Extension/Console/commands.h" +#include "Extension/UI/Overlay/overlay.h" +#include +#include +#include +#include + +namespace dingosdk::console { +namespace { +std::filesystem::path game_folder() { + std::wstring path(MAX_PATH, L'\0'); + for (;;) { + const auto length = GetModuleFileNameW(nullptr, path.data(), static_cast(path.size())); + if (length < path.size()) { + path.resize(length); + break; + } + path.resize(path.size() * 2); + } + return std::filesystem::path(path).parent_path(); +} +} // namespace + +// Research aids for screen captures (reskate.capture), from Fit Check's `fitcheck depth`. +void register_capture_commands(Commands& registry) { + auto arguments = argument("verb", Type::text, true); + arguments.complete = [](const Model&, auto) { return std::vector{"depth"}; }; + auto command = action("capture", + "Research aids for plugins' screen captures (reskate.capture): depth (the game's depth buffers over the next frames, " + "as pictures with depth-probe.txt in Captures\\depth beside Skate.exe). Answers go to the log.", + Group::console, {std::move(arguments)}); + command.run = [](const Model&, const Values& values, const Output& out) { + std::istringstream words(values.empty() ? std::string{} : std::get(values[0])); + std::string verb; + words >> verb; + if (verb == "depth") { + const auto folder = game_folder() / L"Captures" / L"depth"; + std::error_code error; + std::filesystem::create_directories(folder, error); + overlay::request_depth_probe(folder.c_str()); + out("Taking the game's depth buffers over the next frames: depth-probe.txt and depth-*.pgm go to Captures\\depth."); + } else { + out("capture depth"); + } + }; + registry.add(std::move(command)); +} +} // namespace dingosdk::console diff --git a/Extension/Rendering/Capture/capture_encode.h b/Extension/Rendering/Capture/capture_encode.h new file mode 100644 index 00000000..51740d20 --- /dev/null +++ b/Extension/Rendering/Capture/capture_encode.h @@ -0,0 +1,38 @@ +#pragma once +#include "capture_image.h" +#include +#include + +// File formats for a screen capture (reskate.capture). Every frame of an animation is the same size. +namespace dingosdk::capture { +struct Animation { + std::vector frames; + std::uint32_t fps{30}; +}; + +// A still PNG (RGBA). +std::vector encode_png(const Image& image); +// An animated PNG that loops forever. Keeps full 8-bit alpha; flatten the frames over a matte +// (over_matte) first for an opaque one. +std::vector encode_apng(const Animation& animation); + +struct GifOptions { + // GIF transparency is all or nothing. Pixels at least half opaque are kept, blended over this + // colour so their soft edge does not go black; the rest are cut out. Pick something close to + // the page the GIF will sit on. + Rgb fringe{40, 40, 40}; + // A light, fixed (ordered) dither, which does not crawl between frames the way error diffusion does. + bool dither{true}; +}; +// An animated GIF that loops forever: one 256-colour palette for every frame. +// GIF timing is in hundredths of a second, so the frame rate is kept on average, at most 50. +std::vector encode_gif(const Animation& animation, const GifOptions& options = {}); + +struct WebmOptions { + int quality{12}; // VP8 quantizer index for the colour, 0 (best) .. 127 + int alpha_quality{4}; // and for the transparency, which shows every block edge, so finer +}; +// A WebM with VP8 video; frames with any transparency carry it as VP8 alpha (Chrome, Firefox and +// ffmpeg's libvpx decoder show it). WebM has no loop flag: players loop it themselves. +std::vector encode_webm(const Animation& animation, const WebmOptions& options = {}); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_gif.cpp b/Extension/Rendering/Capture/capture_gif.cpp new file mode 100644 index 00000000..6cffadeb --- /dev/null +++ b/Extension/Rendering/Capture/capture_gif.cpp @@ -0,0 +1,229 @@ +#include "capture_encode.h" +#include +#include +#include + +// Animated GIF (GIF89a) for captures: one median-cut palette shared by every frame. +namespace dingosdk::capture { +namespace { +constexpr int bins = 32 * 32 * 32; // 5 bits a channel + +std::uint32_t key_of(int r, int g, int b) noexcept { + return static_cast((r >> 3) << 10 | (g >> 3) << 5 | (b >> 3)); +} + +// 4x4 Bayer matrix centred on zero, in steps of one 5-bit level (8 of 255) across its range. +constexpr std::array bayer{0, 8, 2, 10, 12, 4, 14, 6, 3, 11, 1, 9, 15, 7, 13, 5}; +int dither_offset(std::uint32_t x, std::uint32_t y) noexcept { return bayer[(y & 3) * 4 + (x & 3)] / 2 - 4; } + +struct Histogram { + std::vector count = std::vector(bins); + std::vector> sum = std::vector>(bins); +}; + +struct Box { + std::vector keys; // occupied bins + std::uint64_t weight{}; +}; + +// The bin's channel `c` (0 r, 1 g, 2 b), 0..31. +int channel(std::uint32_t key, int c) noexcept { return static_cast(key >> (10 - c * 5)) & 31; } + +std::vector median_cut(const Histogram& h, std::size_t colours) { + std::vector boxes(1); + for (std::uint32_t k = 0; k < bins; ++k) + if (h.count[k]) boxes[0].keys.push_back(k), boxes[0].weight += h.count[k]; + if (boxes[0].keys.empty()) return {Rgb{0, 0, 0}}; + while (boxes.size() < colours) { + // Split the box with the most pixels times its widest spread. + std::size_t pick = boxes.size(); + std::uint64_t best = 0; + int best_channel = 0; + for (std::size_t i = 0; i < boxes.size(); ++i) { + if (boxes[i].keys.size() < 2) continue; + for (int c = 0; c < 3; ++c) { + int lo = 31, hi = 0; + for (const auto k : boxes[i].keys) lo = std::min(lo, channel(k, c)), hi = std::max(hi, channel(k, c)); + const auto score = boxes[i].weight * static_cast(hi - lo); + if (score > best) best = score, pick = i, best_channel = c; + } + } + if (pick == boxes.size()) break; // nothing left to split + auto& box = boxes[pick]; + std::sort(box.keys.begin(), box.keys.end(), + [c = best_channel](auto a, auto b) { return channel(a, c) < channel(b, c); }); + // Split at the weighted median. + std::uint64_t running = 0; + std::size_t cut = 1; + for (; cut < box.keys.size(); ++cut) { + running += h.count[box.keys[cut - 1]]; + if (running * 2 >= box.weight) break; + } + cut = std::clamp(cut, 1, box.keys.size() - 1); + Box other; + other.keys.assign(box.keys.begin() + static_cast(cut), box.keys.end()); + box.keys.resize(cut); + box.weight = 0; + for (const auto k : box.keys) box.weight += h.count[k]; + for (const auto k : other.keys) other.weight += h.count[k]; + boxes.push_back(std::move(other)); + } + std::vector palette; + for (const auto& box : boxes) { + std::array total{}; + for (const auto k : box.keys) + for (int c = 0; c < 3; ++c) total[c] += h.sum[k][c]; + Rgb colour{}; + for (int c = 0; c < 3; ++c) colour[c] = static_cast((total[c] + box.weight / 2) / box.weight); + palette.push_back(colour); + } + return palette; +} + +// Variable-length LZW (GIF's flavour) of 8-bit indices. +std::vector lzw(const std::vector& indices) { + constexpr int min_bits = 8, clear = 1 << min_bits, end = clear + 1; + std::vector out; + std::uint32_t bit_buffer = 0; + int bit_count = 0, code_bits = min_bits + 1, next = end + 1; + const auto emit = [&](int code) { + bit_buffer |= static_cast(code) << bit_count; + bit_count += code_bits; + while (bit_count >= 8) out.push_back(static_cast(bit_buffer)), bit_buffer >>= 8, bit_count -= 8; + }; + std::vector table(4096 * 256, -1); // (prefix code, next index) -> code + const auto reset = [&] { + std::fill(table.begin(), table.end(), static_cast(-1)); + code_bits = min_bits + 1; + next = end + 1; + }; + emit(clear); + if (indices.empty()) { + emit(end); + } else { + int prefix = indices[0]; + for (std::size_t i = 1; i < indices.size(); ++i) { + const auto byte = indices[i]; + const auto slot = static_cast(prefix) * 256 + byte; + if (table[slot] >= 0) { prefix = table[slot]; continue; } + emit(prefix); + if (next < 4096) { + table[slot] = static_cast(next++); + if (next > (1 << code_bits) && code_bits < 12) ++code_bits; + } else { + emit(clear); + reset(); + } + prefix = byte; + } + emit(prefix); + emit(end); + } + if (bit_count) out.push_back(static_cast(bit_buffer)); + return out; +} + +void put16(std::vector& out, std::uint32_t v) { + out.push_back(static_cast(v)); + out.push_back(static_cast(v >> 8)); +} +} // namespace + +std::vector encode_gif(const Animation& animation, const GifOptions& options) { + if (animation.frames.empty() || animation.frames.front().empty()) throw std::invalid_argument("No frames to save."); + const auto width = animation.frames.front().width, height = animation.frames.front().height; + if (width > 65535 || height > 65535) throw std::invalid_argument("Too large for a GIF."); + for (const auto& frame : animation.frames) + if (frame.width != width || frame.height != height) throw std::invalid_argument("The frames are not all the same size."); + + // Every kept pixel, its soft edge blended over the fringe colour. + const auto kept = [&](const std::uint8_t* p, Rgb& colour) { + if (p[3] < 128) return false; + for (int c = 0; c < 3; ++c) + colour[c] = static_cast((p[c] * p[3] + options.fringe[c] * (255 - p[3]) + 127) / 255); + return true; + }; + Histogram histogram; + bool transparent = false; + for (const auto& frame : animation.frames) + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + Rgb colour; + if (!kept(frame.at(x, y), colour)) { transparent = true; continue; } + const auto k = key_of(colour[0], colour[1], colour[2]); + ++histogram.count[k]; + for (int c = 0; c < 3; ++c) histogram.sum[k][c] += colour[c]; + } + auto palette = median_cut(histogram, transparent ? 255 : 256); + const auto colours = palette.size(); // the rest of the table is padding, never matched + const auto transparent_index = static_cast(colours); + // Nearest palette colour of each bin, worked out when first needed. + std::vector nearest(bins, -1); + const auto map = [&](int r, int g, int b) { + const auto k = key_of(r, g, b); + if (nearest[k] < 0) { + const int cr = (r & ~7) + 4, cg = (g & ~7) + 4, cb = (b & ~7) + 4; + int best = 0, best_distance = INT32_MAX; + for (std::size_t i = 0; i < colours; ++i) { + const int dr = palette[i][0] - cr, dg = palette[i][1] - cg, db = palette[i][2] - cb; + // Weighted roughly by how much the eye cares about each. + const int d = 3 * dr * dr + 4 * dg * dg + 2 * db * db; + if (d < best_distance) best_distance = d, best = static_cast(i); + } + nearest[k] = static_cast(best); + } + return static_cast(nearest[k]); + }; + + std::vector out = {'G', 'I', 'F', '8', '9', 'a'}; + put16(out, width); + put16(out, height); + out.push_back(0xf7); // global colour table of 256 entries + out.push_back(0); + out.push_back(0); + palette.resize(256, Rgb{0, 0, 0}); + for (const auto& colour : palette) out.insert(out.end(), colour.begin(), colour.end()); + // Loop forever (NETSCAPE2.0). + out.insert(out.end(), {0x21, 0xff, 0x0b, 'N', 'E', 'T', 'S', 'C', 'A', 'P', 'E', '2', '.', '0', 0x03, 0x01, 0x00, 0x00, 0x00}); + + const auto fps = std::clamp(animation.fps, 1, 50); + std::uint64_t elapsed = 0; // hundredths of a second written so far + std::vector indices(std::size_t{width} * height); + for (std::size_t f = 0; f < animation.frames.size(); ++f) { + const auto& frame = animation.frames[f]; + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + Rgb colour; + auto& index = indices[std::size_t{y} * width + x]; + if (!kept(frame.at(x, y), colour)) { index = transparent_index; continue; } + const int offset = options.dither ? dither_offset(x, y) : 0; + index = map(std::clamp(colour[0] + offset, 0, 255), std::clamp(colour[1] + offset, 0, 255), + std::clamp(colour[2] + offset, 0, 255)); + } + const auto due = (f + 1) * 100 / fps; + const auto delay = static_cast(std::max(2, due - elapsed)); + elapsed += delay; + // Graphic control: clear to the background between frames when anything is see-through. + out.insert(out.end(), {0x21, 0xf9, 0x04, static_cast(transparent ? 0x09 : 0x04)}); + put16(out, delay); + out.push_back(transparent ? transparent_index : 0); + out.push_back(0); + out.push_back(0x2c); + put16(out, 0); + put16(out, 0); + put16(out, width); + put16(out, height); + out.push_back(0); + out.push_back(8); // LZW minimum code size + const auto data = lzw(indices); + for (std::size_t i = 0; i < data.size(); i += 255) { + const auto n = std::min(255, data.size() - i); + out.push_back(static_cast(n)); + out.insert(out.end(), data.begin() + static_cast(i), data.begin() + static_cast(i + n)); + } + out.push_back(0); + } + out.push_back(0x3b); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_image.cpp b/Extension/Rendering/Capture/capture_image.cpp new file mode 100644 index 00000000..568e9833 --- /dev/null +++ b/Extension/Rendering/Capture/capture_image.cpp @@ -0,0 +1,171 @@ +#include "capture_image.h" +#include + +namespace dingosdk::capture { +namespace { +// Per-pixel weights of one axis of an area (box) resample from `from` to `to` samples. +struct Tap { std::uint32_t first{}; std::vector weights; }; +std::vector area_taps(std::uint32_t from, std::uint32_t to) { + std::vector taps(to); + const double scale = static_cast(from) / to; + for (std::uint32_t i = 0; i < to; ++i) { + auto& tap = taps[i]; + if (scale <= 1.0) { + // Enlarging: linear between the two nearest source samples. + const double centre = (i + 0.5) * scale - 0.5; + const double floor = std::floor(centre); + const auto left = static_cast(floor); + const auto t = static_cast(centre - floor); + const auto a = static_cast(std::clamp(left, 0, from - 1)); + const auto b = static_cast(std::clamp(left + 1, 0, from - 1)); + tap.first = a; + tap.weights.assign(b - a + 1, 0.0f); + tap.weights[0] += 1.0f - t; + tap.weights[b - a] += t; + continue; + } + const double start = i * scale, end = (i + 1) * scale; + tap.first = static_cast(std::floor(start)); + const auto last = std::min(from - 1, static_cast(std::ceil(end)) - 1); + float total = 0; + for (auto s = tap.first; s <= last; ++s) { + const auto w = static_cast(std::min(end, s + 1.0) - std::max(start, s)); + tap.weights.push_back(w); + total += w; + } + for (auto& w : tap.weights) w /= total; + } + return taps; +} +} // namespace + +Image over_matte(const Image& image, Rgb matte) { + Image out = image; + for (std::size_t i = 0; i < out.rgba.size(); i += 4) { + const unsigned a = out.rgba[i + 3]; + for (int c = 0; c < 3; ++c) + out.rgba[i + c] = static_cast((out.rgba[i + c] * a + matte[c] * (255 - a) + 127) / 255); + out.rgba[i + 3] = 255; + } + return out; +} + +Rect content_bounds(const Image& image, std::uint8_t threshold) { + std::uint32_t x0 = image.width, y0 = image.height, x1 = 0, y1 = 0; + for (std::uint32_t y = 0; y < image.height; ++y) + for (std::uint32_t x = 0; x < image.width; ++x) + if (image.at(x, y)[3] >= threshold) { + x0 = std::min(x0, x), y0 = std::min(y0, y); + x1 = std::max(x1, x + 1), y1 = std::max(y1, y + 1); + } + if (x0 >= x1 || y0 >= y1) return {}; + return {x0, y0, x1 - x0, y1 - y0}; +} + +Rect united(Rect a, Rect b) noexcept { + if (a.empty()) return b; + if (b.empty()) return a; + const auto x0 = std::min(a.x, b.x), y0 = std::min(a.y, b.y); + const auto x1 = std::max(a.x + a.width, b.x + b.width), y1 = std::max(a.y + a.height, b.y + b.height); + return {x0, y0, x1 - x0, y1 - y0}; +} + +Rect padded_square(Rect rect, std::uint32_t margin, std::uint32_t width, std::uint32_t height) noexcept { + if (rect.empty() || !width || !height) return {}; + const auto side = std::min({std::max(rect.width, rect.height) + margin * 2, width, height}); + const auto centre_x = rect.x + rect.width / 2, centre_y = rect.y + rect.height / 2; + const auto place = [side](std::uint32_t centre, std::uint32_t limit) { + const auto half = side / 2; + const auto start = centre > half ? centre - half : 0u; + return std::min(start, limit - side); + }; + return {place(centre_x, width), place(centre_y, height), side, side}; +} + +Image square_around(const Image& image, Rect content, std::uint32_t margin) { + if (content.empty()) return {}; + const auto side = std::max(content.width, content.height) + margin * 2; + // The square's corner, which may lie outside the image (left of it, above it). + const auto x0 = static_cast(content.x) + static_cast(content.width / 2) - static_cast(side / 2); + const auto y0 = static_cast(content.y) + static_cast(content.height / 2) - static_cast(side / 2); + Image out(side, side); + for (std::uint32_t y = 0; y < side; ++y) + for (std::uint32_t x = 0; x < side; ++x) { + const auto sx = x0 + x, sy = y0 + y; + if (sx >= 0 && sy >= 0 && sx < image.width && sy < image.height) + std::copy_n(image.at(static_cast(sx), static_cast(sy)), 4, out.at(x, y)); + } + return out; +} + +Image cropped(const Image& image, Rect rect) { + Image out(rect.width, rect.height); + for (std::uint32_t y = 0; y < rect.height; ++y) + for (std::uint32_t x = 0; x < rect.width; ++x) { + const auto sx = rect.x + x, sy = rect.y + y; + if (sx < image.width && sy < image.height) std::copy_n(image.at(sx, sy), 4, out.at(x, y)); + } + return out; +} + +Image resized(const Image& image, std::uint32_t w, std::uint32_t h) { + if (image.empty() || !w || !h) return {}; + const auto columns = area_taps(image.width, w), rows = area_taps(image.height, h); + // Horizontal pass into premultiplied floats, then vertical. + std::vector wide(std::size_t{w} * image.height * 4); + for (std::uint32_t y = 0; y < image.height; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto& tap = columns[x]; + float sum[4]{}; + for (std::size_t k = 0; k < tap.weights.size(); ++k) { + const auto* p = image.at(tap.first + static_cast(k), y); + const float a = p[3] / 255.0f, weight = tap.weights[k]; + for (int c = 0; c < 3; ++c) sum[c] += p[c] * a * weight; + sum[3] += p[3] * weight; + } + std::copy_n(sum, 4, wide.data() + (std::size_t{y} * w + x) * 4); + } + Image out(w, h); + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto& tap = rows[y]; + float sum[4]{}; + for (std::size_t k = 0; k < tap.weights.size(); ++k) { + const auto* p = wide.data() + ((tap.first + k) * w + x) * 4; + for (int c = 0; c < 4; ++c) sum[c] += p[c] * tap.weights[k]; + } + auto* d = out.at(x, y); + const float alpha = std::clamp(sum[3], 0.0f, 255.0f); + d[3] = static_cast(std::lround(alpha)); + for (int c = 0; c < 3; ++c) + d[c] = d[3] ? static_cast(std::clamp(std::lround(sum[c] * 255.0f / alpha), 0L, 255L)) : 0; + } + return out; +} + +Image fitted(const Image& image, std::uint32_t size) { + if (image.empty() || !size) return {}; + const double scale = static_cast(size) / std::max(image.width, image.height); + const auto w = std::max(1, static_cast(std::lround(image.width * scale))); + const auto h = std::max(1, static_cast(std::lround(image.height * scale))); + const auto small = resized(image, w, h); + Image out(size, size); + const auto left = (size - w) / 2, top = (size - h) / 2; + for (std::uint32_t y = 0; y < h; ++y) std::copy_n(small.at(0, y), std::size_t{w} * 4, out.at(left, top + y)); + return out; +} + +Image side_by_side(const Image& a, const Image& b) { + if (a.empty()) return b; + if (b.empty()) return a; + Image out(a.width + b.width, std::max(a.height, b.height)); + const auto place = [&out](const Image& part, std::uint32_t left) { + const auto top = (out.height - part.height) / 2; + for (std::uint32_t y = 0; y < part.height; ++y) + std::copy_n(part.at(0, y), std::size_t{part.width} * 4, out.at(left, top + y)); + }; + place(a, 0); + place(b, a.width); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_image.h b/Extension/Rendering/Capture/capture_image.h new file mode 100644 index 00000000..c759f633 --- /dev/null +++ b/Extension/Rendering/Capture/capture_image.h @@ -0,0 +1,75 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +// The pictures a capture is made of: straight (not premultiplied) 8-bit RGBA, rows top to +// bottom with no padding. Kept free of Windows and the game so the encoders and matting can be +// tested anywhere (Extension/Rendering/Capture/Test). +namespace dingosdk::capture { +using Rgb = std::array; + +struct Image { + std::uint32_t width{}, height{}; + std::vector rgba; // width * height * 4 + + Image() = default; + Image(std::uint32_t w, std::uint32_t h) : width(w), height(h), rgba(std::size_t{w} * h * 4, 0) {} + bool empty() const noexcept { return !width || !height; } + std::uint8_t* at(std::uint32_t x, std::uint32_t y) noexcept { return rgba.data() + (std::size_t{y} * width + x) * 4; } + const std::uint8_t* at(std::uint32_t x, std::uint32_t y) const noexcept { + return rgba.data() + (std::size_t{y} * width + x) * 4; + } +}; + +// A rectangle of pixels in a captured frame. +struct Rect { + std::uint32_t x{}, y{}, width{}, height{}; + bool empty() const noexcept { return !width || !height; } +}; + +// Rows of 4-byte pixels as read back from the GPU: `pitch` bytes apart, channel order given by `bgra`. +struct PixelView { + const std::uint8_t* data{}; + std::uint32_t width{}, height{}, pitch{}; + bool bgra{}; + // The game's depth for the same frame, when there is one: 32-bit floats a pixel, reversed (1 + // near, 0 at the far plane), `depth_pitch` bytes a row, the frame's size. + const std::uint8_t* depth{}; + std::uint32_t depth_pitch{}; + float depth_at(std::uint32_t x, std::uint32_t y) const noexcept { + float v; + std::memcpy(&v, depth + std::size_t{y} * depth_pitch + std::size_t{x} * 4, 4); + return v; + } + std::array rgb(std::uint32_t x, std::uint32_t y) const noexcept { + const auto* p = data + std::size_t{y} * pitch + std::size_t{x} * 4; + return bgra ? std::array{p[2], p[1], p[0]} : std::array{p[0], p[1], p[2]}; + } +}; + +// `image` flattened over `matte`: every pixel opaque. +Image over_matte(const Image& image, Rgb matte); +// The smallest rectangle holding every pixel at least `threshold` opaque, or an empty one. +Rect content_bounds(const Image& image, std::uint8_t threshold = 8); +// The union of two rectangles (an empty one counts as nothing). +Rect united(Rect a, Rect b) noexcept; +// `rect` grown by `margin` on every side and made square, kept inside `width` x `height`. +Rect padded_square(Rect rect, std::uint32_t margin, std::uint32_t width, std::uint32_t height) noexcept; +// A square around `content`, `margin` wider on every side, centred on it: it may reach past the +// image, and what is outside is transparent (a board taller than the area it was taken from). +Image square_around(const Image& image, Rect content, std::uint32_t margin); +// The part of `image` inside `rect`; pixels outside the image are transparent. +Image cropped(const Image& image, Rect rect); +// `image` resized to exactly `width` x `height` (filtered as fitted is). +Image resized(const Image& image, std::uint32_t width, std::uint32_t height); +// `image` resized to fit inside `size` x `size` (keeping its shape, centred, transparent around it), +// filtered in premultiplied alpha so edges do not pick up the colour of transparent pixels. +Image fitted(const Image& image, std::uint32_t size); +// `b` placed to the right of `a`, both centred vertically on a canvas as tall as the taller. +Image side_by_side(const Image& a, const Image& b); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_matte.cpp b/Extension/Rendering/Capture/capture_matte.cpp new file mode 100644 index 00000000..a3062aa4 --- /dev/null +++ b/Extension/Rendering/Capture/capture_matte.cpp @@ -0,0 +1,226 @@ +#include "capture_matte.h" +#include +#include +#include +#include + +namespace dingosdk::capture { +Backgrounds measure_backgrounds(const PixelView& dark, const PixelView& light, Rect area) noexcept { + // The median of each channel along the edge: a stray UI element on it does not shift it. + Backgrounds out; + if (area.empty()) return out; + std::array, 6> counts{}; + std::uint64_t total = 0; + const auto add = [&](std::uint32_t x, std::uint32_t y) { + const auto d = dark.rgb(x, y), l = light.rgb(x, y); + for (std::size_t c = 0; c < 3; ++c) ++counts[c][d[c]], ++counts[3 + c][l[c]]; + ++total; + }; + const auto right = area.x + area.width - 1, bottom = area.y + area.height - 1; + for (auto x = area.x; x <= right; ++x) add(x, area.y), add(x, bottom); + for (auto y = area.y + 1; y < bottom; ++y) add(area.x, y), add(right, y); + for (std::size_t k = 0; k < 6; ++k) { + std::uint64_t seen = 0; + std::size_t value = 0; + while (value < 255 && (seen += counts[k][value]) * 2 < total) ++value; + (k < 3 ? out.dark : out.light)[k % 3] = static_cast(value); + } + return out; +} + +Image matte_pair(const PixelView& dark, const PixelView& light, Rect area, const Backgrounds& backgrounds, + float floor) { + Image out(area.width, area.height); + std::array step{}; + float step_power = 0; + for (int c = 0; c < 3; ++c) { + step[c] = backgrounds.light[c] - backgrounds.dark[c]; + step_power += step[c] * step[c]; + } + // Backgrounds that look the same tell nothing apart: everything is taken as opaque. + const bool usable = step_power > 3 * 16.0f * 16.0f; + floor = std::clamp(floor, 0.0f, 0.45f); + for (std::uint32_t y = 0; y < area.height; ++y) + for (std::uint32_t x = 0; x < area.width; ++x) { + const auto d = dark.rgb(area.x + x, area.y + y), l = light.rgb(area.x + x, area.y + y); + auto* p = out.at(x, y); + float seen = 0; // how much of the background shows: 0 opaque .. 1 empty + if (usable) { + // Least squares over the three channels of (light - dark) = seen * step. + float dot = 0; + for (int c = 0; c < 3; ++c) dot += (static_cast(l[c]) - d[c]) * step[c]; + seen = std::clamp((dot / step_power - floor) / (1.0f - 2.0f * floor), 0.0f, 1.0f); + } + const float alpha = 1.0f - seen; + if (alpha <= 0.0f) { p[0] = p[1] = p[2] = p[3] = 0; continue; } + for (int c = 0; c < 3; ++c) { + // The board's own colour, from both captures with their background taken away. + const float a = d[c] - seen * backgrounds.dark[c], b = l[c] - seen * backgrounds.light[c]; + p[c] = static_cast(std::clamp(std::lround((a + b) * 0.5f / alpha), 0L, 255L)); + } + p[3] = static_cast(std::lround(alpha * 255.0f)); + } + return out; +} +// Without a background of its own to swap, the board is keyed out against the backdrop, found from +// the turn itself. The backdrop keeps still and, across any one row, changes only gently (the Board +// screen's vignette); the board covers part of each row at most. So: +// 1. each pixel's steady colour over the turn (the one it keeps most of the time) is a candidate; +// 2. along each row, a candidate is the backdrop when it is close to the row's typical candidate: +// one the board covers for most of the turn (a white deck, a white grip in the middle) has a +// board colour for its steady one, and is not; +// 3. where a row has no backdrop candidate the backdrop is carried across from those either side; +// 4. the backdrop at a pixel is then the colour it shows, in any frame, close to that estimate. +// An exact backdrop lets the key be tight, so a black grip against the dark backdrop still reads as +// board. Matting (a stage that swaps the background) is better still; this is for when there is none. +void key_against_backdrop(std::vector& frames, bool solid_rows) { + if (frames.empty()) return; + const auto raw = frames; + const auto w = frames.front().width, h = frames.front().height; + if (w < 8 || h < 8) return; + const auto luma = [](const std::uint8_t* p) { return (77 * p[0] + 150 * p[1] + 29 * p[2]) >> 8; }; + Image plate(w, h); + std::vector values(frames.size()), sorted(frames.size()); + std::vector> candidate(w); + std::vector has(w), backdrop(w); + std::vector> previous_row; + for (std::uint32_t y = 0; y < h; ++y) { + // 1. Steady colours along the row. + for (std::uint32_t x = 0; x < w; ++x) { + for (std::size_t f = 0; f < frames.size(); ++f) values[f] = luma(frames[f].at(x, y)); + sorted = values; + std::nth_element(sorted.begin(), sorted.begin() + static_cast(sorted.size() / 2), sorted.end()); + const int middle = sorted[sorted.size() / 2]; + std::array sum{}; + std::size_t steady = 0; + for (std::size_t f = 0; f < frames.size(); ++f) + if (std::abs(values[f] - middle) <= 4) { + ++steady; + for (std::size_t c = 0; c < 3; ++c) sum[c] += frames[f].at(x, y)[c]; + } + has[x] = steady * 100 >= frames.size() * 35; + for (std::size_t c = 0; c < 3; ++c) candidate[x][c] = has[x] ? static_cast(sum[c] / static_cast(steady)) : 0.0f; + } + // 2. The row's typical candidate, and those close to it. Close to it locally too: compared + // with the median of the candidates within half the row each way, so a vignette's slope across + // the row is followed. + const std::uint32_t reach = std::max(4, w / 2); + std::array, 3> window; + for (std::uint32_t x = 0; x < w; ++x) { + backdrop[x] = false; + if (!has[x]) continue; + const auto from = x > reach ? x - reach : 0u, to = std::min(w, x + reach + 1); + for (auto& v : window) v.clear(); + for (auto k = from; k < to; ++k) + if (has[k]) + for (std::size_t c = 0; c < 3; ++c) window[c].push_back(candidate[k][c]); + float off = 0; + for (std::size_t c = 0; c < 3; ++c) { + auto& v = window[c]; + std::nth_element(v.begin(), v.begin() + static_cast(v.size() / 2), v.end()); + off = std::max(off, std::abs(candidate[x][c] - v[v.size() / 2])); + } + backdrop[x] = off <= 12.0f; + } + // 3. Carried across the gaps from the backdrop either side (or from the row above). + std::vector> estimate(w); + std::int64_t left = -1; + for (std::uint32_t x = 0; x <= w; ++x) { + if (x < w && !backdrop[x]) continue; + const auto right = static_cast(x); + for (auto k = left + 1; k < right; ++k) { + const auto i = static_cast(k); + if (left < 0 && right >= static_cast(w)) { + estimate[i] = previous_row.empty() ? std::array{} : previous_row[i]; + } else if (left < 0) { + estimate[i] = candidate[static_cast(right)]; + } else if (right >= static_cast(w)) { + estimate[i] = candidate[static_cast(left)]; + } else { + const float t = static_cast(k - left) / static_cast(right - left); + for (std::size_t c = 0; c < 3; ++c) + estimate[i][c] = candidate[static_cast(left)][c] + + (candidate[static_cast(right)][c] - candidate[static_cast(left)][c]) * t; + } + } + if (x < w) estimate[x] = candidate[x], left = right; + } + previous_row = estimate; + // 4. What each pixel shows that is close to its estimate: the backdrop, exactly. + for (std::uint32_t x = 0; x < w; ++x) { + std::array sum{}; + std::size_t fits = 0; + for (const auto& frame : frames) { + const auto* p = frame.at(x, y); + float off = 0; + for (std::size_t c = 0; c < 3; ++c) off = std::max(off, std::abs(static_cast(p[c]) - estimate[x][c])); + if (off > 8.0f) continue; + ++fits; + for (std::size_t c = 0; c < 3; ++c) sum[c] += p[c]; + } + auto* b = plate.at(x, y); + for (std::size_t c = 0; c < 3; ++c) + b[c] = static_cast(std::clamp(std::lround(fits >= 2 ? sum[c] / static_cast(fits) : estimate[x][c]), 0L, 255L)); + } + } + for (auto& frame : frames) + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + auto* p = frame.at(x, y); + const auto* b = plate.at(x, y); + int distance = 0; + for (std::size_t c = 0; c < 3; ++c) distance = std::max(distance, std::abs(p[c] - b[c])); + const float alpha = std::clamp((static_cast(distance) - 3.0f) / 9.0f, 0.0f, 1.0f); + if (alpha <= 0) { p[0] = p[1] = p[2] = p[3] = 0; continue; } + for (std::size_t c = 0; c < 3; ++c) + p[c] = static_cast(std::clamp(std::lround((p[c] - (1 - alpha) * b[c]) / alpha), 0L, 255L)); + p[3] = static_cast(std::lround(alpha * 255)); + } + // Part of what turns can match the backdrop and key out as a hole in it: a dark joint or shirt, + // part of a deck's graphic. A pocket it closes all round is filled from the frame. A board is + // solid, so any pocket in it is board; a skater's bigger pockets are real backdrop (between an + // arm and the body). (Not filled across each row: the backdrop between a deck and its wheels is + // open above and below the truck, and stays clear.) + for (std::size_t f = 0; f < frames.size(); ++f) { + auto& frame = frames[f]; + const std::size_t n = std::size_t{w} * h; + std::vector outside(n); + std::vector todo; + std::size_t opaque = 0; + const auto clear = [&](std::size_t p) { return frame.rgba[p * 4 + 3] < 128; }; + for (std::size_t p = 0; p < n; ++p) { + if (!clear(p)) { ++opaque; continue; } + const auto x = p % w, y = p / w; + if (x == 0 || y == 0 || x + 1 == w || y + 1 == h) outside[p] = 1, todo.push_back(static_cast(p)); + } + const auto spread = [&](std::vector& mark, std::vector& stack, std::vector* region) { + while (!stack.empty()) { + const auto p = stack.back(); + stack.pop_back(); + if (region) region->push_back(p); + const auto x = p % w, y = p / w; + const std::uint32_t next[4] = {x > 0 ? p - 1 : p, x + 1 < w ? p + 1 : p, y > 0 ? p - w : p, y + 1 < h ? p + w : p}; + for (const auto q : next) + if (!mark[q] && clear(q)) mark[q] = 1, stack.push_back(q); + } + }; + spread(outside, todo, nullptr); + const auto pocket_most = std::max(16, solid_rows ? opaque / 2 : opaque * 3 / 100); + std::vector region; + for (std::size_t p = 0; p < n; ++p) { + if (outside[p] || !clear(p)) continue; + region.clear(); + outside[p] = 1; + todo.push_back(static_cast(p)); + spread(outside, todo, ®ion); + if (region.size() > pocket_most) continue; + for (const auto q : region) { + auto* px = frame.rgba.data() + std::size_t{q} * 4; + std::copy_n(raw[f].rgba.data() + std::size_t{q} * 4, 3, px); + px[3] = 255; + } + } + } +} + +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_matte.h b/Extension/Rendering/Capture/capture_matte.h new file mode 100644 index 00000000..f6db48f8 --- /dev/null +++ b/Extension/Rendering/Capture/capture_matte.h @@ -0,0 +1,29 @@ +#pragma once +#include "capture_image.h" + +// Transparency from two captures of the same still picture, one over a dark background and one over +// a light one ("difference matting"). Where the board covers the background both captures agree; +// where the background shows through they differ by the full background step; part-covered pixels +// (antialiased edges, the holo's see-through bits) differ by part of it. That gives soft edges the +// game's backbuffer cannot: it has no alpha of its own. +// +// The backgrounds do not have to be pure black and white. The game tone-maps and grades them, so +// what they actually look like on screen is measured (`Backgrounds`) and used instead. +namespace dingosdk::capture { +struct Backgrounds { + std::array dark{}, light{}; // the empty background's colour in each capture, 0..255 +}; +// The typical (median) colour of the pixels along the edge of `area` in each capture: the background, as long +// as the board is not touching the edge of the area. +Backgrounds measure_backgrounds(const PixelView& dark, const PixelView& light, Rect area) noexcept; +// The board cut out of `area`: straight RGBA, colour taken from the dark capture. +// `floor` is how much difference (0..1 of the background step) is still counted as fully opaque, to +// keep the noise of the game's dithering and film grain from leaving the board faintly see-through. +Image matte_pair(const PixelView& dark, const PixelView& light, Rect area, const Backgrounds& backgrounds, + float floor = 0.04f); +// Without a second background to compare against: the board keyed out of every frame of a whole +// turn (opaque captures of the same area) against the backdrop the turn reveals. See the .cpp. +// `solid_rows`: what turns is solid (a board), so any hole it closes all round is part of it. For a +// skater only small ones are: their arms and legs have backdrop between them. +void key_against_backdrop(std::vector& frames, bool solid_rows = true); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_panel.cpp b/Extension/Rendering/Capture/capture_panel.cpp new file mode 100644 index 00000000..05358585 --- /dev/null +++ b/Extension/Rendering/Capture/capture_panel.cpp @@ -0,0 +1,206 @@ +#include "capture_panel.h" +#include +#include +#include + +// stb_truetype comes with Dear ImGui; imgui_draw.cpp keeps its copy private (STBTT_STATIC), so +// this file compiles its own. +#define STBTT_STATIC +#define STB_TRUETYPE_IMPLEMENTATION +#if defined(_MSC_VER) +#pragma warning(push, 0) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wsign-conversion" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wdouble-promotion" +#endif +#include +#if defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +namespace dingosdk::capture { +namespace { +constexpr Rgb card{18, 20, 24}, white{245, 246, 248}, grey{150, 156, 168}, accent{40, 120, 255}; +constexpr std::uint8_t card_alpha = 230; + +// Code points of UTF-8 text; bad bytes come out as U+FFFD. +std::u32string decode(std::string_view text) { + std::u32string out; + for (std::size_t i = 0; i < text.size();) { + const auto b = static_cast(text[i]); + const int extra = b < 0x80 ? 0 : (b >> 5) == 6 ? 1 : (b >> 4) == 14 ? 2 : (b >> 3) == 30 ? 3 : -1; + if (extra < 0 || i + static_cast(extra) >= text.size() + (extra ? 0 : 1)) { + out.push_back(0xfffd), ++i; + continue; + } + char32_t c = extra ? b & (0x3f >> extra) : b; + bool ok = true; + for (int k = 1; k <= extra; ++k) { + const auto next = static_cast(text[i + static_cast(k)]); + if ((next & 0xc0) != 0x80) { ok = false; break; } + c = c << 6 | (next & 0x3f); + } + out.push_back(ok ? c : 0xfffd); + i += ok ? static_cast(extra) + 1 : 1; + } + return out; +} + +// Straight-alpha "over": `colour` at `coverage` (0..255) onto the pixel. +void blend(Image& image, std::int64_t x, std::int64_t y, Rgb colour, unsigned coverage) { + if (x < 0 || y < 0 || x >= image.width || y >= image.height || !coverage) return; + auto* p = image.at(static_cast(x), static_cast(y)); + const float a = static_cast(coverage) / 255.0f, under = static_cast(p[3]) / 255.0f, out = a + under * (1 - a); + for (int c = 0; c < 3; ++c) + p[c] = static_cast(std::lround((colour[c] * a + p[c] * under * (1 - a)) / out)); + p[3] = static_cast(std::lround(out * 255)); +} + +struct Font { + stbtt_fontinfo info{}; + explicit Font(std::span file) { + if (file.empty() || !stbtt_InitFont(&info, file.data(), stbtt_GetFontOffsetForIndex(file.data(), 0))) + throw std::runtime_error("A menu font could not be read."); + } + float scale(float pixels) const { return stbtt_ScaleForPixelHeight(&info, pixels); } + float ascent(float pixels) const { + int a = 0, d = 0, g = 0; + stbtt_GetFontVMetrics(&info, &a, &d, &g); + return static_cast(a) * scale(pixels); + } + float width(std::u32string_view text, float pixels) const { + const float s = scale(pixels); + float x = 0; + for (std::size_t i = 0; i < text.size(); ++i) { + int advance = 0, bearing = 0; + stbtt_GetCodepointHMetrics(&info, static_cast(text[i]), &advance, &bearing); + x += static_cast(advance) * s; + if (i + 1 < text.size()) + x += static_cast(stbtt_GetCodepointKernAdvance(&info, static_cast(text[i]), static_cast(text[i + 1]))) * s; + } + return x; + } + // Draws `text` with its top at `top`. + void draw(Image& image, std::u32string_view text, float left, float top, float pixels, Rgb colour) const { + const float s = scale(pixels), baseline = top + ascent(pixels); + float x = left; + std::vector glyph; + for (std::size_t i = 0; i < text.size(); ++i) { + const int code = static_cast(text[i]); + const float shift = x - std::floor(x); + int x0 = 0, y0 = 0, x1 = 0, y1 = 0; + stbtt_GetCodepointBitmapBoxSubpixel(&info, code, s, s, shift, 0, &x0, &y0, &x1, &y1); + const int w = x1 - x0, h = y1 - y0; + if (w > 0 && h > 0) { + glyph.assign(static_cast(w) * static_cast(h), 0); + stbtt_MakeCodepointBitmapSubpixel(&info, glyph.data(), w, h, w, s, s, shift, 0, code); + const auto gx = static_cast(std::floor(x)) + x0; + const auto gy = static_cast(std::lround(baseline)) + y0; + for (int row = 0; row < h; ++row) + for (int col = 0; col < w; ++col) + blend(image, gx + col, gy + row, colour, glyph[static_cast(row * w + col)]); + } + int advance = 0, bearing = 0; + stbtt_GetCodepointHMetrics(&info, code, &advance, &bearing); + x += static_cast(advance) * s; + if (i + 1 < text.size()) x += static_cast(stbtt_GetCodepointKernAdvance(&info, code, static_cast(text[i + 1]))) * s; + } + } +}; + +// `text` broken at spaces into at most `max_lines` lines of `width`; the last one cut short with "...". +std::vector wrapped(const Font& font, std::u32string text, float pixels, float width, std::size_t max_lines) { + std::vector lines; + while (!text.empty() && lines.size() < max_lines) { + if (font.width(text, pixels) <= width) { lines.push_back(text); break; } + if (lines.size() + 1 == max_lines) { + while (!text.empty() && font.width(text + U"...", pixels) > width) text.pop_back(); + while (!text.empty() && text.back() == U' ') text.pop_back(); + lines.push_back(text + U"..."); + break; + } + // The longest run of words that fits; a single word too long for a line is split anywhere. + std::size_t cut = 0; + for (std::size_t i = 1; i <= text.size(); ++i) { + if (font.width(std::u32string_view(text).substr(0, i), pixels) > width) break; + if (i == text.size() || text[i] == U' ') cut = i; + } + if (!cut) { + cut = 1; + while (cut < text.size() && font.width(std::u32string_view(text).substr(0, cut + 1), pixels) <= width) ++cut; + } + lines.push_back(text.substr(0, cut)); + text.erase(0, cut); + while (!text.empty() && text.front() == U' ') text.erase(0, 1); + } + return lines; +} + +// Coverage (0..1) of a rounded rectangle at a pixel centre. +float rounded(float px, float py, float w, float h, float radius) { + const float dx = std::max({radius - px, px - (w - radius), 0.0f}); + const float dy = std::max({radius - py, py - (h - radius), 0.0f}); + return std::clamp(radius - std::sqrt(dx * dx + dy * dy) + 0.5f, 0.0f, 1.0f); +} +} // namespace + +Image render_parts_panel(std::string_view title, std::span parts, std::uint32_t height, const PanelFonts& fonts) { + if (height < 64) throw std::invalid_argument("The parts card is too small."); + const Font brush(fonts.brush), heading(fonts.heading), body(fonts.body); + const auto H = static_cast(height); + const auto width = static_cast(std::lround(H * 0.72f)); + const float pad = H * 0.07f, inner = static_cast(width) - pad * 2; + + // Lay the card out at full size; if it does not fit, again with everything smaller. + struct Line { const Font* font; std::u32string text; float x, y, size; Rgb colour; }; + std::vector lines; + float scale = 1.0f, bottom = 0; + for (int attempt = 0; attempt < 8; ++attempt) { + lines.clear(); + const float title_size = H * 0.11f * scale, label_size = H * 0.042f * scale, value_size = H * 0.058f * scale; + float y = pad; + auto title_text = decode(title); + while (!title_text.empty() && brush.width(title_text, title_size) > inner) title_text.pop_back(); + lines.push_back({&brush, title_text, pad, y, title_size, white}); + y += title_size * 1.05f + H * 0.035f * scale; // room for the accent rule + for (const auto& part : parts) { + std::string label = part.slot; + for (auto& c : label) c = static_cast(std::toupper(static_cast(c))); + lines.push_back({&body, decode(label), pad, y, label_size, grey}); + y += label_size * 1.25f; + const auto name = part.name.empty() ? std::string("None") : part.name; + for (auto& text : wrapped(heading, decode(name), value_size, inner, 2)) { + lines.push_back({&heading, std::move(text), pad, y, value_size, white}); + y += value_size * 1.1f; + } + y += H * 0.03f * scale; + } + bottom = y; + if (bottom <= H - pad * 0.5f) break; + scale *= 0.88f; + } + + Image out(width, height); + const float radius = H * 0.04f; + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + const float cover = rounded(static_cast(x) + 0.5f, static_cast(y) + 0.5f, static_cast(width), H, radius); + auto* p = out.at(x, y); + p[0] = card[0], p[1] = card[1], p[2] = card[2]; + p[3] = static_cast(std::lround(cover * card_alpha)); + } + // The accent rule under the title. + const auto& title_line = lines.front(); + const float rule_y = title_line.y + title_line.size * 1.05f + H * 0.008f; + const float rule_w = std::min(inner, std::max(H * 0.12f, brush.width(title_line.text, title_line.size))); + for (auto y = static_cast(rule_y); y < static_cast(rule_y + std::max(2.0f, H * 0.008f)); ++y) + for (auto x = static_cast(pad); x < static_cast(pad + rule_w); ++x) blend(out, x, y, accent, 255); + for (const auto& line : lines) line.font->draw(out, line.text, line.x, line.y, line.size, line.colour); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_panel.h b/Extension/Rendering/Capture/capture_panel.h new file mode 100644 index 00000000..9ee8d54f --- /dev/null +++ b/Extension/Rendering/Capture/capture_panel.h @@ -0,0 +1,23 @@ +#pragma once +#include "capture_image.h" +#include +#include + +// The optional card drawn beside what a capture shows: a heading, then each +// part's slot and item name, in the menu's fonts. +namespace dingosdk::capture { +struct Part { + std::string slot; // "Deck", "Grip tape", "Trucks", "Wheels", ... + std::string name; // the item's title as the game shows it (UTF-8) +}; +// TrueType font files (the menu's: ReSkate builds them in as FONT_HEADING / FONT_BODY / FONT_BRUSH). +struct PanelFonts { + std::span heading; // the item names + std::span body; // the slot labels + std::span brush; // the title, in the game's brush lettering +}; +// A card `height` pixels tall (and about three quarters as wide), on a dark, mostly opaque rounded +// panel, so it reads over a transparent background as well as a matte. +Image render_parts_panel(std::string_view title, std::span parts, std::uint32_t height, + const PanelFonts& fonts); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_png.cpp b/Extension/Rendering/Capture/capture_png.cpp new file mode 100644 index 00000000..1b534711 --- /dev/null +++ b/Extension/Rendering/Capture/capture_png.cpp @@ -0,0 +1,150 @@ +#include "capture_encode.h" +#include +#include +#include +#include +#include + +// PNG and APNG (https://www.w3.org/TR/png-3/#apng-frame-control) for captures. +namespace dingosdk::capture { +namespace { +std::uint32_t png_crc(std::span bytes, std::uint32_t crc = 0) noexcept { + static const auto table = [] { + std::array t{}; + for (std::uint32_t n = 0; n < 256; ++n) { + auto c = n; + for (int k = 0; k < 8; ++k) c = c & 1 ? 0xedb88320u ^ (c >> 1) : c >> 1; + t[n] = c; + } + return t; + }(); + crc = ~crc; + for (const auto b : bytes) crc = table[(crc ^ b) & 0xff] ^ (crc >> 8); + return ~crc; +} + +struct Writer { + std::vector out; + void u32(std::vector& to, std::uint32_t v) { + to.insert(to.end(), {static_cast(v >> 24), static_cast(v >> 16), + static_cast(v >> 8), static_cast(v)}); + } + void u16(std::vector& to, std::uint16_t v) { + to.insert(to.end(), {static_cast(v >> 8), static_cast(v)}); + } + void chunk(std::string_view type, std::span data) { + u32(out, static_cast(data.size())); + const auto start = out.size(); + out.insert(out.end(), type.begin(), type.end()); + out.insert(out.end(), data.begin(), data.end()); + u32(out, png_crc({out.data() + start, out.size() - start})); + } +}; + +// The image's rows, each with the PNG filter that leaves the smallest sum of magnitudes (the usual +// heuristic), compressed with zlib. +std::vector image_data(const Image& image) { + const std::size_t stride = std::size_t{image.width} * 4; + std::vector filtered; + filtered.reserve((stride + 1) * image.height); + std::vector zero(stride, 0), candidate(stride), best(stride); + for (std::uint32_t y = 0; y < image.height; ++y) { + const auto* row = image.rgba.data() + y * stride; + const auto* up = y ? row - stride : zero.data(); + std::uint64_t best_cost = ~0ull; + std::uint8_t best_filter = 0; + for (std::uint8_t filter = 0; filter < 5; ++filter) { + std::uint64_t cost = 0; + for (std::size_t i = 0; i < stride; ++i) { + const int a = i >= 4 ? row[i - 4] : 0, b = up[i], c = i >= 4 ? up[i - 4] : 0; + int predicted = 0; + switch (filter) { + case 1: predicted = a; break; + case 2: predicted = b; break; + case 3: predicted = (a + b) / 2; break; + case 4: { + const int p = a + b - c, pa = std::abs(p - a), pb = std::abs(p - b), pc = std::abs(p - c); + predicted = pa <= pb && pa <= pc ? a : pb <= pc ? b : c; + break; + } + default: break; + } + candidate[i] = static_cast(row[i] - predicted); + cost += static_cast(std::abs(static_cast(candidate[i]))); + } + if (cost < best_cost) best_cost = cost, best_filter = filter, best.swap(candidate); + } + filtered.push_back(best_filter); + filtered.insert(filtered.end(), best.begin(), best.end()); + } + std::size_t size = 0; + void* packed = tdefl_compress_mem_to_heap(filtered.data(), filtered.size(), &size, TDEFL_WRITE_ZLIB_HEADER | 256); + if (!packed) throw std::runtime_error("PNG compression failed."); + std::vector out(static_cast(packed), static_cast(packed) + size); + mz_free(packed); + return out; +} + +void header(Writer& w, const Image& image) { + static constexpr std::uint8_t signature[] = {0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}; + w.out.assign(std::begin(signature), std::end(signature)); + std::vector ihdr; + w.u32(ihdr, image.width); + w.u32(ihdr, image.height); + ihdr.insert(ihdr.end(), {8, 6, 0, 0, 0}); // 8-bit RGBA, deflate, adaptive filters, not interlaced + w.chunk("IHDR", ihdr); +} +} // namespace + +std::vector encode_png(const Image& image) { + if (image.empty()) throw std::invalid_argument("The picture is empty."); + Writer w; + header(w, image); + w.chunk("IDAT", image_data(image)); + w.chunk("IEND", {}); + return std::move(w.out); +} + +std::vector encode_apng(const Animation& animation) { + if (animation.frames.empty() || animation.frames.front().empty()) throw std::invalid_argument("No frames to save."); + const auto& first = animation.frames.front(); + for (const auto& frame : animation.frames) + if (frame.width != first.width || frame.height != first.height) + throw std::invalid_argument("The frames are not all the same size."); + Writer w; + header(w, first); + std::vector actl; + w.u32(actl, static_cast(animation.frames.size())); + w.u32(actl, 0); // loop forever + w.chunk("acTL", actl); + std::uint32_t sequence = 0; + const auto fps = static_cast(std::clamp(animation.fps, 1, 120)); + for (std::size_t i = 0; i < animation.frames.size(); ++i) { + const auto& frame = animation.frames[i]; + std::vector fctl; + w.u32(fctl, sequence++); + w.u32(fctl, frame.width); + w.u32(fctl, frame.height); + w.u32(fctl, 0); + w.u32(fctl, 0); + w.u16(fctl, 1); // delay: 1 / fps seconds + w.u16(fctl, fps); + // Each frame is whole: clear what the last one left (dispose to background) and replace, + // not blend, so see-through parts of the board stay see-through. + fctl.insert(fctl.end(), {1, 0}); + w.chunk("fcTL", fctl); + auto data = image_data(frame); + if (!i) { + w.chunk("IDAT", data); + continue; + } + std::vector fdat; + fdat.reserve(data.size() + 4); + w.u32(fdat, sequence++); + fdat.insert(fdat.end(), data.begin(), data.end()); + w.chunk("fdAT", fdat); + } + w.chunk("IEND", {}); + return std::move(w.out); +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_recorder.cpp b/Extension/Rendering/Capture/capture_recorder.cpp new file mode 100644 index 00000000..8a1945d7 --- /dev/null +++ b/Extension/Rendering/Capture/capture_recorder.cpp @@ -0,0 +1,465 @@ +#include "capture_recorder.h" +#include "capture_matte.h" +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::capture { +namespace { +// `area` of a frame, copied out in the frame's own channel order. +std::vector copy_area(const PixelView& frame, Rect area) { + std::vector out(std::size_t{area.width} * area.height * 4); + for (std::uint32_t y = 0; y < area.height; ++y) + std::copy_n(frame.data + std::size_t{area.y + y} * frame.pitch + std::size_t{area.x} * 4, std::size_t{area.width} * 4, + out.data() + std::size_t{y} * area.width * 4); + return out; +} + +Image opaque_image(const PixelView& view) { + Image out(view.width, view.height); + for (std::uint32_t y = 0; y < view.height; ++y) + for (std::uint32_t x = 0; x < view.width; ++x) { + const auto rgb = view.rgb(x, y); + auto* p = out.at(x, y); + p[0] = rgb[0], p[1] = rgb[1], p[2] = rgb[2], p[3] = 255; + } + return out; +} +} // namespace + +// What turns, by the game's depth: nearer than the backdrop behind it. The backdrop's distance for +// each row is that of the area's outer columns (the backdrop, or the floor where the row meets it: +// the camera is level, so the floor is about as far all along a row), and what is 8% nearer +// than that is the board or skater. The game's UI is never in the depth, and neither are a colour's +// likeness to the backdrop or the floor's reflection. Into `picture`'s alpha; false when the depth +// shows nothing there. +bool depth_mask(const PixelView& frame, Rect area, Image& picture) { + if (!frame.depth || picture.width != area.width || picture.height != area.height) return false; + const auto w = area.width, h = area.height; + const auto margin = std::max(2, w * 6 / 100); + std::vector edge; + float behind = 0; + std::size_t found = 0; + for (std::uint32_t y = 0; y < h; ++y) { + edge.clear(); + for (std::uint32_t x = 0; x < w; x = x + 1 == margin ? w - margin : x + 1) { + const float d = frame.depth_at(area.x + x, area.y + y); + if (d > 1e-7f) edge.push_back(1.0f / d); + } + if (!edge.empty()) { + std::nth_element(edge.begin(), edge.begin() + static_cast(edge.size() / 2), edge.end()); + behind = edge[edge.size() / 2]; + } + for (std::uint32_t x = 0; x < w; ++x) { + const float d = frame.depth_at(area.x + x, area.y + y); + const bool in_front = d > 1e-7f && (behind <= 0 || 1.0f / d < behind * 0.92f); + picture.at(x, y)[3] = in_front ? 255 : 0; + found += in_front; + } + } + // The soles: where the feet meet the floor the depth cannot tell them from it. Carried down from + // the lowest part of the skater in each column while the colour is unlike the floor's along that + // row (the area's outer columns again), a little way at most. + std::vector> floor_colour(h); + std::vector channel; + for (std::uint32_t y = 0; y < h; ++y) + for (int c = 0; c < 3; ++c) { + channel.clear(); + for (std::uint32_t x = 0; x < w; x = x + 1 == margin ? w - margin : x + 1) channel.push_back(picture.at(x, y)[c]); + std::nth_element(channel.begin(), channel.begin() + static_cast(channel.size() / 2), channel.end()); + floor_colour[y][static_cast(c)] = channel[channel.size() / 2]; + } + const auto reach = std::max(2, h / 25); + for (std::uint32_t x = margin; x + margin < w; ++x) { + std::uint32_t lowest = h; + for (std::uint32_t y = h; y-- > 0;) + if (picture.at(x, y)[3]) { lowest = y; break; } + if (lowest >= h) continue; + for (std::uint32_t y = lowest + 1; y < h && y <= lowest + reach; ++y) { + auto* p = picture.at(x, y); + int off = 0; + for (int c = 0; c < 3; ++c) off = std::max(off, std::abs(p[c] - floor_colour[y][static_cast(c)])); + if (off <= 24) break; + p[3] = 255; + } + } + return found > 64; +} + +// Keeps only what turned, and says where it is. What moves over the turn (opaque and unlike the +// first frame) is the subject; what is solid in any frame, changes at all, and is joined to that is +// the rest of it (the nose and tail sit on the axis the board turns about and change little); +// everything else in the area is the game's UI, and is cleared from every frame. A rim two pixels wide is kept round it +// for the soft edges. +// The pixels the game drew the same in every frame, as captured (before the backdrop is keyed out, +// which can shade a still pixel differently from frame to frame): the game's UI (a prompt, its +// leader line) stays exactly as it is, while every part of what turns, its nose and tail too, +// takes the light differently as it goes round. +std::vector unchanging(const std::vector& frames) { + const auto& first = frames.front(); + std::vector low(first.rgba), high(first.rgba), out(first.rgba.size() / 4); + for (const auto& frame : frames) + for (std::size_t i = 0; i < frame.rgba.size(); ++i) + low[i] = std::min(low[i], frame.rgba[i]), high[i] = std::max(high[i], frame.rgba[i]); + for (std::size_t p = 0; p < out.size(); ++p) { + int spread = 0; + for (std::size_t c = 0; c < 3; ++c) spread = std::max(spread, high[p * 4 + c] - low[p * 4 + c]); + out[p] = spread <= 3; + } + return out; +} + +Rect keep_moving(std::vector& frames, const std::vector& fixed) { + const auto& first = frames.front(); + const auto w = first.width, h = first.height; + std::vector moved(std::size_t{w} * h), solid(std::size_t{w} * h); + for (const auto& frame : frames) + for (std::size_t p = 0, i = 0; p < moved.size(); ++p, i += 4) { + const auto alpha = frame.rgba[i + 3]; + if (alpha >= 64 && !fixed[p]) solid[p] = 1; + if (moved[p] || fixed[p] || alpha < 16) continue; + int change = 0; + for (std::size_t c = 0; c < 4; ++c) change = std::max(change, std::abs(frame.rgba[i + c] - first.rgba[i + c])); + if (change > 24) moved[p] = 1; + } + // Flood from what moved over what is solid, one piece at a time: each piece's size (in pixels + // that moved) and bounds. + std::vector piece(moved.size()); // 0: none, else the piece's number + 1 + struct Piece { std::size_t moved{}; std::uint32_t x0{}, y0{}, x1{}, y1{}; }; + std::vector pieces; + std::vector todo; + for (std::uint32_t start = 0; start < moved.size(); ++start) { + if (!moved[start] || piece[start]) continue; + pieces.push_back({0, w, h, 0, 0}); + const auto label = static_cast(pieces.size()); + auto& info = pieces.back(); + piece[start] = label; + todo.push_back(start); + while (!todo.empty()) { + const auto p = todo.back(); + todo.pop_back(); + const auto x = static_cast(p % w), y = static_cast(p / w); + info.moved += moved[p]; + info.x0 = std::min(info.x0, static_cast(x)), info.x1 = std::max(info.x1, static_cast(x + 1)); + info.y0 = std::min(info.y0, static_cast(y)), info.y1 = std::max(info.y1, static_cast(y + 1)); + for (std::int64_t dy = -1; dy <= 1; ++dy) + for (std::int64_t dx = -1; dx <= 1; ++dx) { + const auto nx = x + dx, ny = y + dy; + if (nx < 0 || ny < 0 || nx >= w || ny >= h) continue; + const auto q = static_cast(ny * w + nx); + if (!piece[q] && (solid[q] || moved[q])) piece[q] = label, todo.push_back(q); + } + } + } + // What turns is the biggest piece, with any piece centred within its bounds (a board held + // apart from the hand); pieces elsewhere are the scene's own movement (a light, its haze). + std::vector kept(moved.size()); + if (!pieces.empty()) { + const auto biggest = static_cast(std::max_element(pieces.begin(), pieces.end(), [](const Piece& l, const Piece& r) { + return l.moved < r.moved; + }) - pieces.begin()); + const auto& big = pieces[biggest]; + std::vector keep(pieces.size()); + for (std::size_t i = 0; i < pieces.size(); ++i) { + const auto cx = (pieces[i].x0 + pieces[i].x1) / 2, cy = (pieces[i].y0 + pieces[i].y1) / 2; + keep[i] = i == biggest || (cx >= big.x0 && cx < big.x1 && cy >= big.y0 && cy < big.y1); + } + for (std::size_t p = 0; p < kept.size(); ++p) kept[p] = piece[p] && keep[piece[p] - 1]; + } + // The rim (of what changes: never a still prompt touching the edge). + for (int pass = 0; pass < 2; ++pass) { + auto grown = kept; + const auto add = [&](std::size_t q) { if (!fixed[q]) grown[q] = 1; }; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + if (!kept[std::size_t{y} * w + x]) continue; + if (x > 0) add(std::size_t{y} * w + x - 1); + if (x + 1 < w) add(std::size_t{y} * w + x + 1); + if (y > 0) add(std::size_t{y - 1} * w + x); + if (y + 1 < h) add(std::size_t{y + 1} * w + x); + } + kept = std::move(grown); + } + std::uint32_t x0 = w, y0 = h, x1 = 0, y1 = 0; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) + if (kept[std::size_t{y} * w + x] && solid[std::size_t{y} * w + x]) + x0 = std::min(x0, x), y0 = std::min(y0, y), x1 = std::max(x1, x + 1), y1 = std::max(y1, y + 1); + for (auto& frame : frames) + for (std::size_t p = 0; p < kept.size(); ++p) + if (!kept[p]) std::fill_n(frame.rgba.data() + p * 4, 4, std::uint8_t{0}); + if (x1 <= x0 || y1 <= y0) return {}; + return {x0, y0, x1 - x0, y1 - y0}; +} +// A 24x48 grey thumbnail of a frame's area, for telling how far round the board is and whether it +// has stopped. +std::vector thumbnail(const PixelView& frame, Rect area) { + constexpr std::uint32_t tw = 24, th = 48; + std::vector out(tw * th); + for (std::uint32_t ty = 0; ty < th; ++ty) + for (std::uint32_t tx = 0; tx < tw; ++tx) { + const auto x0 = tx * area.width / tw, x1 = std::max(x0 + 1, (tx + 1) * area.width / tw); + const auto y0 = ty * area.height / th, y1 = std::max(y0 + 1, (ty + 1) * area.height / th); + float sum = 0; + for (auto y = y0; y < y1; y += 2) + for (auto x = x0; x < x1; x += 2) { + const auto rgb = frame.rgb(area.x + x, area.y + y); + sum += 0.299f * rgb[0] + 0.587f * rgb[1] + 0.114f * rgb[2]; + } + out[ty * tw + tx] = sum / static_cast(((x1 - x0 + 1) / 2) * ((y1 - y0 + 1) / 2)); + } + return out; +} +float unlike(const std::vector& a, const std::vector& b) noexcept { + float sum = 0; + for (std::size_t i = 0; i < a.size() && i < b.size(); ++i) sum += std::abs(a[i] - b[i]); + return sum / static_cast(std::max(a.size(), 1)); +} +Rect area_on(const Area& a, std::uint32_t width, std::uint32_t height) noexcept { + const double frame_width = a.menu ? std::min(width, height * 21.0 / 9.0) : width; + const double frame_left = (width - frame_width) / 2; + const auto clamp01 = [](float v) { return std::clamp(static_cast(v), 0.0, 1.0); }; + const auto x0 = static_cast(frame_left + frame_width * clamp01(a.left)); + const auto x1 = static_cast(frame_left + frame_width * clamp01(a.right)); + const auto y0 = static_cast(height * clamp01(a.top)), y1 = static_cast(height * clamp01(a.bottom)); + if (x1 <= x0 || y1 <= y0) return {}; + return {x0, y0, std::min(x1, width) - x0, std::min(y1, height) - y0}; +} + +const char* extension(Format format) noexcept { + switch (format) { + case Format::apng: + case Format::png: return "png"; + case Format::webm: return "webm"; + default: return "gif"; + } +} + +bool Recorder::start(const RecordRequest& request, std::string& why) { + if (saving_.load()) { + why = "a capture is being saved"; + return false; + } + std::lock_guard lock(mutex_); + request_ = request; + request_.most = std::clamp(request.most, 1, 20000); + area_ = {}; + skip_ = request.skip; + stride_ = 1; + started_ = std::chrono::steady_clock::now(); + pictures_.clear(), kept_at_.clear(), kept_depth_.clear(), thumbnails_.clear(); + status_ = {State::recording, 0, 0, false, {}, {}}; + return true; +} + +void Recorder::stop() noexcept { + std::lock_guard lock(mutex_); + if (status_.state == State::recording) status_.state = State::stopped; +} + +void Recorder::discard() noexcept { + if (saving_.load()) return; + std::lock_guard lock(mutex_); + pictures_.clear(), pictures_.shrink_to_fit(); + kept_at_.clear(), kept_depth_.clear(), thumbnails_.clear(), thumbnails_.shrink_to_fit(); + status_ = {}; +} + +Status Recorder::status() const { + std::lock_guard lock(mutex_); + return status_; +} + +float Recorder::compare(std::uint32_t a, std::uint32_t b) const { + std::lock_guard lock(mutex_); + if (a >= thumbnails_.size() || b >= thumbnails_.size()) return -1; + return unlike(thumbnails_[a], thumbnails_[b]); +} + +std::vector Recorder::kept() const { + std::lock_guard lock(mutex_); + return kept_at_; +} + +bool Recorder::active() const noexcept { + std::lock_guard lock(mutex_); + return status_.state == State::recording; +} + +void Recorder::fail(std::string message) { + status_.state = State::failed; + status_.message = std::move(message); +} + +bool Recorder::wants_frame() noexcept { + std::lock_guard lock(mutex_); + if (status_.state != State::recording) return false; + // However it goes, a recording lets go within its time. + if (std::chrono::steady_clock::now() - started_ > std::chrono::duration(request_.seconds)) { + status_.state = State::stopped; + status_.message = "the recording reached its time"; + return false; + } + if (skip_) { + --skip_; + return false; + } + return true; +} + +void Recorder::take_frame(const PixelView& frame) noexcept { + try { + Rect area; + RecordRequest request; + { + std::lock_guard lock(mutex_); + if (status_.state != State::recording) return; + if (!frame.data || !frame.width || !frame.height) { + fail("the game's picture could not be read"); + return; + } + if (area_.empty()) { + area_ = area_on(request_.area, frame.width, frame.height); + if (area_.empty()) { + fail("the area is not on the screen"); + return; + } + } + area = area_; + request = request_; + } + // The heavy part outside the lock: the area copied, cut out, and made the size it is kept at. + auto small = thumbnail(frame, area); + const auto copy = copy_area(frame, area); + auto picture = opaque_image({copy.data(), area.width, area.height, area.width * 4, frame.bgra}); + const bool by_depth = request.depth && depth_mask(frame, area, picture); + if (!by_depth) + for (std::size_t i = 3; i < picture.rgba.size(); i += 4) picture.rgba[i] = 255; + if (request.height && request.height < picture.height) + picture = resized(picture, std::max(1, picture.width * request.height / picture.height), request.height); + const auto most = std::clamp(kept_bytes / std::max(picture.rgba.size(), 1), 60, most_kept); + std::lock_guard lock(mutex_); + if (status_.state != State::recording) return; + const auto n = static_cast(thumbnails_.size()); + thumbnails_.push_back(std::move(small)); + if (n % stride_ == 0) { + kept_at_.push_back(n); + kept_depth_.push_back(by_depth); + pictures_.push_back(std::move(picture)); + if (pictures_.size() > most) { // keep every other one, and from now on every other + std::size_t w = 0; + for (std::size_t r = 0; r < pictures_.size(); r += 2) + pictures_[w] = std::move(pictures_[r]), kept_depth_[w] = kept_depth_[r], kept_at_[w++] = kept_at_[r]; + pictures_.resize(w), kept_at_.resize(w), kept_depth_.resize(w); + stride_ *= 2; + } + } + status_.taken = n + 1; + status_.kept = static_cast(pictures_.size()); + status_.depth = !kept_depth_.empty() && std::all_of(kept_depth_.begin(), kept_depth_.end(), [](bool d) { return d; }); + if (status_.taken >= request_.most) status_.state = State::stopped; + } catch (const std::exception& error) { + std::lock_guard lock(mutex_); + fail(error.what()); + } catch (...) { + std::lock_guard lock(mutex_); + fail("the frame could not be kept"); + } +} + +bool Recorder::save(SaveRequest request, std::string& why) { + if (request.frames.empty() || request.frames.size() > most_saved) { + why = "a capture saves 1 to " + std::to_string(most_saved) + " frames"; + return false; + } + if (saving_.exchange(true)) { + why = "a capture is being saved"; + return false; + } + { + std::lock_guard lock(mutex_); + if (pictures_.empty()) { + saving_.store(false); + why = "nothing was recorded"; + return false; + } + status_.state = State::saving; + status_.message.clear(); + status_.file.clear(); + } + std::thread([this, request = std::move(request)]() mutable { encode(std::move(request)); }).detach(); + return true; +} + +void Recorder::encode(SaveRequest o) noexcept { + try { + std::vector frames; + bool by_depth{}; + { + std::lock_guard lock(mutex_); + // Each frame asked for: the nearest one kept. + for (const auto wanted : o.frames) { + std::size_t best = 0; + for (std::size_t k = 1; k < kept_at_.size(); ++k) + if (std::llabs(static_cast(kept_at_[k]) - wanted) < std::llabs(static_cast(kept_at_[best]) - wanted)) best = k; + frames.push_back(pictures_[best]); + } + by_depth = std::all_of(kept_depth_.begin(), kept_depth_.end(), [](bool d) { return d; }); + } + if (o.cutout == Cutout::depth && !by_depth) throw std::runtime_error("not every frame had the game's depth"); + const bool depth_cut = o.cutout == Cutout::depth || (o.cutout == Cutout::automatic && by_depth); + const bool colour_cut = o.cutout == Cutout::colour || (o.cutout == Cutout::automatic && !by_depth); + if (!depth_cut) + for (auto& frame : frames) + for (std::size_t i = 3; i < frame.rgba.size(); i += 4) frame.rgba[i] = 255; + Rect through{0, 0, frames.front().width, frames.front().height}; + if (o.cutout != Cutout::none) { + const auto fixed = depth_cut ? std::vector(frames.front().rgba.size() / 4) : unchanging(frames); + if (colour_cut) key_against_backdrop(frames, o.solid); + // A still cut out by depth: the depth alone says what it is (nothing moves in one frame). + if (depth_cut && frames.size() < 2) through = content_bounds(frames.front()); + else through = keep_moving(frames, fixed); + if (through.empty()) throw std::runtime_error("nothing that moved was found in the area"); + } + const auto margin = std::max(through.width, through.height) / 25 + 2; + const auto size = std::clamp(o.size, 64, 2048); + std::optional card; + if (!o.card.empty() && !o.fonts.heading.empty()) card = render_parts_panel(o.card_title, o.card, size, o.fonts); + Animation animation; + animation.fps = std::clamp(o.fps, 1, 60); + for (auto& frame : frames) { + auto shown = fitted(square_around(frame, through, margin), size); + frame = {}; + if (card) shown = side_by_side(shown, *card); + if (o.background) shown = over_matte(shown, *o.background); + animation.frames.push_back(std::move(shown)); + } + std::vector bytes; + switch (o.format) { + case Format::apng: bytes = encode_apng(animation); break; + case Format::png: bytes = encode_png(animation.frames.front()); break; + case Format::webm: bytes = encode_webm(animation, {.quality = 1, .alpha_quality = 0}); break; // the highest VP8 has + default: bytes = encode_gif(animation, {.fringe = o.background.value_or(Rgb{24, 25, 28}), .dither = true}); break; + } + std::filesystem::create_directories(o.file.parent_path()); + std::ofstream out(o.file, std::ios::binary); + out.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (!out) throw std::runtime_error("the file could not be written"); + std::lock_guard lock(mutex_); + status_.state = State::saved; + status_.file = o.file; + status_.message = depth_cut ? "cut out by depth" : colour_cut ? "cut out by colour" : "not cut out"; + } catch (const std::exception& error) { + std::lock_guard lock(mutex_); + fail(std::string("not saved: ") + error.what()); + } catch (...) { + std::lock_guard lock(mutex_); + fail("not saved"); + } + saving_.store(false); +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_recorder.h b/Extension/Rendering/Capture/capture_recorder.h new file mode 100644 index 00000000..a2bacfdc --- /dev/null +++ b/Extension/Rendering/Capture/capture_recorder.h @@ -0,0 +1,117 @@ +#pragma once +#include "capture_encode.h" +#include "capture_panel.h" +#include +#include +#include +#include +#include +#include +#include + +// A recording of part of the screen as the game draws it, for reskate.capture: every presented frame +// of an area, read back before ReSkate's own UI goes on (the overlay's frame grab), cut out by the +// game's depth where it has one, with a small grey thumbnail of each frame to compare them by; then +// chosen frames saved as a GIF, APNG, PNG or WebM, on a thread of its own. Ported from Fit Check +// (pull request 326's turnaround_session.cpp), whose turn finding is now the plugin's. No Windows or +// game code: the overlay hands frames to take_frame (render thread), the plugin side asks the rest +// (any thread). +namespace dingosdk::capture { +// Where on screen: fractions of the whole screen, or of the game's menu frame (`menu`: the whole +// width up to 21:9, a 21:9 frame in the middle of anything wider), left to right and top to bottom. +struct Area { + float left{}, top{}, right{1}, bottom{1}; + bool menu{}; +}; +Rect area_on(const Area& area, std::uint32_t width, std::uint32_t height) noexcept; + +enum class Format { gif, apng, png, webm }; +const char* extension(Format format) noexcept; + +struct RecordRequest { + Area area; + bool depth{true}; // cut each frame out by the game's depth where it has one + std::uint32_t height{}; // each kept picture this tall at most (0: as on screen) + std::uint32_t skip{}; // presented frames let go first + std::uint32_t most{2400}; // frames taken at most; then it stops by itself + double seconds{60}; // and after this long +}; +// Pictures and bytes held at once, at most: past them every other kept frame is let go, and from +// then on only every other frame taken is kept (and so on). +inline constexpr std::size_t most_kept = 480; +inline constexpr std::size_t kept_bytes = std::size_t{1000} << 20; +inline constexpr std::uint32_t most_saved = 360; // frames in one saved file +inline constexpr std::uint32_t thumbnail_width = 24, thumbnail_height = 48; + +enum class State { idle, recording, stopped, saving, saved, failed }; +struct Status { + State state{State::idle}; + std::uint32_t taken{}, kept{}; + bool depth{}; // every kept frame was cut out by the game's depth + std::string message; // what went wrong, or what was saved + std::filesystem::path file; +}; + +enum class Cutout { automatic, depth, colour, none }; +struct SaveRequest { + std::vector frames; // frames taken (from 0), each saved as the nearest one kept + Format format{Format::gif}; + std::uint32_t fps{30}; + std::uint32_t size{512}; // the square what was recorded is fitted into, in pixels + std::optional background; // flattened over it; none: transparent + Cutout cutout{Cutout::automatic}; // automatic: by depth when every frame had it, else by colour + bool solid{true}; // a hole what turns closes all round is part of it (a board), or only a small one (a skater) + std::string card_title; // a card beside it: this title, then each line (none without lines) + std::vector card; + PanelFonts fonts; // for the card; must outlive the save + std::filesystem::path file; // where it is written (its folder is made) +}; + +class Recorder { +public: + // A new recording, letting go of the last one; false (and why) while one is being saved. + bool start(const RecordRequest& request, std::string& why); + // No more frames; what was taken is kept for compare, kept and save. + void stop() noexcept; + // Lets go of everything taken. + void discard() noexcept; + Status status() const; + // How unlike two frames taken look: the mean difference of their thumbnails' grey levels + // (0..255), or a negative number when either is not known. + float compare(std::uint32_t a, std::uint32_t b) const; + // The frames still held, by the number they were taken as, in order. + std::vector kept() const; + // Saves on a thread of its own (the recording stops first); false (and why) when there is + // nothing to save or a save is going on. + bool save(SaveRequest request, std::string& why); + + // Render thread, every presented frame. + bool active() const noexcept; + bool wants_frame() noexcept; + void take_frame(const PixelView& frame) noexcept; + +private: + void fail(std::string message); + void encode(SaveRequest request) noexcept; + + mutable std::mutex mutex_; + Status status_; + RecordRequest request_; + Rect area_; + std::uint32_t skip_{}, stride_{1}; + std::chrono::steady_clock::time_point started_{}; + std::vector pictures_; + std::vector kept_at_; + std::vector kept_depth_; + std::vector> thumbnails_; // every frame taken + std::atomic saving_{false}; +}; + +// The pieces of saving, for the tests: what turns cut out by depth into `picture`'s alpha (false when +// the depth shows nothing), a frame's grey thumbnail, and only what moved across the frames kept. +bool depth_mask(const PixelView& frame, Rect area, Image& picture); +std::vector thumbnail(const PixelView& frame, Rect area); +float unlike(const std::vector& a, const std::vector& b) noexcept; +std::vector unchanging(const std::vector& frames); +Rect keep_moving(std::vector& frames, const std::vector& fixed); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_runtime.cpp b/Extension/Rendering/Capture/capture_runtime.cpp new file mode 100644 index 00000000..ac4cde8c --- /dev/null +++ b/Extension/Rendering/Capture/capture_runtime.cpp @@ -0,0 +1,34 @@ +#include "capture_runtime.h" +#include "Extension/UI/Overlay/overlay.h" +#include + +namespace dingosdk::capture { +namespace { +std::span built_in(const wchar_t* name) { + HMODULE module{}; + if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&built_in), &module)) + return {}; + const auto resource = FindResourceW(module, name, MAKEINTRESOURCEW(10) /* RT_RCDATA */); + const auto handle = resource ? LoadResource(module, resource) : nullptr; + const auto* data = handle ? static_cast(LockResource(handle)) : nullptr; + return data ? std::span(data, SizeofResource(module, resource)) : std::span{}; +} +bool grab_active() noexcept { return recorder().active(); } +bool grab_wants() noexcept { return recorder().wants_frame(); } +void grab_take(const overlay::GrabbedFrame& frame) noexcept { + recorder().take_frame({frame.data, frame.width, frame.height, frame.pitch, frame.bgra, frame.depth, frame.depth_pitch}); +} +constexpr overlay::FrameGrabHost frame_grab_host{&grab_active, &grab_wants, &grab_take}; +} // namespace + +Recorder& recorder() noexcept { + static auto* value = new Recorder; // process lifetime: its saving thread may outlive statics + return *value; +} +const overlay::FrameGrabHost* frame_grab() noexcept { return &frame_grab_host; } +PanelFonts menu_fonts() noexcept { + static const PanelFonts fonts{built_in(L"FONT_HEADING"), built_in(L"FONT_BODY"), built_in(L"FONT_BRUSH")}; + return fonts; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_runtime.h b/Extension/Rendering/Capture/capture_runtime.h new file mode 100644 index 00000000..81970ab1 --- /dev/null +++ b/Extension/Rendering/Capture/capture_runtime.h @@ -0,0 +1,13 @@ +#pragma once +#include "capture_recorder.h" + +namespace dingosdk::overlay { struct FrameGrabHost; } + +// The process's one screen recorder (reskate.capture), fed by the overlay's frame grab, and the menu's +// fonts its cards are drawn in (built into ReSkate.dll as FONT_HEADING, FONT_BODY and FONT_BRUSH). +namespace dingosdk::capture { +Recorder& recorder() noexcept; +// What the overlay asks before drawing each frame (overlay::set_frame_grab); process lifetime. +const overlay::FrameGrabHost* frame_grab() noexcept; +PanelFonts menu_fonts() noexcept; +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_vp8.cpp b/Extension/Rendering/Capture/capture_vp8.cpp new file mode 100644 index 00000000..dab9db78 --- /dev/null +++ b/Extension/Rendering/Capture/capture_vp8.cpp @@ -0,0 +1,458 @@ +#include "capture_vp8.h" +#include "capture_vp8_tables.h" +#include +#include +#include +#include +#include + +// Prediction and reconstruction follow libvpx's decoder exactly (vp8/decoder/decodeframe.c, +// vp8/common/idctllm.c, reconintra.c): the encoder predicts from what the decoder will have, so +// any difference would build up across the frame. +namespace dingosdk::capture::vp8 { +namespace { +// --- the boolean entropy coder (RFC 6386, section 7.3) +struct BoolEncoder { + std::vector out; + std::uint32_t range = 255, bottom = 0; + int bit_count = 24; + void carry() { + for (auto i = out.size(); i-- > 0;) { + if (out[i] == 255) { out[i] = 0; continue; } + ++out[i]; + break; + } + } + void put(bool bit, int probability) { + const std::uint32_t split = 1 + (((range - 1) * static_cast(probability)) >> 8); + if (bit) bottom += split, range -= split; + else range = split; + while (range < 128) { + range <<= 1; + if (bottom & (1u << 31)) carry(); + bottom <<= 1; + if (!--bit_count) { + out.push_back(static_cast(bottom >> 24)); + bottom &= (1u << 24) - 1; + bit_count = 8; + } + } + } + void literal(std::uint32_t value, int bits) { + for (int b = bits - 1; b >= 0; --b) put((value >> b) & 1, 128); + } + std::vector finish() { + for (int i = 0; i < 32; ++i) put(false, 128); // as libvpx's vp8_stop_encode + return std::move(out); + } +}; + +enum Mode { dc_pred, v_pred, h_pred, tm_pred }; +constexpr std::array zigzag{0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15}; +constexpr std::array bands{0, 1, 2, 3, 6, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 0}; +constexpr std::uint8_t pcat3[] = {173, 148, 140}, pcat4[] = {176, 155, 140, 135}, pcat5[] = {180, 157, 141, 134, 130}, + pcat6[] = {254, 254, 243, 230, 196, 177, 153, 140, 133, 130, 129}; +constexpr std::array kf_ymode_prob{145, 156, 163, 128}; +constexpr std::array kf_uv_mode_prob{142, 114, 183}; +enum BlockType { y_after_y2 = 0, y2 = 1, chroma = 2 }; + +const std::uint8_t* probabilities(int type, int band, int context) { + return default_coef_probs + ((type * 8 + band) * 3 + context) * 11; +} + +// One block's tokens, its quantized coefficients in raster order; the "has coefficients" flag +// the neighbours' contexts take. +bool put_tokens(BoolEncoder& e, const std::array& raster, int type, int first, int context) { + int last = -1; + for (int i = first; i < 16; ++i) + if (raster[static_cast(zigzag[static_cast(i)])]) last = i; + const auto* p = probabilities(type, bands[static_cast(first)], context); + if (last < 0) { + e.put(false, p[0]); + return false; + } + e.put(true, p[0]); + for (int i = first; i <= last; ++i) { + const int v = raster[static_cast(zigzag[static_cast(i)])], a = std::abs(v); + const auto next = static_cast(i + 1); + if (!a) { + e.put(false, p[1]); + p = probabilities(type, bands[next], 0); + continue; + } + e.put(true, p[1]); + int next_context = 2; + if (a == 1) { + e.put(false, p[2]); + next_context = 1; + } else { + e.put(true, p[2]); + if (a <= 4) { + e.put(false, p[3]); + if (a == 2) e.put(false, p[4]); + else e.put(true, p[4]), e.put(a == 4, p[5]); + } else { + e.put(true, p[3]); + if (a <= 10) { + e.put(false, p[6]); + if (a <= 6) e.put(false, p[7]), e.put(a == 6, 159); + else { + e.put(true, p[7]); + const int extra = a - 7; + e.put((extra >> 1) & 1, 165); + e.put(extra & 1, 145); + } + } else { + e.put(true, p[6]); + const int category = a <= 18 ? 0 : a <= 34 ? 1 : a <= 66 ? 2 : 3; + static constexpr int base[] = {11, 19, 35, 67}, bits[] = {3, 4, 5, 11}; + static constexpr const std::uint8_t* table[] = {pcat3, pcat4, pcat5, pcat6}; + e.put(category >> 1, p[8]); + e.put(category & 1, p[9 + (category >> 1)]); + const int extra = a - base[category]; + for (int b = bits[category] - 1, k = 0; b >= 0; --b, ++k) e.put((extra >> b) & 1, table[category][k]); + } + } + } + e.put(v < 0, 128); + if (next == 16) break; + p = probabilities(type, bands[next], next_context); + e.put(i != last, p[0]); // more to come, or the end of the block + } + return true; +} + +// --- transforms (libvpx: vp8/encoder/dct.c, vp8/common/idctllm.c) +void forward_dct(const short* in, short* out) { // in: 4x4 raster + short temp[16]; + for (int i = 0; i < 4; ++i) { + const short* ip = in + i * 4; + short* op = temp + i * 4; + const int a1 = (ip[0] + ip[3]) * 8, b1 = (ip[1] + ip[2]) * 8, c1 = (ip[1] - ip[2]) * 8, d1 = (ip[0] - ip[3]) * 8; + op[0] = static_cast(a1 + b1); + op[2] = static_cast(a1 - b1); + op[1] = static_cast((c1 * 2217 + d1 * 5352 + 14500) >> 12); + op[3] = static_cast((d1 * 2217 - c1 * 5352 + 7500) >> 12); + } + for (int i = 0; i < 4; ++i) { + const short* ip = temp + i; + short* op = out + i; + const int a1 = ip[0] + ip[12], b1 = ip[4] + ip[8], c1 = ip[4] - ip[8], d1 = ip[0] - ip[12]; + op[0] = static_cast((a1 + b1 + 7) >> 4); + op[8] = static_cast((a1 - b1 + 7) >> 4); + op[4] = static_cast(((c1 * 2217 + d1 * 5352 + 12000) >> 16) + (d1 != 0)); + op[12] = static_cast((d1 * 2217 - c1 * 5352 + 51000) >> 16); + } +} +void forward_walsh(const short* in, short* out) { + short temp[16]; + for (int i = 0; i < 4; ++i) { + const short* ip = in + i * 4; + short* op = temp + i * 4; + const int a1 = (ip[0] + ip[2]) * 4, d1 = (ip[1] + ip[3]) * 4, c1 = (ip[1] - ip[3]) * 4, b1 = (ip[0] - ip[2]) * 4; + op[0] = static_cast(a1 + d1 + (a1 != 0)); + op[1] = static_cast(b1 + c1); + op[2] = static_cast(b1 - c1); + op[3] = static_cast(a1 - d1); + } + for (int i = 0; i < 4; ++i) { + const short* ip = temp + i; + short* op = out + i; + const int a1 = ip[0] + ip[8], d1 = ip[4] + ip[12], c1 = ip[4] - ip[12], b1 = ip[0] - ip[8]; + int a2 = a1 + d1, b2 = b1 + c1, c2 = b1 - c1, d2 = a1 - d1; + a2 += a2 < 0, b2 += b2 < 0, c2 += c2 < 0, d2 += d2 < 0; + op[0] = static_cast((a2 + 3) >> 3); + op[4] = static_cast((b2 + 3) >> 3); + op[8] = static_cast((c2 + 3) >> 3); + op[12] = static_cast((d2 + 3) >> 3); + } +} +void inverse_walsh(const int* in, int* out) { + int temp[16]; + for (int i = 0; i < 4; ++i) { + const int* ip = in + i; + int* op = temp + i; + const int a1 = ip[0] + ip[12], b1 = ip[4] + ip[8], c1 = ip[4] - ip[8], d1 = ip[0] - ip[12]; + op[0] = static_cast(a1 + b1); + op[4] = static_cast(c1 + d1); + op[8] = static_cast(a1 - b1); + op[12] = static_cast(d1 - c1); + } + for (int i = 0; i < 4; ++i) { + const int* ip = temp + i * 4; + const int a1 = ip[0] + ip[3], b1 = ip[1] + ip[2], c1 = ip[1] - ip[2], d1 = ip[0] - ip[3]; + out[i * 4 + 0] = static_cast((a1 + b1 + 3) >> 3); + out[i * 4 + 1] = static_cast((c1 + d1 + 3) >> 3); + out[i * 4 + 2] = static_cast((a1 - b1 + 3) >> 3); + out[i * 4 + 3] = static_cast((d1 - c1 + 3) >> 3); + } +} +// Adds the inverse DCT of dequantized `in` (raster) to the prediction in place. +void inverse_dct_add(const int* in, std::uint8_t* dst, std::size_t stride) { + constexpr int cospi8sqrt2minus1 = 20091, sinpi8sqrt2 = 35468; + short temp[16]; + for (int i = 0; i < 4; ++i) { + const short ip0 = static_cast(in[i]), ip4 = static_cast(in[4 + i]), ip8 = static_cast(in[8 + i]), + ip12 = static_cast(in[12 + i]); + const int a1 = ip0 + ip8, b1 = ip0 - ip8; + int t1 = (ip4 * sinpi8sqrt2) >> 16, t2 = ip12 + ((ip12 * cospi8sqrt2minus1) >> 16); + const int c1 = t1 - t2; + t1 = ip4 + ((ip4 * cospi8sqrt2minus1) >> 16); + t2 = (ip12 * sinpi8sqrt2) >> 16; + const int d1 = t1 + t2; + temp[i] = static_cast(a1 + d1); + temp[12 + i] = static_cast(a1 - d1); + temp[4 + i] = static_cast(b1 + c1); + temp[8 + i] = static_cast(b1 - c1); + } + for (int r = 0; r < 4; ++r) { + const short* ip = temp + r * 4; + const int a1 = ip[0] + ip[2], b1 = ip[0] - ip[2]; + int t1 = (ip[1] * sinpi8sqrt2) >> 16, t2 = ip[3] + ((ip[3] * cospi8sqrt2minus1) >> 16); + const int c1 = t1 - t2; + t1 = ip[1] + ((ip[1] * cospi8sqrt2minus1) >> 16); + t2 = (ip[3] * sinpi8sqrt2) >> 16; + const int d1 = t1 + t2; + const int row[4] = {static_cast((a1 + d1 + 4) >> 3), static_cast((b1 + c1 + 4) >> 3), + static_cast((b1 - c1 + 4) >> 3), static_cast((a1 - d1 + 4) >> 3)}; + for (int c = 0; c < 4; ++c) { + auto& d = dst[static_cast(r) * stride + static_cast(c)]; + d = static_cast(std::clamp(row[c] + d, 0, 255)); + } + } +} +void dc_only_add(int dc, std::uint8_t* dst, std::size_t stride) { + const int a1 = (dc + 4) >> 3; + for (std::size_t r = 0; r < 4; ++r) + for (std::size_t c = 0; c < 4; ++c) dst[r * stride + c] = static_cast(std::clamp(a1 + dst[r * stride + c], 0, 255)); +} + +int quantize(int coefficient, int step, bool dc) { + // Rounding with a slight dead zone for AC coefficients: fewer stray small ones to code. + const int a = std::abs(coefficient), bias = dc ? step / 2 : step * 3 / 8; + const int q = std::min((a + bias) / step, 2048); + return coefficient < 0 ? -q : q; +} + +// A padded plane and its reconstruction (what the decoder will see), with VP8's frame borders: +// 127 above the frame (top-left corner included), 129 left of it. +struct Plane { + std::size_t width{}, height{}; + std::vector source, recon; + Plane(const std::vector& from, std::size_t w, std::size_t h, std::size_t padded_w, std::size_t padded_h) + : width(padded_w), height(padded_h), source(padded_w * padded_h), recon(padded_w * padded_h) { + for (std::size_t y = 0; y < padded_h; ++y) + for (std::size_t x = 0; x < padded_w; ++x) source[y * padded_w + x] = from[std::min(y, h - 1) * w + std::min(x, w - 1)]; + } + int above(std::size_t x, std::size_t y) const { return y ? recon[(y - 1) * width + x] : 127; } + int left(std::size_t x, std::size_t y) const { return x ? recon[y * width + x - 1] : 129; } + int corner(std::size_t x, std::size_t y) const { return !y ? 127 : !x ? 129 : recon[(y - 1) * width + x - 1]; } + // The `size` x `size` prediction at (x, y). + void predict(std::size_t x0, std::size_t y0, std::size_t size, Mode mode, std::uint8_t* out) const { + for (std::size_t r = 0; r < size; ++r) + for (std::size_t c = 0; c < size; ++c) { + int v = 128; + switch (mode) { + case v_pred: v = above(x0 + c, y0); break; + case h_pred: v = left(x0, y0 + r); break; + case tm_pred: v = std::clamp(left(x0, y0 + r) + above(x0 + c, y0) - corner(x0, y0), 0, 255); break; + case dc_pred: { + int sum = 0, count = 0; + if (y0) for (std::size_t i = 0; i < size; ++i) sum += above(x0 + i, y0), ++count; + if (x0) for (std::size_t i = 0; i < size; ++i) sum += left(x0, y0 + i), ++count; + v = count ? (sum + count / 2) / count : 128; + break; + } + } + out[r * size + c] = static_cast(v); + } + } + // The mode whose prediction is closest to the source. + Mode best_mode(std::size_t x0, std::size_t y0, std::size_t size) const { + std::vector guess(size * size); + Mode best = dc_pred; + std::uint64_t best_error = std::numeric_limits::max(); + for (const Mode mode : {dc_pred, v_pred, h_pred, tm_pred}) { + predict(x0, y0, size, mode, guess.data()); + std::uint64_t error = 0; + for (std::size_t r = 0; r < size; ++r) + for (std::size_t c = 0; c < size; ++c) { + const int d = source[(y0 + r) * width + x0 + c] - guess[r * size + c]; + error += static_cast(d * d); + } + if (error < best_error) best_error = error, best = mode; + } + return best; + } + // Writes the prediction into the reconstruction and returns the 4x4 residual blocks (raster + // order of blocks, each raster order) of the source against it. + std::vector> residual(std::size_t x0, std::size_t y0, std::size_t size, Mode mode) { + std::vector guess(size * size); + predict(x0, y0, size, mode, guess.data()); + const std::size_t per_row = size / 4; + std::vector> blocks(per_row * per_row); + for (std::size_t r = 0; r < size; ++r) + for (std::size_t c = 0; c < size; ++c) { + recon[(y0 + r) * width + x0 + c] = guess[r * size + c]; + blocks[(r / 4) * per_row + c / 4][(r % 4) * 4 + c % 4] = + static_cast(source[(y0 + r) * width + x0 + c] - guess[r * size + c]); + } + return blocks; + } + std::uint8_t* at(std::size_t x, std::size_t y) { return recon.data() + y * width + x; } +}; + +void put_y_mode(BoolEncoder& e, Mode mode) { + e.put(true, kf_ymode_prob[0]); // not B_PRED + const bool second = mode == h_pred || mode == tm_pred; + e.put(second, kf_ymode_prob[1]); + if (second) e.put(mode == tm_pred, kf_ymode_prob[3]); + else e.put(mode == v_pred, kf_ymode_prob[2]); +} +void put_uv_mode(BoolEncoder& e, Mode mode) { + e.put(mode != dc_pred, kf_uv_mode_prob[0]); + if (mode == dc_pred) return; + e.put(mode != v_pred, kf_uv_mode_prob[1]); + if (mode != v_pred) e.put(mode == tm_pred, kf_uv_mode_prob[2]); +} + +// Token contexts: whether the block above / to the left had coefficients. Per macroblock column +// (above) or for the current row (left): Y 0..3, U 4..5, V 6..7, Y2 8. +using Contexts = std::array; +} // namespace + +std::vector encode_keyframe(const Planes& planes, int quality) { + const auto w = planes.width, h = planes.height; + if (!w || !h || w > 16383 || h > 16383) throw std::invalid_argument("VP8 frame size out of range."); + const std::size_t cw = (w + 1) / 2, ch = (h + 1) / 2; + if (planes.y.size() < std::size_t{w} * h || planes.u.size() < cw * ch || planes.v.size() < cw * ch) + throw std::invalid_argument("VP8 planes too small."); + const std::size_t mb_cols = (w + 15) / 16, mb_rows = (h + 15) / 16; + Plane Y(planes.y, w, h, mb_cols * 16, mb_rows * 16), U(planes.u, cw, ch, mb_cols * 8, mb_rows * 8), + V(planes.v, cw, ch, mb_cols * 8, mb_rows * 8); + const int qi = std::clamp(quality, 0, 127); + const int y1_dc = dc_qlookup[qi], y1_ac = ac_qlookup[qi], y2_dc = dc_qlookup[qi] * 2, + y2_ac = std::max((ac_qlookup[qi] * 101581) >> 16, 8), uv_dc = std::min(dc_qlookup[qi], 132), + uv_ac = ac_qlookup[qi]; + + BoolEncoder header, tokens; + header.put(false, 128); // colour space + header.put(false, 128); // clamping required + header.put(false, 128); // no segmentation + header.put(false, 128); // normal loop filter... + header.literal(0, 6); // ...at level 0: off + header.literal(0, 3); // sharpness + header.put(false, 128); // no loop filter deltas + header.literal(0, 2); // one token partition + header.literal(static_cast(qi), 7); + for (int i = 0; i < 5; ++i) header.put(false, 128); // no quantizer deltas + header.put(false, 128); // refresh_entropy_probs: keep the defaults + for (const auto probability : coef_update_probs) header.put(false, probability); // no probability updates + header.put(false, 128); // mb_no_coeff_skip off: every macroblock has tokens + + std::vector above(mb_cols); + for (std::size_t my = 0; my < mb_rows; ++my) { + Contexts left{}; + for (std::size_t mx = 0; mx < mb_cols; ++mx) { + auto& top = above[mx]; + const std::size_t x = mx * 16, y = my * 16, cx = mx * 8, cy = my * 8; + const Mode y_mode = Y.best_mode(x, y, 16); + // Chroma: one mode for U and V, chosen on U and V together. + Mode uv_mode = dc_pred; + { + std::uint64_t best = std::numeric_limits::max(); + std::array gu{}, gv{}; + for (const Mode mode : {dc_pred, v_pred, h_pred, tm_pred}) { + U.predict(cx, cy, 8, mode, gu.data()); + V.predict(cx, cy, 8, mode, gv.data()); + std::uint64_t error = 0; + for (std::size_t r = 0; r < 8; ++r) + for (std::size_t c = 0; c < 8; ++c) { + const int du = U.source[(cy + r) * U.width + cx + c] - gu[r * 8 + c]; + const int dv = V.source[(cy + r) * V.width + cx + c] - gv[r * 8 + c]; + error += static_cast(du * du + dv * dv); + } + if (error < best) best = error, uv_mode = mode; + } + } + put_y_mode(header, y_mode); + put_uv_mode(header, uv_mode); + + // Luma: 16 blocks, their DCs gathered into the second-order (Y2) block. + const auto y_res = Y.residual(x, y, 16, y_mode); + std::array, 16> y_coeff{}; + std::array dcs{}, y2_coeff{}; + for (std::size_t b = 0; b < 16; ++b) { + forward_dct(y_res[b].data(), y_coeff[b].data()); + dcs[b] = y_coeff[b][0]; + } + forward_walsh(dcs.data(), y2_coeff.data()); + std::array y2_q{}; + for (std::size_t i = 0; i < 16; ++i) y2_q[i] = quantize(y2_coeff[i], i ? y2_ac : y2_dc, true); + left[8] = top[8] = put_tokens(tokens, y2_q, y2, 0, top[8] + left[8]); + // The DCs the decoder will have, from the quantized Y2. + std::array y2_dq{}, block_dc{}; + bool y2_has_ac = false; + for (std::size_t i = 1; i < 16; ++i) y2_has_ac |= y2_q[static_cast(zigzag[i])] != 0; + for (std::size_t i = 0; i < 16; ++i) y2_dq[i] = y2_q[i] * (i ? y2_ac : y2_dc); + if (y2_has_ac) inverse_walsh(y2_dq.data(), block_dc.data()); + else block_dc.fill(static_cast((y2_dq[0] + 3) >> 3)); + for (std::size_t b = 0; b < 16; ++b) { + std::array q{}; + bool has_ac = false; + for (std::size_t i = 1; i < 16; ++i) has_ac |= (q[i] = quantize(y_coeff[b][i], y1_ac, false)) != 0; + const std::size_t col = b % 4, row = b / 4; + left[row] = top[col] = put_tokens(tokens, q, y_after_y2, 1, top[col] + left[row]); + auto* dst = Y.at(x + col * 4, y + row * 4); + if (has_ac) { + std::array dq{}; + dq[0] = block_dc[b]; + for (std::size_t i = 1; i < 16; ++i) dq[i] = q[i] * y1_ac; + inverse_dct_add(dq.data(), dst, Y.width); + } else { + dc_only_add(block_dc[b], dst, Y.width); + } + } + (void)y1_dc; // luma DCs go through Y2; the first-order DC step is unused without B_PRED + + // Chroma: four 4x4 blocks each for U and V. + for (int plane = 0; plane < 2; ++plane) { + auto& P = plane ? V : U; + const auto res = P.residual(cx, cy, 8, uv_mode); + for (std::size_t b = 0; b < 4; ++b) { + std::array coeff{}; + forward_dct(res[b].data(), coeff.data()); + std::array q{}; + bool beyond_first = false; + for (std::size_t i = 0; i < 16; ++i) q[i] = quantize(coeff[i], i ? uv_ac : uv_dc, i == 0); + for (std::size_t i = 1; i < 16; ++i) beyond_first |= q[static_cast(zigzag[i])] != 0; + const std::size_t col = b % 2, row = b / 2, slot = 4 + static_cast(plane) * 2; + left[slot + row] = top[slot + col] = put_tokens(tokens, q, chroma, 0, top[slot + col] + left[slot + row]); + auto* dst = P.at(cx + col * 4, cy + row * 4); + if (beyond_first) { + std::array dq{}; + for (std::size_t i = 0; i < 16; ++i) dq[i] = q[i] * (i ? uv_ac : uv_dc); + inverse_dct_add(dq.data(), dst, P.width); + } else { + dc_only_add(q[0] * uv_dc, dst, P.width); + } + } + } + } + } + + const auto first = header.finish(); + const auto second = tokens.finish(); + if (first.size() >= (1u << 19)) throw std::runtime_error("VP8 header partition too large."); + std::vector out; + out.reserve(10 + first.size() + second.size()); + const std::uint32_t tag = 0 /* key frame */ | (0 << 1) /* version */ | (1 << 4) /* shown */ | + (static_cast(first.size()) << 5); + out.insert(out.end(), {static_cast(tag), static_cast(tag >> 8), static_cast(tag >> 16), + 0x9d, 0x01, 0x2a, static_cast(w), static_cast(w >> 8), + static_cast(h), static_cast(h >> 8)}); + out.insert(out.end(), first.begin(), first.end()); + out.insert(out.end(), second.begin(), second.end()); + return out; +} +} // namespace dingosdk::capture::vp8 diff --git a/Extension/Rendering/Capture/capture_vp8.h b/Extension/Rendering/Capture/capture_vp8.h new file mode 100644 index 00000000..9cd1c5f7 --- /dev/null +++ b/Extension/Rendering/Capture/capture_vp8.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +// A small VP8 encoder (RFC 6386) for the captures' WebM files: every frame a key frame, 16x16 +// intra prediction only, one token partition, no loop filter. Simple rather than small, which +// suits a few seconds of a turning board. +namespace dingosdk::capture::vp8 { +// 8-bit 4:2:0: `y` is width x height, `u` and `v` are half that, rounded up. +struct Planes { + std::uint32_t width{}, height{}; + std::vector y, u, v; +}; +// One key frame. `quality` is VP8's quantizer index, 0 (best) .. 127. +std::vector encode_keyframe(const Planes& planes, int quality); +} // namespace dingosdk::capture::vp8 diff --git a/Extension/Rendering/Capture/capture_vp8_tables.h b/Extension/Rendering/Capture/capture_vp8_tables.h new file mode 100644 index 00000000..92d9e6c0 --- /dev/null +++ b/Extension/Rendering/Capture/capture_vp8_tables.h @@ -0,0 +1,226 @@ +#pragma once +#include + +// VP8 constants (RFC 6386) for the captures' WebM writer, taken from libvpx: +// vp8/common/coefupdateprobs.h, vp8/common/default_coef_probs.h, vp8/common/quant_common.c. +// Copyright (c) 2010 The WebM project authors. All Rights Reserved. BSD-style license: +// https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/LICENSE +namespace dingosdk::capture::vp8 { +// [block type 4][band 8][context 3][node 11], flattened. +inline constexpr std::uint8_t coef_update_probs[1056] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 176, 246, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 223, 241, 252, 255, 255, 255, 255, 255, 255, 255, 255, + 249, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 244, 252, 255, 255, 255, 255, 255, 255, 255, 255, + 234, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 246, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 239, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 248, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 251, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 251, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 253, 255, 254, 255, 255, 255, 255, 255, 255, + 250, 255, 254, 255, 254, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 225, 252, 241, 253, 255, 255, 254, 255, 255, 255, 255, + 234, 250, 241, 250, 253, 255, 253, 254, 255, 255, 255, + 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 223, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 238, 253, 254, 254, 255, 255, 255, 255, 255, 255, 255, + 255, 248, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 249, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 247, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 252, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 250, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 186, 251, 250, 255, 255, 255, 255, 255, 255, 255, 255, + 234, 251, 244, 254, 255, 255, 255, 255, 255, 255, 255, + 251, 251, 243, 253, 254, 255, 254, 255, 255, 255, 255, + 255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 236, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 251, 253, 253, 254, 254, 255, 255, 255, 255, 255, 255, + 255, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 248, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 250, 254, 252, 254, 255, 255, 255, 255, 255, 255, 255, + 248, 254, 249, 253, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 246, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 252, 254, 251, 254, 254, 255, 255, 255, 255, 255, 255, + 255, 254, 252, 255, 255, 255, 255, 255, 255, 255, 255, + 248, 254, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 255, 254, 254, 255, 255, 255, 255, 255, 255, 255, + 255, 251, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 245, 251, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 251, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 252, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 252, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 249, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 250, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +}; +inline constexpr std::uint8_t default_coef_probs[1056] = { + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 253, 136, 254, 255, 228, 219, 128, 128, 128, 128, 128, + 189, 129, 242, 255, 227, 213, 255, 219, 128, 128, 128, + 106, 126, 227, 252, 214, 209, 255, 255, 128, 128, 128, + 1, 98, 248, 255, 236, 226, 255, 255, 128, 128, 128, + 181, 133, 238, 254, 221, 234, 255, 154, 128, 128, 128, + 78, 134, 202, 247, 198, 180, 255, 219, 128, 128, 128, + 1, 185, 249, 255, 243, 255, 128, 128, 128, 128, 128, + 184, 150, 247, 255, 236, 224, 128, 128, 128, 128, 128, + 77, 110, 216, 255, 236, 230, 128, 128, 128, 128, 128, + 1, 101, 251, 255, 241, 255, 128, 128, 128, 128, 128, + 170, 139, 241, 252, 236, 209, 255, 255, 128, 128, 128, + 37, 116, 196, 243, 228, 255, 255, 255, 128, 128, 128, + 1, 204, 254, 255, 245, 255, 128, 128, 128, 128, 128, + 207, 160, 250, 255, 238, 128, 128, 128, 128, 128, 128, + 102, 103, 231, 255, 211, 171, 128, 128, 128, 128, 128, + 1, 152, 252, 255, 240, 255, 128, 128, 128, 128, 128, + 177, 135, 243, 255, 234, 225, 128, 128, 128, 128, 128, + 80, 129, 211, 255, 194, 224, 128, 128, 128, 128, 128, + 1, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 246, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 255, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 198, 35, 237, 223, 193, 187, 162, 160, 145, 155, 62, + 131, 45, 198, 221, 172, 176, 220, 157, 252, 221, 1, + 68, 47, 146, 208, 149, 167, 221, 162, 255, 223, 128, + 1, 149, 241, 255, 221, 224, 255, 255, 128, 128, 128, + 184, 141, 234, 253, 222, 220, 255, 199, 128, 128, 128, + 81, 99, 181, 242, 176, 190, 249, 202, 255, 255, 128, + 1, 129, 232, 253, 214, 197, 242, 196, 255, 255, 128, + 99, 121, 210, 250, 201, 198, 255, 202, 128, 128, 128, + 23, 91, 163, 242, 170, 187, 247, 210, 255, 255, 128, + 1, 200, 246, 255, 234, 255, 128, 128, 128, 128, 128, + 109, 178, 241, 255, 231, 245, 255, 255, 128, 128, 128, + 44, 130, 201, 253, 205, 192, 255, 255, 128, 128, 128, + 1, 132, 239, 251, 219, 209, 255, 165, 128, 128, 128, + 94, 136, 225, 251, 218, 190, 255, 255, 128, 128, 128, + 22, 100, 174, 245, 186, 161, 255, 199, 128, 128, 128, + 1, 182, 249, 255, 232, 235, 128, 128, 128, 128, 128, + 124, 143, 241, 255, 227, 234, 128, 128, 128, 128, 128, + 35, 77, 181, 251, 193, 211, 255, 205, 128, 128, 128, + 1, 157, 247, 255, 236, 231, 255, 255, 128, 128, 128, + 121, 141, 235, 255, 225, 227, 255, 255, 128, 128, 128, + 45, 99, 188, 251, 195, 217, 255, 224, 128, 128, 128, + 1, 1, 251, 255, 213, 255, 128, 128, 128, 128, 128, + 203, 1, 248, 255, 255, 128, 128, 128, 128, 128, 128, + 137, 1, 177, 255, 224, 255, 128, 128, 128, 128, 128, + 253, 9, 248, 251, 207, 208, 255, 192, 128, 128, 128, + 175, 13, 224, 243, 193, 185, 249, 198, 255, 255, 128, + 73, 17, 171, 221, 161, 179, 236, 167, 255, 234, 128, + 1, 95, 247, 253, 212, 183, 255, 255, 128, 128, 128, + 239, 90, 244, 250, 211, 209, 255, 255, 128, 128, 128, + 155, 77, 195, 248, 188, 195, 255, 255, 128, 128, 128, + 1, 24, 239, 251, 218, 219, 255, 205, 128, 128, 128, + 201, 51, 219, 255, 196, 186, 128, 128, 128, 128, 128, + 69, 46, 190, 239, 201, 218, 255, 228, 128, 128, 128, + 1, 191, 251, 255, 255, 128, 128, 128, 128, 128, 128, + 223, 165, 249, 255, 213, 255, 128, 128, 128, 128, 128, + 141, 124, 248, 255, 255, 128, 128, 128, 128, 128, 128, + 1, 16, 248, 255, 255, 128, 128, 128, 128, 128, 128, + 190, 36, 230, 255, 236, 255, 128, 128, 128, 128, 128, + 149, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 1, 226, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 247, 192, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 240, 128, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 1, 134, 252, 255, 255, 128, 128, 128, 128, 128, 128, + 213, 62, 250, 255, 255, 128, 128, 128, 128, 128, 128, + 55, 93, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 202, 24, 213, 235, 186, 191, 220, 160, 240, 175, 255, + 126, 38, 182, 232, 169, 184, 228, 174, 255, 187, 128, + 61, 46, 138, 219, 151, 178, 240, 170, 255, 216, 128, + 1, 112, 230, 250, 199, 191, 247, 159, 255, 255, 128, + 166, 109, 228, 252, 211, 215, 255, 174, 128, 128, 128, + 39, 77, 162, 232, 172, 180, 245, 178, 255, 255, 128, + 1, 52, 220, 246, 198, 199, 249, 220, 255, 255, 128, + 124, 74, 191, 243, 183, 193, 250, 221, 255, 255, 128, + 24, 71, 130, 219, 154, 170, 243, 182, 255, 255, 128, + 1, 182, 225, 249, 219, 240, 255, 224, 128, 128, 128, + 149, 150, 226, 252, 216, 205, 255, 171, 128, 128, 128, + 28, 108, 170, 242, 183, 194, 254, 223, 255, 255, 128, + 1, 81, 230, 252, 204, 203, 255, 192, 128, 128, 128, + 123, 102, 209, 247, 188, 196, 255, 233, 128, 128, 128, + 20, 95, 153, 243, 164, 173, 255, 203, 128, 128, 128, + 1, 222, 248, 255, 216, 213, 128, 128, 128, 128, 128, + 168, 175, 246, 252, 235, 205, 255, 255, 128, 128, 128, + 47, 116, 215, 255, 211, 212, 255, 255, 128, 128, 128, + 1, 121, 236, 253, 212, 214, 255, 255, 128, 128, 128, + 141, 84, 213, 252, 201, 202, 255, 219, 128, 128, 128, + 42, 80, 160, 240, 162, 185, 255, 205, 128, 128, 128, + 1, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 244, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 238, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, +}; +inline constexpr std::uint16_t dc_qlookup[128] = { + 4, 5, 6, 7, 8, 9, 10, 10, 11, 12, 13, 14, 15, 16, 17, 17, + 18, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 25, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 91, 93, 95, 96, 98, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, + 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 143, 145, 148, 151, 154, 157, +}; +inline constexpr std::uint16_t ac_qlookup[128] = { + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, + 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, + 110, 112, 114, 116, 119, 122, 125, 128, 131, 134, 137, 140, 143, 146, 149, 152, + 155, 158, 161, 164, 167, 170, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, + 213, 217, 221, 225, 229, 234, 239, 245, 249, 254, 259, 264, 269, 274, 279, 284, +}; +} // namespace dingosdk::capture::vp8 diff --git a/Extension/Rendering/Capture/capture_webm.cpp b/Extension/Rendering/Capture/capture_webm.cpp new file mode 100644 index 00000000..8b1b9835 --- /dev/null +++ b/Extension/Rendering/Capture/capture_webm.cpp @@ -0,0 +1,183 @@ +#include "capture_encode.h" +#include "capture_vp8.h" +#include +#include +#include +#include + +// WebM (Matroska) with VP8 video, and transparency the way Chrome, Firefox and ffmpeg read it: the +// alpha as a second VP8 picture (its luma) in each frame's BlockAdditional, the track marked with +// AlphaMode 1. +namespace dingosdk::capture { +namespace { +// EBML element ids and sizes. +void put_id(std::vector& out, std::uint32_t id) { + int bytes = id > 0xffffff ? 4 : id > 0xffff ? 3 : id > 0xff ? 2 : 1; + while (bytes--) out.push_back(static_cast(id >> (bytes * 8))); +} +// Always 8 bytes: sizes are known, and fixed-width ones keep this simple. +void put_size(std::vector& out, std::uint64_t size) { + out.push_back(0x01); + for (int i = 6; i >= 0; --i) out.push_back(static_cast(size >> (i * 8))); +} +void element(std::vector& out, std::uint32_t id, const std::vector& body) { + put_id(out, id); + put_size(out, body.size()); + out.insert(out.end(), body.begin(), body.end()); +} +void uint_element(std::vector& out, std::uint32_t id, std::uint64_t value) { + std::vector body; + int bytes = 1; + while (bytes < 8 && (value >> (bytes * 8))) ++bytes; + for (int i = bytes - 1; i >= 0; --i) body.push_back(static_cast(value >> (i * 8))); + element(out, id, body); +} +void string_element(std::vector& out, std::uint32_t id, std::string_view text) { + element(out, id, std::vector(text.begin(), text.end())); +} +void float_element(std::vector& out, std::uint32_t id, double value) { + std::uint64_t bits; + std::memcpy(&bits, &value, 8); + std::vector body; + for (int i = 7; i >= 0; --i) body.push_back(static_cast(bits >> (i * 8))); + element(out, id, body); +} + +// BT.601 limited range, the colour VP8 decoders assume. +vp8::Planes to_planes(const Image& image, bool alpha) { + vp8::Planes planes; + planes.width = image.width; + planes.height = image.height; + const std::size_t cw = (image.width + 1) / 2, ch = (image.height + 1) / 2; + planes.y.resize(std::size_t{image.width} * image.height); + planes.u.assign(cw * ch, 128); + planes.v.assign(cw * ch, 128); + for (std::uint32_t y = 0; y < image.height; ++y) + for (std::uint32_t x = 0; x < image.width; ++x) { + const auto* p = image.at(x, y); + planes.y[std::size_t{y} * image.width + x] = + alpha ? p[3] : static_cast(std::lround(16 + 0.2568 * p[0] + 0.5041 * p[1] + 0.0979 * p[2])); + } + if (alpha) return planes; + for (std::size_t cy = 0; cy < ch; ++cy) + for (std::size_t cx = 0; cx < cw; ++cx) { + double r = 0, g = 0, b = 0; + int n = 0; + for (std::size_t dy = 0; dy < 2; ++dy) + for (std::size_t dx = 0; dx < 2; ++dx) { + const auto x = cx * 2 + dx, y = cy * 2 + dy; + if (x >= image.width || y >= image.height) continue; + const auto* p = image.at(static_cast(x), static_cast(y)); + r += p[0], g += p[1], b += p[2], ++n; + } + r /= n, g /= n, b /= n; + planes.u[cy * cw + cx] = static_cast(std::clamp(std::lround(128 - 0.1482 * r - 0.2910 * g + 0.4392 * b), 16L, 240L)); + planes.v[cy * cw + cx] = static_cast(std::clamp(std::lround(128 + 0.4392 * r - 0.3678 * g - 0.0714 * b), 16L, 240L)); + } + return planes; +} + +// Fully transparent pixels take the colour of the nearest opaque ones along their row, so VP8's +// blocks do not smear an arbitrary hidden colour into the board's edge. +Image bleed_colour(const Image& image) { + Image out = image; + for (std::uint32_t y = 0; y < out.height; ++y) { + std::int64_t last = -1; + for (std::uint32_t x = 0; x < out.width; ++x) { + if (out.at(x, y)[3]) { + if (last < 0) for (std::uint32_t k = 0; k < x; ++k) std::copy_n(out.at(x, y), 3, out.at(k, y)); + last = x; + } else if (last >= 0) { + std::copy_n(out.at(static_cast(last), y), 3, out.at(x, y)); + } + } + } + return out; +} +} // namespace + +std::vector encode_webm(const Animation& animation, const WebmOptions& options) { + if (animation.frames.empty() || animation.frames.front().empty()) throw std::invalid_argument("No frames to save."); + const auto width = animation.frames.front().width, height = animation.frames.front().height; + for (const auto& frame : animation.frames) + if (frame.width != width || frame.height != height) throw std::invalid_argument("The frames are not all the same size."); + bool transparent = false; + for (const auto& frame : animation.frames) + for (std::size_t i = 3; i < frame.rgba.size() && !transparent; i += 4) transparent = frame.rgba[i] != 255; + const auto fps = std::clamp(animation.fps, 1, 120); + + std::vector out; + { + std::vector ebml; + uint_element(ebml, 0x4286, 1); // EBMLVersion + uint_element(ebml, 0x42f7, 1); // EBMLReadVersion + uint_element(ebml, 0x42f2, 4); // EBMLMaxIDLength + uint_element(ebml, 0x42f3, 8); // EBMLMaxSizeLength + string_element(ebml, 0x4282, "webm"); + uint_element(ebml, 0x4287, 4); // DocTypeVersion (BlockAdditions, AlphaMode) + uint_element(ebml, 0x4285, 2); // DocTypeReadVersion + element(out, 0x1a45dfa3, ebml); + } + std::vector segment; + { + std::vector info; + uint_element(info, 0x2ad7b1, 1000000); // TimestampScale: milliseconds + float_element(info, 0x4489, 1000.0 * static_cast(animation.frames.size()) / fps); // Duration + string_element(info, 0x4d80, "ReSkate Fit Check"); + string_element(info, 0x5741, "ReSkate Fit Check"); + element(segment, 0x1549a966, info); + } + { + std::vector video, track, tracks; + uint_element(video, 0xb0, width); + uint_element(video, 0xba, height); + if (transparent) uint_element(video, 0x53c0, 1); // AlphaMode + uint_element(track, 0xd7, 1); // TrackNumber + uint_element(track, 0x73c5, 1); // TrackUID + uint_element(track, 0x83, 1); // TrackType: video + uint_element(track, 0x9c, 0); // FlagLacing + uint_element(track, 0x23e383, 1000000000ull / fps); // DefaultDuration (ns) + string_element(track, 0x86, "V_VP8"); + element(track, 0xe0, video); + element(tracks, 0xae, track); + element(segment, 0x1654ae6b, tracks); + } + // Clusters of at most 20 seconds, so the 16-bit block timestamps stay in range. + std::vector cluster; + std::uint64_t cluster_start = 0; + const auto close_cluster = [&] { + if (!cluster.empty()) element(segment, 0x1f43b675, cluster); + cluster.clear(); + }; + for (std::size_t i = 0; i < animation.frames.size(); ++i) { + const std::uint64_t time = i * 1000 / fps; + if (cluster.empty() || time - cluster_start >= 20000) { + close_cluster(); + cluster_start = time; + uint_element(cluster, 0xe7, cluster_start); // Timestamp + } + const auto& frame = animation.frames[i]; + const auto picture = vp8::encode_keyframe(to_planes(transparent ? bleed_colour(frame) : frame, false), options.quality); + const auto relative = static_cast(time - cluster_start); + std::vector block{0x81, static_cast(static_cast(relative) >> 8), + static_cast(relative), 0x00}; + block.insert(block.end(), picture.begin(), picture.end()); + if (!transparent) { + // A SimpleBlock, flagged as a key frame. + block[3] = 0x80; + element(cluster, 0xa3, block); + continue; + } + std::vector group, additions, more; + element(group, 0xa1, block); // Block + uint_element(more, 0xee, 1); // BlockAddID + element(more, 0xa5, vp8::encode_keyframe(to_planes(frame, true), options.alpha_quality)); // BlockAdditional + element(additions, 0xa6, more); + element(group, 0x75a1, additions); + element(cluster, 0xa0, group); + } + close_cluster(); + element(out, 0x18538067, segment); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/UI/Overlay/depth_probe.cpp b/Extension/UI/Overlay/depth_probe.cpp new file mode 100644 index 00000000..003cb714 --- /dev/null +++ b/Extension/UI/Overlay/depth_probe.cpp @@ -0,0 +1,520 @@ +#include "overlay_internal.h" +#include "Engine/Core/Log/logging.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Research aid for screen captures (`capture depth`; Fit Check's `fitcheck depth`): where the game keeps its depth while it draws a +// frame, for cutting the board or skater out exactly. Each time a depth buffer the size of the +// screen (or near it) stops being written to over the next two frames (a transition barrier out of +// DEPTH_WRITE), a copy of it is put on that same command list, just before the barrier; a few frames +// later the copies are read back and written to Captures\depth as 16-bit PGM pictures (near +// and far stretched to black and white), with a log of what each was. +// +// ID3D12GraphicsCommandList::ResourceBarrier (slot 26) is hooked once, when the overlay starts +// (through its own command list: every list of the runtime shares it). Unarmed, the hook only +// passes the call on. +namespace dingosdk::overlay { +namespace { +using ResourceBarrierFn = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList*, UINT, const D3D12_RESOURCE_BARRIER*); +std::atomic original_barrier{}; + +struct Snapshot { + Microsoft::WRL::ComPtr readback; + D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout{}; + D3D12_RESOURCE_DESC desc{}; + D3D12_RESOURCE_STATES after{}; + std::uintptr_t resource{}; + unsigned frame{}, order{}; +}; +struct Probe { + std::mutex mutex; + std::atomic armed{}; // presents left in the window (frames recorded then are probed) + std::atomic settle{}; // presents to wait after the window before reading back + std::atomic requested{}; + std::filesystem::path folder; + std::uint32_t screen_width{}, screen_height{}; + unsigned frame{}, order{}; + std::vector snapshots; + // Readback buffers made when the probe starts, on the overlay's device, each big enough for a + // screen-sized depth plane: making them in the hook, on the game's list's device, failed. + std::vector> spare; + UINT64 spare_size{}; + unsigned barriers{}, depth_barriers{}, skipped_small{}, skipped_type{}; + std::vector notes; // each depth barrier, and why one was not copied (the first few) + std::vector seen; // each depth resource's description, once +}; +Probe& probe() { static auto* value = new Probe; return *value; } + +bool depth_capable(const D3D12_RESOURCE_DESC& d) { + return d.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE2D && (d.Flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL) && d.SampleDesc.Count == 1; +} + +// The depth tap: while a capture records (reskate.capture), the game's main depth buffer (the screen's size, the +// first one each frame that goes from being written to being read, found with `capture depth`, +// 2026-10-21) is copied each frame into one of a few readback buffers, with a number after it so +// that the copy the GPU has made can be told from one still waiting. +struct TapSlot { + Microsoft::WRL::ComPtr readback, marker; + std::uint32_t* marker_cpu{}; + std::uint32_t seq{}; + D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout{}; + UINT64 total{}; +}; +struct Tap { + std::mutex mutex; + std::atomic on{}; + std::vector slots; + std::size_t next{}; + std::uint32_t seq{}; + std::uint64_t epoch{}, copied_epoch{~std::uint64_t{0}}; + std::uint32_t width{}, height{}; + UINT64 slot_size{}; + unsigned idle{}; // presents since a grab last wanted it + TapSlot* mapped{}; // the copy handed out by depth_tap_latest + bool said{}; +}; +Tap& tap() { static auto* value = new Tap; return *value; } + +constexpr std::size_t tap_slots = 4; + +void tap_barrier(ID3D12GraphicsCommandList* list, const D3D12_RESOURCE_BARRIER& b, ResourceBarrierFn original) { + auto& t = tap(); + if (b.Type != D3D12_RESOURCE_BARRIER_TYPE_TRANSITION || !b.Transition.pResource) return; + if (!(b.Transition.StateBefore & D3D12_RESOURCE_STATE_DEPTH_WRITE)) return; + constexpr auto read = D3D12_RESOURCE_STATE_DEPTH_READ | D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE | D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE; + if (!(b.Transition.StateAfter & read)) return; + if (b.Flags & D3D12_RESOURCE_BARRIER_FLAG_END_ONLY) return; + if (b.Transition.Subresource != D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES && b.Transition.Subresource != 0) return; + std::lock_guard lock(t.mutex); + if (t.slots.empty() || t.copied_epoch == t.epoch) return; + const auto desc = b.Transition.pResource->GetDesc(); + if (!depth_capable(desc) || desc.Width != t.width || desc.Height != t.height || desc.DepthOrArraySize != 1) return; + if (desc.Format != DXGI_FORMAT_R32G8X24_TYPELESS && desc.Format != DXGI_FORMAT_D32_FLOAT_S8X24_UINT && + desc.Format != DXGI_FORMAT_R32_TYPELESS && desc.Format != DXGI_FORMAT_D32_FLOAT) + return; + if (list->GetType() != D3D12_COMMAND_LIST_TYPE_DIRECT) return; + auto& slot = t.slots[t.next]; + t.next = (t.next + 1) % t.slots.size(); + slot.layout = {}; + slot.layout.Footprint.Format = DXGI_FORMAT_R32_TYPELESS; + slot.layout.Footprint.Width = static_cast(desc.Width); + slot.layout.Footprint.Height = desc.Height; + slot.layout.Footprint.Depth = 1; + slot.layout.Footprint.RowPitch = (static_cast(desc.Width) * 4 + 255) / 256 * 256; + slot.total = std::uint64_t{slot.layout.Footprint.RowPitch} * desc.Height; + if (slot.total + 4 > t.slot_size) return; + D3D12_RESOURCE_BARRIER ours[2]{}; + ours[0].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + ours[0].Transition.pResource = b.Transition.pResource; + ours[0].Transition.Subresource = b.Transition.Subresource; + ours[0].Transition.StateBefore = b.Transition.StateBefore; + ours[0].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + ours[1] = ours[0]; + std::swap(ours[1].Transition.StateBefore, ours[1].Transition.StateAfter); + original(list, 1, &ours[0]); + D3D12_TEXTURE_COPY_LOCATION to{}, from{}; + to.pResource = slot.readback.Get(); + to.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + to.PlacedFootprint = slot.layout; + from.pResource = b.Transition.pResource; + from.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + from.SubresourceIndex = 0; + list->CopyTextureRegion(&to, 0, 0, 0, &from, nullptr); + original(list, 1, &ours[1]); + // The number, after the copy on the same list: when it is in the readback buffer, so is the copy. + slot.seq = ++t.seq; + *slot.marker_cpu = slot.seq; + list->CopyBufferRegion(slot.readback.Get(), slot.total, slot.marker.Get(), 0, 4); + t.copied_epoch = t.epoch; +} + +void STDMETHODCALLTYPE hooked_barrier(ID3D12GraphicsCommandList* list, UINT count, const D3D12_RESOURCE_BARRIER* barriers) { + const auto original = original_barrier.load(); + if (tap().on.load(std::memory_order_relaxed) && list && barriers) { + try { + for (UINT i = 0; i < count; ++i) tap_barrier(list, barriers[i], original); + } catch (...) {} + } + auto& p = probe(); + if (p.armed.load(std::memory_order_relaxed) > 0 && list && barriers) { + try { + std::lock_guard lock(p.mutex); + p.barriers += count; + for (UINT i = 0; i < count; ++i) { + const auto& b = barriers[i]; + if (b.Type != D3D12_RESOURCE_BARRIER_TYPE_TRANSITION || !b.Transition.pResource) continue; + if (!(b.Transition.StateBefore & D3D12_RESOURCE_STATE_DEPTH_WRITE)) continue; + const auto desc = b.Transition.pResource->GetDesc(); + if (!depth_capable(desc)) continue; + ++p.depth_barriers; + const auto key = reinterpret_cast(b.Transition.pResource); + const auto line = std::format("{:#x} {}x{} format {} mips {} array {} flags {:#x}", key, desc.Width, desc.Height, + static_cast(desc.Format), desc.MipLevels, desc.DepthOrArraySize, + static_cast(desc.Flags)); + if (std::find(p.seen.begin(), p.seen.end(), line) == p.seen.end() && p.seen.size() < 64) p.seen.push_back(line); + // Only near the screen's size, on a direct list, whole or its first subresource. + const auto note = [&](const char* what) { + if (p.notes.size() < 80) + p.notes.push_back(std::format("frame {} {:#x} sub {:#x} {:#x}->{:#x} flags {:#x}: {}", p.frame, key, b.Transition.Subresource, + static_cast(b.Transition.StateBefore), static_cast(b.Transition.StateAfter), + static_cast(b.Flags), what)); + }; + if (desc.Width * 3 < std::uint64_t{p.screen_width} || desc.Height * 3 < p.screen_height) { ++p.skipped_small; continue; } + if (list->GetType() != D3D12_COMMAND_LIST_TYPE_DIRECT) { ++p.skipped_type; continue; } + // A split barrier's start leaves the buffer as it is until then (copied before it); its + // end finds it already on its way. + if (b.Flags & D3D12_RESOURCE_BARRIER_FLAG_END_ONLY) { note("end of a split barrier, not copied"); continue; } + if (b.Transition.Subresource != D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES && b.Transition.Subresource != 0) { note("another plane, not copied"); continue; } + if (desc.DepthOrArraySize > 1) { note("an array (shadow cascades), not copied"); continue; } + if (std::any_of(p.snapshots.begin(), p.snapshots.end(), [&](const Snapshot& taken) { return taken.resource == key; })) { + note("copied already"); + continue; + } + if (p.spare.empty()) { note("no readback buffer left"); continue; } + Microsoft::WRL::ComPtr device; + if (FAILED(list->GetDevice(IID_PPV_ARGS(&device)))) { note("no device"); continue; } + Snapshot snap; + UINT rows{}; + UINT64 row_bytes{}, total{}; + device->GetCopyableFootprints(&desc, 0, 1, 0, &snap.layout, &rows, &row_bytes, &total); + // The runtime turns down the game's depth buffers (flag 0x400, newer than it knows) and + // says so with a total of all ones: the depth plane is laid out by hand then, four bytes + // a pixel, rows 256-byte aligned. + const bool four_byte_depth = desc.Format == DXGI_FORMAT_R32G8X24_TYPELESS || desc.Format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT || + desc.Format == DXGI_FORMAT_R32_TYPELESS || desc.Format == DXGI_FORMAT_D32_FLOAT || + desc.Format == DXGI_FORMAT_R24G8_TYPELESS || desc.Format == DXGI_FORMAT_D24_UNORM_S8_UINT; + if ((total == 0 || total == ~UINT64{0}) && four_byte_depth) { + snap.layout = {}; + snap.layout.Footprint.Format = desc.Format == DXGI_FORMAT_R24G8_TYPELESS || desc.Format == DXGI_FORMAT_D24_UNORM_S8_UINT + ? DXGI_FORMAT_R24G8_TYPELESS + : DXGI_FORMAT_R32_TYPELESS; + snap.layout.Footprint.Width = static_cast(desc.Width); + snap.layout.Footprint.Height = desc.Height; + snap.layout.Footprint.Depth = 1; + snap.layout.Footprint.RowPitch = (static_cast(desc.Width) * 4 + 255) / 256 * 256; + total = std::uint64_t{snap.layout.Footprint.RowPitch} * desc.Height; + } + if (!total || total == ~UINT64{0}) { note("no copyable footprint"); continue; } + if (total > p.spare_size) { + note(std::format("bigger than the readback buffers ({} bytes)", total).c_str()); + continue; + } + snap.readback = std::move(p.spare.back()); + p.spare.pop_back(); + // DEPTH_WRITE -> COPY_SOURCE, the copy, and back, before the game's own barrier. + D3D12_RESOURCE_BARRIER ours[2]{}; + ours[0].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + ours[0].Transition.pResource = b.Transition.pResource; + ours[0].Transition.Subresource = b.Transition.Subresource; + ours[0].Transition.StateBefore = b.Transition.StateBefore; + ours[0].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + ours[1] = ours[0]; + std::swap(ours[1].Transition.StateBefore, ours[1].Transition.StateAfter); + original(list, 1, &ours[0]); + D3D12_TEXTURE_COPY_LOCATION to{}, from{}; + to.pResource = snap.readback.Get(); + to.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + to.PlacedFootprint = snap.layout; + from.pResource = b.Transition.pResource; + from.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + from.SubresourceIndex = 0; + list->CopyTextureRegion(&to, 0, 0, 0, &from, nullptr); + original(list, 1, &ours[1]); + snap.desc = desc; + snap.after = b.Transition.StateAfter; + snap.resource = key; + snap.frame = p.frame; + snap.order = p.order++; + p.snapshots.push_back(std::move(snap)); + note("copied"); + } + } catch (...) {} + } + original(list, count, barriers); +} + +// A depth value as 0..1 from a copied row: floats as they are, 24-bit and 16-bit as fractions. +double depth_at(const std::uint8_t* row, std::uint32_t x, DXGI_FORMAT format, std::uint32_t bytes) { + switch (format) { + case DXGI_FORMAT_R32_TYPELESS: + case DXGI_FORMAT_D32_FLOAT: + case DXGI_FORMAT_R32_FLOAT: { + float v; + std::memcpy(&v, row + std::size_t{x} * 4, 4); + return v; + } + case DXGI_FORMAT_R24G8_TYPELESS: + case DXGI_FORMAT_D24_UNORM_S8_UINT: + case DXGI_FORMAT_R24_UNORM_X8_TYPELESS: { + std::uint32_t v; + std::memcpy(&v, row + std::size_t{x} * 4, 4); + return (v & 0xffffff) / 16777215.0; + } + default: + if (bytes == 2) { + std::uint16_t v; + std::memcpy(&v, row + std::size_t{x} * 2, 2); + return v / 65535.0; + } + float v; + std::memcpy(&v, row + std::size_t{x} * 4, 4); + return v; + } +} + +void write_snapshots(detail::State& s) { + auto& p = probe(); + std::vector snapshots; + std::vector seen, notes; + unsigned barriers, depth_barriers, too_small, other_list; + { + std::lock_guard lock(p.mutex); + snapshots.swap(p.snapshots); + seen.swap(p.seen); + notes.swap(p.notes); + barriers = p.barriers, depth_barriers = p.depth_barriers, too_small = p.skipped_small, other_list = p.skipped_type; + } + std::error_code error; + std::filesystem::create_directories(p.folder, error); + std::ofstream log(p.folder / L"depth-probe.txt", std::ios::trunc); + log << std::format("Screen {}x{}. Barriers seen: {}; out of DEPTH_WRITE on a depth texture: {} ({} too small, {} not on a direct list).\n", + p.screen_width, p.screen_height, barriers, depth_barriers, too_small, other_list); + log << "Depth textures seen:\n"; + for (const auto& line : seen) log << " " << line << "\n"; + log << "Depth barriers:\n"; + for (const auto& line : notes) log << " " << line << "\n"; + // The copies are on the game's lists, submitted by now; this makes sure they are done. + const UINT64 signal = ++s.next_fence; + const bool done = SUCCEEDED(s.queue->Signal(s.fence.Get(), signal)) && detail::completed(signal, 5000); + if (!done) log << "The GPU did not finish in time; the pictures may be empty.\n"; + for (std::size_t i = 0; i < snapshots.size(); ++i) { + const auto& snap = snapshots[i]; + const auto w = static_cast(snap.desc.Width), h = snap.desc.Height; + const auto format = snap.layout.Footprint.Format; + const auto bytes = w ? static_cast(snap.layout.Footprint.RowPitch / std::max(w, 1)) : 4; + void* mapped{}; + const D3D12_RANGE range{0, static_cast(snap.layout.Footprint.RowPitch) * h}; + if (FAILED(snap.readback->Map(0, &range, &mapped)) || !mapped) { + log << std::format("#{}: could not be read back.\n", i); + continue; + } + const auto* base = static_cast(mapped) + snap.layout.Offset; + std::vector values(std::size_t{w} * h); + double low = 1e30, high = -1e30; + std::size_t zeros = 0, ones = 0; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto v = depth_at(base + std::size_t{y} * snap.layout.Footprint.RowPitch, x, format, bytes); + values[std::size_t{y} * w + x] = v; + if (v <= 0.0) ++zeros; + else if (v >= 1.0) ++ones; + else low = std::min(low, v), high = std::max(high, v); + } + const D3D12_RANGE nothing{}; + snap.readback->Unmap(0, ¬hing); + if (high < low) low = 0, high = 1; + const auto name = std::format("depth-{:02}-frame{}-{}x{}.pgm", i, snap.frame, w, h); + log << std::format("#{} {}: resource {:#x}, footprint format {}, then state {:#x}, order {}; values {:.6f}..{:.6f}, {} at 0, {} at 1.\n", + i, name, snap.resource, static_cast(format), static_cast(snap.after), snap.order, low, high, zeros, + ones); + std::ofstream picture(p.folder / name, std::ios::binary | std::ios::trunc); + picture << "P5\n" << w << " " << h << "\n65535\n"; + std::vector row(std::size_t{w} * 2); + for (std::uint32_t y = 0; y < h; ++y) { + for (std::uint32_t x = 0; x < w; ++x) { + const auto v = std::clamp((values[std::size_t{y} * w + x] - low) / std::max(high - low, 1e-12), 0.0, 1.0); + const auto q = static_cast(std::lround(v * 65535)); + row[std::size_t{x} * 2] = static_cast(q >> 8), row[std::size_t{x} * 2 + 1] = static_cast(q); + } + picture.write(reinterpret_cast(row.data()), static_cast(row.size())); + } + } + log << std::format("{} picture(s).\n", snapshots.size()); + { + std::lock_guard lock(p.mutex); + p.spare.clear(); + } + logging::printf(logging::Level::info, logging::Channel::graphics, "Fit Check depth probe: %zu depth picture(s) and depth-probe.txt written.", + snapshots.size()); +} +} // namespace + +void request_depth_probe(const wchar_t* folder) noexcept { + try { + auto& p = probe(); + std::lock_guard lock(p.mutex); + p.folder = folder; + p.requested.store(true); + } catch (...) {} +} + +namespace detail { +void install_depth_probe(ID3D12GraphicsCommandList* list) noexcept { + static std::atomic installed{}; + if (!list || installed.exchange(true)) return; + auto* const table = *reinterpret_cast(list); + void* original{}; + if (hook_prepare(table[26], reinterpret_cast(&hooked_barrier), &original) != HookOk) return; + original_barrier.store(reinterpret_cast(original)); + if (hook_enable(table[26]) != HookOk) + logging::printf(logging::Level::warning, logging::Channel::graphics, "Fit Check depth probe: the barrier hook could not start."); +} + +bool depth_probe_pending() { + auto& p = probe(); + if (p.requested.load() || p.armed.load() > 0 || p.settle.load() > 0) return true; + auto& t = tap(); + std::lock_guard lock(t.mutex); + return !t.slots.empty(); // presents until its buffers are let go +} + +void depth_tap_present(State& s, std::uint32_t width, std::uint32_t height, bool wanted) { + auto& t = tap(); + if (!original_barrier.load()) return; + std::lock_guard lock(t.mutex); + ++t.epoch; + if (!wanted) { + t.on.store(false); + // Let go a few frames after the last grab: the GPU may still be copying into them. + if (!t.slots.empty() && ++t.idle > 8) t.slots.clear(); + return; + } + t.idle = 0; + if (t.slots.empty() || t.width != width || t.height != height) { + t.on.store(false); + t.slots.clear(); + t.width = width, t.height = height; + t.slot_size = ((std::uint64_t{width} * 4 + 255) / 256 * 256) * height + 256; + D3D12_HEAP_PROPERTIES heap{}; + heap.Type = D3D12_HEAP_TYPE_READBACK; + D3D12_RESOURCE_DESC bytes{}; + bytes.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bytes.Width = t.slot_size; + bytes.Height = 1; + bytes.DepthOrArraySize = 1; + bytes.MipLevels = 1; + bytes.SampleDesc.Count = 1; + bytes.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + D3D12_HEAP_PROPERTIES upload{}; + upload.Type = D3D12_HEAP_TYPE_UPLOAD; + D3D12_RESOURCE_DESC marker_desc = bytes; + marker_desc.Width = 256; + for (std::size_t i = 0; i < tap_slots; ++i) { + TapSlot slot; + void* cpu{}; + if (FAILED(s.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &bytes, D3D12_RESOURCE_STATE_COPY_DEST, nullptr, + IID_PPV_ARGS(&slot.readback))) || + FAILED(s.device->CreateCommittedResource(&upload, D3D12_HEAP_FLAG_NONE, &marker_desc, D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, + IID_PPV_ARGS(&slot.marker))) || + FAILED(slot.marker->Map(0, nullptr, &cpu)) || !cpu) { + if (!t.said) + logging::printf(logging::Level::warning, logging::Channel::graphics, + "Capture depth: no buffers for the depth copies; cutting out by colour."); + t.said = true; + t.slots.clear(); + return; + } + slot.marker_cpu = static_cast(cpu); + *slot.marker_cpu = 0; + t.slots.push_back(std::move(slot)); + } + t.next = 0; + } + t.on.store(true); +} + +const std::uint8_t* depth_tap_latest(std::uint32_t& width, std::uint32_t& height, std::uint32_t& pitch) { + auto& t = tap(); + std::lock_guard lock(t.mutex); + t.mapped = nullptr; + TapSlot* best{}; + for (auto& slot : t.slots) { + if (!slot.seq || !slot.readback) continue; + void* cpu{}; + const D3D12_RANGE tail{static_cast(slot.total), static_cast(slot.total + 4)}; + if (FAILED(slot.readback->Map(0, &tail, &cpu)) || !cpu) continue; + std::uint32_t written{}; + std::memcpy(&written, static_cast(cpu) + slot.total, 4); + const D3D12_RANGE nothing{}; + slot.readback->Unmap(0, ¬hing); + if (written == slot.seq && (!best || slot.seq > best->seq)) best = &slot; + } + if (!best) return nullptr; + void* cpu{}; + const D3D12_RANGE all{0, static_cast(best->total)}; + if (FAILED(best->readback->Map(0, &all, &cpu)) || !cpu) return nullptr; + t.mapped = best; + width = best->layout.Footprint.Width, height = best->layout.Footprint.Height, pitch = best->layout.Footprint.RowPitch; + return static_cast(cpu); +} + +void depth_tap_done() { + auto& t = tap(); + std::lock_guard lock(t.mutex); + if (!t.mapped) return; + const D3D12_RANGE nothing{}; + t.mapped->readback->Unmap(0, ¬hing); + t.mapped = nullptr; +} + +void depth_probe_present(State& s, std::uint32_t width, std::uint32_t height) { + auto& p = probe(); + if (!original_barrier.load()) { + if (p.requested.exchange(false)) + logging::printf(logging::Level::warning, logging::Channel::graphics, "Fit Check depth probe: no barrier hook, nothing to probe."); + return; + } + if (p.requested.exchange(false)) { + std::lock_guard lock(p.mutex); + p.screen_width = width, p.screen_height = height; + p.snapshots.clear(), p.seen.clear(), p.notes.clear(); + p.barriers = p.depth_barriers = p.skipped_small = p.skipped_type = 0; + p.frame = 0, p.order = 0; + // Readback buffers: up to eight, each for a full-screen plane of four bytes a pixel. + p.spare.clear(); + p.spare_size = ((std::uint64_t{width} * 4 + 255) / 256 * 256) * height + 65536; + D3D12_HEAP_PROPERTIES heap{}; + heap.Type = D3D12_HEAP_TYPE_READBACK; + D3D12_RESOURCE_DESC bytes{}; + bytes.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bytes.Width = p.spare_size; + bytes.Height = 1; + bytes.DepthOrArraySize = 1; + bytes.MipLevels = 1; + bytes.SampleDesc.Count = 1; + bytes.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + HRESULT made = S_OK; + for (int i = 0; i < 8; ++i) { + Microsoft::WRL::ComPtr buffer; + made = s.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &bytes, D3D12_RESOURCE_STATE_COPY_DEST, nullptr, IID_PPV_ARGS(&buffer)); + if (FAILED(made)) break; + p.spare.push_back(std::move(buffer)); + } + if (p.spare.empty()) { + logging::printf(logging::Level::warning, logging::Channel::graphics, "Fit Check depth probe: no readback buffer could be made (0x%08lx).", + static_cast(made)); + return; + } + p.armed.store(3); // this frame's recording onwards, two whole frames + return; + } + if (p.armed.load() > 0) { + std::lock_guard lock(p.mutex); + ++p.frame; + if (p.armed.fetch_sub(1) == 1) p.settle.store(4); + return; + } + if (p.settle.load() > 0 && p.settle.fetch_sub(1) == 1) { + try { write_snapshots(s); } catch (...) {} + } +} +} // namespace detail +} // namespace dingosdk::overlay diff --git a/Extension/UI/Overlay/frame_grab.cpp b/Extension/UI/Overlay/frame_grab.cpp new file mode 100644 index 00000000..196b28d3 --- /dev/null +++ b/Extension/UI/Overlay/frame_grab.cpp @@ -0,0 +1,185 @@ +#include "overlay_internal.h" +#include "Engine/Core/Log/logging.h" +#include +#include +#include +#include +#include + +// Reads a presented frame back from the GPU for a screen capture (overlay.h, FrameGrabHost; +// reskate.capture), from Fit Check (pull request 326). The copy is made on the game's own queue right +// before the overlay draws, and waited for: a capture asks for a few hundred frames at most, and a +// frame's hitch then does not matter. +namespace dingosdk::overlay { +namespace { +std::atomic frame_grab_host{}; +} +void set_frame_grab(const FrameGrabHost* host) noexcept { frame_grab_host.store(host); } +} // namespace dingosdk::overlay + +namespace dingosdk::overlay::detail { +namespace { +// Render thread only (under the render mutex), like the rest of the overlay's graphics state. +struct Grab { + ComPtr allocator; + ComPtr commands; + ComPtr readback; + UINT64 readback_size{}; + std::vector converted; // 8-bit copy of a frame in a wider format + DXGI_FORMAT warned{DXGI_FORMAT_UNKNOWN}; +}; +Grab& grab() { static auto* value = new Grab; return *value; } // process lifetime + +float half_to_float(std::uint16_t h) noexcept { + const std::uint32_t sign = (h >> 15) & 1, exponent = (h >> 10) & 31, mantissa = h & 0x3ff; + float value; + if (!exponent) value = std::ldexp(static_cast(mantissa), -24); + else if (exponent == 31) value = mantissa ? 0.0f : 65504.0f; + else value = std::ldexp(static_cast(mantissa | 0x400), static_cast(exponent) - 25); + return sign ? -value : value; +} +std::uint8_t linear_to_srgb8(float v) noexcept { + v = std::clamp(v, 0.0f, 1.0f); + const float s = v <= 0.0031308f ? v * 12.92f : 1.055f * std::pow(v, 1.0f / 2.4f) - 0.055f; + return static_cast(std::lround(s * 255.0f)); +} + +const FrameGrabHost* host() { return frame_grab_host.load(); } +} // namespace + +bool frame_grab_pending() { + const auto* h = host(); + if (!h || !h->active) return false; + try { return h->active(); } catch (...) { return false; } +} + +void release_frame_grab() { + auto& g = grab(); + g.commands.Reset(); + g.allocator.Reset(); + g.readback.Reset(); + g.readback_size = 0; + g.converted = {}; +} + +void grab_presented_frame(State& s, ID3D12Resource* buffer) { + const auto* h = host(); + if (!h || !h->wants_frame || !h->take_frame || !buffer || !s.device || !s.queue || !s.fence) return; + bool wanted = false; + try { wanted = h->wants_frame(); } catch (...) {} + if (!wanted) return; + auto& g = grab(); + const auto description = buffer->GetDesc(); + const auto format = description.Format; + const bool eight_bit = format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB || + format == DXGI_FORMAT_B8G8R8A8_UNORM || format == DXGI_FORMAT_B8G8R8A8_UNORM_SRGB; + const bool ten_bit = format == DXGI_FORMAT_R10G10B10A2_UNORM; + const bool half = format == DXGI_FORMAT_R16G16B16A16_FLOAT; + if (!eight_bit && !ten_bit && !half) { + if (g.warned != format) + logging::printf(logging::Level::warning, logging::Channel::graphics, + "Capture: the backbuffer format %u cannot be read back.", static_cast(format)); + g.warned = format; + h->take_frame({}); + return; + } + if ((ten_bit || half) && g.warned != format) { + // An HDR swapchain: the picture is converted, but its colours will not match what SDR shows. + logging::printf(logging::Level::warning, logging::Channel::graphics, + "Capture: the game is presenting in HDR (format %u); captured colours are converted to SDR and may look off.", + static_cast(format)); + g.warned = format; + } + D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout{}; + UINT rows{}; + UINT64 row_bytes{}, total{}; + s.device->GetCopyableFootprints(&description, 0, 1, 0, &layout, &rows, &row_bytes, &total); + const auto fail = [&](const char* what) { + logging::printf(logging::Level::warning, logging::Channel::graphics, "Capture: frame read-back failed at %s.", what); + try { h->take_frame({}); } catch (...) {} + }; + if (!total || rows != description.Height) return fail("its layout"); + if (!g.allocator && + FAILED(s.device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(&g.allocator)))) + return fail("CreateCommandAllocator"); + if (!g.commands) { + if (FAILED(s.device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, g.allocator.Get(), nullptr, IID_PPV_ARGS(&g.commands)))) + return fail("CreateCommandList"); + g.commands->Close(); + } + if (g.readback_size < total) { + g.readback.Reset(); + D3D12_HEAP_PROPERTIES heap{}; + heap.Type = D3D12_HEAP_TYPE_READBACK; + D3D12_RESOURCE_DESC bytes{}; + bytes.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bytes.Width = total; + bytes.Height = 1; + bytes.DepthOrArraySize = 1; + bytes.MipLevels = 1; + bytes.SampleDesc.Count = 1; + bytes.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + if (FAILED(s.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &bytes, D3D12_RESOURCE_STATE_COPY_DEST, + nullptr, IID_PPV_ARGS(&g.readback)))) + return fail("CreateCommittedResource"); + g.readback_size = total; + } + if (FAILED(g.allocator->Reset()) || FAILED(g.commands->Reset(g.allocator.Get(), nullptr))) return fail("Reset"); + D3D12_RESOURCE_BARRIER barrier{}; + barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + barrier.Transition.pResource = buffer; + barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_PRESENT; + barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + g.commands->ResourceBarrier(1, &barrier); + D3D12_TEXTURE_COPY_LOCATION to{}, from{}; + to.pResource = g.readback.Get(); + to.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + to.PlacedFootprint = layout; + from.pResource = buffer; + from.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + from.SubresourceIndex = 0; + g.commands->CopyTextureRegion(&to, 0, 0, 0, &from, nullptr); + std::swap(barrier.Transition.StateBefore, barrier.Transition.StateAfter); + g.commands->ResourceBarrier(1, &barrier); + if (FAILED(g.commands->Close())) return fail("Close"); + ID3D12CommandList* lists[] = {g.commands.Get()}; + s.queue->ExecuteCommandLists(1, lists); + const UINT64 signal = ++s.next_fence; + if (FAILED(s.queue->Signal(s.fence.Get(), signal)) || !completed(signal, render_fence_timeout_ms)) return fail("the GPU wait"); + void* mapped{}; + const D3D12_RANGE range{0, static_cast(total)}; + if (FAILED(g.readback->Map(0, &range, &mapped)) || !mapped) return fail("Map"); + const auto* pixels = static_cast(mapped) + layout.Offset; + const auto width = static_cast(description.Width), height = description.Height; + GrabbedFrame frame{pixels, width, height, layout.Footprint.RowPitch, + format == DXGI_FORMAT_B8G8R8A8_UNORM || format == DXGI_FORMAT_B8G8R8A8_UNORM_SRGB}; + if (!eight_bit) { + g.converted.resize(std::size_t{width} * height * 4); + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + auto* out = g.converted.data() + (std::size_t{y} * width + x) * 4; + const auto* in = pixels + std::size_t{y} * layout.Footprint.RowPitch; + if (ten_bit) { + std::uint32_t packed; + std::memcpy(&packed, in + std::size_t{x} * 4, 4); + for (int c = 0; c < 3; ++c) out[c] = static_cast(((packed >> (10 * c)) & 0x3ff) * 255 / 1023); + } else { + std::uint16_t channel[4]; + std::memcpy(channel, in + std::size_t{x} * 8, 8); + for (int c = 0; c < 3; ++c) out[c] = linear_to_srgb8(half_to_float(channel[c])); + } + out[3] = 255; + } + frame = {g.converted.data(), width, height, width * 4, false}; + } + // The depth of this frame (a capture cuts what it shows out by it), when there is one. + std::uint32_t depth_width{}, depth_height{}; + frame.depth = depth_tap_latest(depth_width, depth_height, frame.depth_pitch); + if (frame.depth && (depth_width != width || depth_height != height)) frame.depth = nullptr; + try { h->take_frame(frame); } catch (...) {} + depth_tap_done(); + const D3D12_RANGE nothing{}; + g.readback->Unmap(0, ¬hing); +} +} // namespace dingosdk::overlay::detail diff --git a/Extension/UI/Overlay/overlay.h b/Extension/UI/Overlay/overlay.h index 6bf40e9e..7a146fca 100644 --- a/Extension/UI/Overlay/overlay.h +++ b/Extension/UI/Overlay/overlay.h @@ -494,6 +494,30 @@ struct ItemBrowserHost { }; // `host` must stay valid for the life of the process. void set_item_browser(const ItemBrowserHost *host) noexcept; +// A presented frame as the game drew it, before ReSkate's own UI goes on top: 8-bit RGBA rows +// (BGRA when `bgra`), `pitch` bytes apart. Valid only during the call it is handed to. +struct GrabbedFrame { + const std::uint8_t *data{}; + std::uint32_t width{}, height{}, pitch{}; + bool bgra{}; + // The game's depth for the same frame, when it was found (depth_probe.cpp): 32-bit floats, + // reversed (1 near, 0 at the far plane), `depth_pitch` bytes a row, the size of the frame. + const std::uint8_t *depth{}; + std::uint32_t depth_pitch{}; +}; +// Screen captures (Extension/Rendering/Capture, reskate.capture): asked on every presented frame, on +// the render thread, whether they want its pixels, and handed them when they do. While `active` says +// so the overlay keeps working even with nothing of its own on screen. +struct FrameGrabHost { + bool (*active)() noexcept {}; + bool (*wants_frame)() noexcept {}; + void (*take_frame)(const GrabbedFrame &frame) noexcept {}; +}; +// `host` must stay valid for the life of the process. +void set_frame_grab(const FrameGrabHost *host) noexcept; +// Research aid for captures (`capture depth`): the game's depth buffers over the next two frames are +// copied and written to `folder` as pictures, with depth-probe.txt (depth_probe.cpp). +void request_depth_probe(const wchar_t *folder) noexcept; using ParkSurfaceQueue = bool (*)(const EditorSurfaceRequest &); void set_park_surface_queue(ParkSurfaceQueue) noexcept; using ParkPreviewQueue = bool (*)(const EditorPreviewRequest &); diff --git a/Extension/UI/Overlay/overlay_internal.h b/Extension/UI/Overlay/overlay_internal.h index c93038ef..07c6c29d 100644 --- a/Extension/UI/Overlay/overlay_internal.h +++ b/Extension/UI/Overlay/overlay_internal.h @@ -282,6 +282,23 @@ LRESULT CALLBACK window_proc(HWND window, UINT message, WPARAM wp, LPARAM lp); // Swapchain selection, device objects and the per-frame draw (overlay_render.cpp). bool completed(UINT64 value, DWORD timeout_ms); void destroy_graphics(); +// Screen captures (frame_grab.cpp): a frame is being asked for, so the overlay keeps working; the +// presented frame's pixels handed over before the overlay draws on it; its GPU objects let go. +bool frame_grab_pending(); +void grab_presented_frame(State& s, ID3D12Resource* buffer); +void release_frame_grab(); +// The depth probe (depth_probe.cpp, `capture depth`): its hook, installed with the overlay's command +// list; whether it needs this present; and its turn at it. +void install_depth_probe(ID3D12GraphicsCommandList* list) noexcept; +bool depth_probe_pending(); +void depth_probe_present(State& s, std::uint32_t width, std::uint32_t height); +// The depth tap (depth_probe.cpp): while a frame grab runs, the game's main depth buffer is copied +// every frame. depth_tap_present runs at each present (`wanted`: a grab is running); after the +// frame's own work is known finished, depth_tap_latest maps the newest copy the GPU has made (or +// gives null) until depth_tap_done. +void depth_tap_present(State& s, std::uint32_t width, std::uint32_t height, bool wanted); +const std::uint8_t* depth_tap_latest(std::uint32_t& width, std::uint32_t& height, std::uint32_t& pitch); +void depth_tap_done(); ComPtr object_identity(IUnknown* object); Binding* find_binding(IUnknown* identity); void guarded_render(IDXGISwapChain* chain, UINT flags) noexcept; diff --git a/Extension/UI/Overlay/overlay_render.cpp b/Extension/UI/Overlay/overlay_render.cpp index 9f469a01..49df09f5 100644 --- a/Extension/UI/Overlay/overlay_render.cpp +++ b/Extension/UI/Overlay/overlay_render.cpp @@ -87,6 +87,7 @@ void destroy_graphics() { menu_map_pixels_lost(); s.mod_icon_texture.Reset(); s.mod_icon_upload.Reset(); mod_icon_lost(); + release_frame_grab(); for (auto& texture : s.plugin_textures) texture.Reset(); for (auto& upload : s.plugin_uploads) upload.Reset(); plugin_images_lost(); @@ -367,6 +368,7 @@ bool setup_graphics() { s.commands->SetName(L"ReSkate overlay draw"); dingosdk::overlay::watch_draws(s.commands.Get()); dingosdk::overlay::watch_addresses(s.commands.Get()); + install_depth_probe(s.commands.Get()); s.fence->SetName(L"ReSkate overlay fence"); s.fence_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); if (!s.fence_event) return false; @@ -771,6 +773,7 @@ void render(IDXGISwapChain* presented, UINT flags) { const bool item_browser_frame = item_browser_pending(); const bool perf_frame = perf_hud_pending() || trainer_hud_pending(); const bool hub_frame = hub_page_pending(); + const bool grab_frame = frame_grab_pending() || depth_probe_pending(); // A plugin's screen look is drawn under every frame it asks for (reskate.screen.look). const bool look_frame = screen_look::pending(); if (trainer_open_requested()) s.visible.store(true); @@ -791,7 +794,8 @@ void render(IDXGISwapChain* presented, UINT flags) { } // Hidden, the overlay still draws while a notice or chat line is on screen. if (s.loaded_notice_posted && !notices_pending() && !chat_frame && !game_text_frame && !skate_hud_frame && - !nametag_frame && !plugins_frame && !item_browser_frame && !perf_frame && !hub_frame && !map_download_frame && !look_frame) return; + !nametag_frame && !plugins_frame && !item_browser_frame && !perf_frame && !hub_frame && !map_download_frame && !look_frame && + !grab_frame) return; } else if (!s.ui_was_interactive) { s.ui_was_interactive = true; s.last_model = {}; // Reopening immediately reads fresh state. @@ -813,6 +817,16 @@ void render(IDXGISwapChain* presented, UINT flags) { restore_input(true); return; } + // A screen capture takes the game's frame before anything of ReSkate's is drawn on it. + { + const auto size = frame.buffer->GetDesc(); + const bool grabbing = frame_grab_pending(); + depth_tap_present(s, static_cast(size.Width), size.Height, grabbing); + if (grab_frame) { + if (grabbing) grab_presented_frame(s, frame.buffer.Get()); + depth_probe_present(s, static_cast(size.Width), size.Height); + } + } ImGuiContext* previous = ImGui::GetCurrentContext(); ImGui::SetCurrentContext(s.context); { diff --git a/Extension/UI/Overlay/scripted_pad.cpp b/Extension/UI/Overlay/scripted_pad.cpp index 2ee54409..cc7a0dba 100644 --- a/Extension/UI/Overlay/scripted_pad.cpp +++ b/Extension/UI/Overlay/scripted_pad.cpp @@ -1,6 +1,8 @@ #include "scripted_pad.h" +#include #include #include +#include #include namespace dingosdk::overlay { @@ -13,6 +15,10 @@ struct State { std::chrono::steady_clock::time_point last_tick, done_at; bool active{}; DWORD packet{}; + // A stick held (hold_stick). + int stick{}; + SHORT value{}; + std::chrono::steady_clock::time_point held_until{}; }; State &state() { static auto *value = new State; return *value; } constexpr std::chrono::milliseconds linger{2000}, no_ticks{250}, tick_length{17}; @@ -41,12 +47,27 @@ void pad_tick() noexcept { std::lock_guard lock(s.mutex); if (s.active) advance(s, std::chrono::steady_clock::now()); } +void hold_stick(int stick, float value, unsigned milliseconds) noexcept { + auto &s = state(); + std::lock_guard lock(s.mutex); + s.stick = stick < 0 || stick > 3 ? 2 : stick; + s.value = static_cast(std::lround(std::clamp(value, -1.0f, 1.0f) * 32767.0f)); + s.held_until = value == 0 ? std::chrono::steady_clock::time_point{} + : std::chrono::steady_clock::now() + std::chrono::milliseconds(milliseconds); +} bool scripted_pad(DWORD user, XINPUT_STATE &output) noexcept { auto &s = state(); if (user != 0) return false; std::lock_guard lock(s.mutex); - if (!s.active) return false; const auto now = std::chrono::steady_clock::now(); + if (!s.active) { + if (now >= s.held_until) return false; + output.Gamepad = XINPUT_GAMEPAD{}; + SHORT* axes[] = {&output.Gamepad.sThumbLX, &output.Gamepad.sThumbLY, &output.Gamepad.sThumbRX, &output.Gamepad.sThumbRY}; + *axes[s.stick] = s.value; + output.dwPacketNumber = ++s.packet; + return true; + } // With no game ticks, time alone moves the script on. while (s.current + 1 < s.steps.size() && now - s.last_tick > no_ticks) advance(s, s.last_tick + tick_length); const bool done = s.current + 1 == s.steps.size(); diff --git a/Extension/UI/Overlay/scripted_pad.h b/Extension/UI/Overlay/scripted_pad.h index c275b1db..f03761f1 100644 --- a/Extension/UI/Overlay/scripted_pad.h +++ b/Extension/UI/Overlay/scripted_pad.h @@ -19,4 +19,8 @@ void play_pad(std::vector steps); void pad_tick() noexcept; // True when it wrote the script's pad to `state`. bool scripted_pad(DWORD user, XINPUT_STATE &state) noexcept; +// One stick of the stand-in pad held at `value` (-1..1) for `milliseconds` more, the pad otherwise +// neutral, for turning the game menu's preview (reskate.room.turn); 0 lets go. `stick`: 0 left x, +// 1 left y, 2 right x, 3 right y. A script playing (play_pad) comes first. Any thread. +void hold_stick(int stick, float value, unsigned milliseconds) noexcept; } // namespace dingosdk::overlay diff --git a/Extension/World/character_room.cpp b/Extension/World/character_room.cpp index 6daac750..c37129fc 100644 --- a/Extension/World/character_room.cpp +++ b/Extension/World/character_room.cpp @@ -410,4 +410,6 @@ void tick(std::uintptr_t base, std::uintptr_t skater, bool ready, bool prepare) if (s.way != Way::none) fail(s, "an error in the walk"); } } +std::string menu_buttons(std::uintptr_t base) noexcept { return menu_page(base); } +bool is_skaters_page(const std::string& buttons) noexcept { return skaters_page(buttons); } } // namespace dingosdk::character_room diff --git a/Extension/World/character_room.h b/Extension/World/character_room.h index 18f76838..e1e57608 100644 --- a/Extension/World/character_room.h +++ b/Extension/World/character_room.h @@ -21,4 +21,8 @@ void leave() noexcept; // `prepare`: the room is going to be wanted, so the game's navigation event is looked for now (once a map, // in the background), and the walk does not wait for it. void tick(std::uintptr_t base, std::uintptr_t skater, bool ready, bool prepare) noexcept; +// The game menu's open page, as the navigations of its buttons in one line ("CASPresetPurchase ..."), +// empty when none shows; and whether that is the Skaters page. Client thread. +[[nodiscard]] std::string menu_buttons(std::uintptr_t base) noexcept; +[[nodiscard]] bool is_skaters_page(const std::string& buttons) noexcept; } // namespace dingosdk::character_room diff --git a/Runtime/runtime_init.cpp b/Runtime/runtime_init.cpp index a4bb764c..b254016f 100644 --- a/Runtime/runtime_init.cpp +++ b/Runtime/runtime_init.cpp @@ -1,5 +1,6 @@ #include "runtime_internal.h" #include "Extension/Customization/item_browser.h" +#include "Extension/Rendering/Capture/capture_runtime.h" #include "bootstrap.h" #include "Engine/Core/Hooks/hooks.h" #include "Engine/Core/Log/logging.h" @@ -250,6 +251,7 @@ extern "C" __declspec(dllexport) BOOL WINAPI DingoSDKDebugInitialize() { dingosdk::overlay::set_hub_callbacks(native_callbacks); dingosdk::overlay::set_ui_sound(dingosdk::multiplayer::queue_ui_sound); dingosdk::overlay::set_item_browser(dingosdk::item_browser::overlay_host()); + dingosdk::overlay::set_frame_grab(dingosdk::capture::frame_grab()); dingosdk::overlay::set_game_text_feed(dingosdk::multiplayer::skate_debug_text); dingosdk::overlay::set_skate_hud_feed(dingosdk::multiplayer::skate_hud); dingosdk::overlay::set_nametag_feed(dingosdk::multiplayer::custom_nametags); diff --git a/cmake/Core.cmake b/cmake/Core.cmake index 5d875fb4..9e928e3b 100644 --- a/cmake/Core.cmake +++ b/cmake/Core.cmake @@ -146,6 +146,7 @@ add_library(dingosdk_plugins STATIC Extension/Plugins/plugin_api_world.cpp Extension/Plugins/plugin_api_services.cpp Extension/Plugins/plugin_api_throwdowns.cpp + Extension/Plugins/plugin_api_capture.cpp # The throwdown kit's store and the plugins' modes, which the plugin host and the game's # native UI share (kit::flag is the game's: Extension/Throwdowns/native_flag_session.cpp). Extension/Throwdowns/throwdown_kit.cpp diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index afe8d9f5..3412928f 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -32,7 +32,8 @@ endif() enable_language(C) add_library(dingosdk_miniz STATIC External/miniz/miniz.c) target_include_directories(dingosdk_miniz SYSTEM PUBLIC External/miniz) -target_compile_definitions(dingosdk_miniz PUBLIC MINIZ_NO_ARCHIVE_WRITING_APIS MINIZ_NO_DEFLATE_APIS MINIZ_NO_ZLIB_APIS MINIZ_NO_STDIO MINIZ_NO_TIME) +# Deflate is on for the captures' PNG and APNG files (Extension/Rendering/Capture). +target_compile_definitions(dingosdk_miniz PUBLIC MINIZ_NO_ARCHIVE_WRITING_APIS MINIZ_NO_ZLIB_APIS MINIZ_NO_STDIO MINIZ_NO_TIME) set_target_properties(dingosdk_miniz PROPERTIES FOLDER "Dependencies") add_library(dingosdk_lz4 STATIC External/lz4/lz4.c) target_include_directories(dingosdk_lz4 SYSTEM PUBLIC External/lz4) diff --git a/cmake/Features.cmake b/cmake/Features.cmake index 8b8e81a1..6ffd7d45 100644 --- a/cmake/Features.cmake +++ b/cmake/Features.cmake @@ -35,3 +35,19 @@ target_link_libraries(dingosdk_local_profile PUBLIC dingosdk_fast_travel_unlock include(cmake/ProfileData.cmake) +# Screen captures (Extension/Rendering/Capture, reskate.capture): the recording, matting, the card and the +# GIF / APNG / PNG writers, and WebM through a small VP8 encoder of its own (from Fit Check, pull request +# 326). No Windows or game code, so its tests run anywhere. +add_library(dingosdk_capture STATIC + Extension/Rendering/Capture/capture_image.cpp + Extension/Rendering/Capture/capture_matte.cpp + Extension/Rendering/Capture/capture_panel.cpp + Extension/Rendering/Capture/capture_png.cpp + Extension/Rendering/Capture/capture_gif.cpp + Extension/Rendering/Capture/capture_vp8.cpp + Extension/Rendering/Capture/capture_webm.cpp + Extension/Rendering/Capture/capture_recorder.cpp) +target_include_directories(dingosdk_capture PUBLIC "${PROJECT_SOURCE_DIR}") +# stb_truetype, as shipped with Dear ImGui. +target_include_directories(dingosdk_capture SYSTEM PRIVATE "${PROJECT_SOURCE_DIR}/External/imgui") +target_link_libraries(dingosdk_capture PUBLIC dingosdk_miniz) diff --git a/cmake/Runtime.cmake b/cmake/Runtime.cmake index c4a73140..74809b4e 100644 --- a/cmake/Runtime.cmake +++ b/cmake/Runtime.cmake @@ -208,6 +208,9 @@ add_library(dingosdk_runtime SHARED Extension/Customization/item_origins.cpp Extension/Customization/local_customization_runtime.cpp Extension/Customization/item_browser.cpp + Extension/Customization/worn_items.cpp + Extension/Rendering/Capture/capture_runtime.cpp + Extension/Rendering/Capture/capture_commands.cpp Extension/Progression/local_entitlement_trigger_runtime.cpp Extension/Progression/local_neighborhood_runtime.cpp Extension/Customization/local_player_card_runtime.cpp @@ -399,3 +402,4 @@ if(EXISTS "${dingosdk_startup_art}") "STARTUP_SPLASH RCDATA \"${dingosdk_startup_art_native}\"\n") target_sources(dingosdk_runtime PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/generated/startup_art.rc") endif() +target_link_libraries(dingosdk_runtime PRIVATE dingosdk_capture) diff --git a/cmake/UI.cmake b/cmake/UI.cmake index dfb6996c..36ab9f5a 100644 --- a/cmake/UI.cmake +++ b/cmake/UI.cmake @@ -9,6 +9,8 @@ add_library(dingosdk_overlay STATIC Extension/UI/Overlay/overlay_console.cpp Extension/UI/Overlay/overlay_input.cpp Extension/UI/Overlay/item_browser_overlay.cpp + Extension/UI/Overlay/frame_grab.cpp + Extension/UI/Overlay/depth_probe.cpp Extension/UI/Overlay/overlay_notices.cpp Extension/UI/Overlay/chat_overlay.cpp Extension/UI/Overlay/map_download_card.cpp diff --git a/contrib/plugins/fit-check/README.md b/contrib/plugins/fit-check/README.md new file mode 100644 index 00000000..89df8c15 --- /dev/null +++ b/contrib/plugins/fit-check/README.md @@ -0,0 +1,67 @@ +# Fit Check (plugin) + +Pull request 326's Fit Check as a sandboxed Luau plugin. Press V on the Board screen to spin the +board, or on the Skaters screen (with no item grid open) to spin the skater. It turns once by itself, +hands off, and is saved as a GIF, APNG or WebM, cut out from the backdrop, with an info card beside +it: the deck, grip, trucks and wheels, or the outfit, by name. V again stops it. + +How it works: + +- It holds a stand-in pad's stick over with `reskate.room.turn`, as a player turns the board on the + stick, so it works on mouse and keyboard too. +- `reskate.capture` captures every frame the game draws, before ReSkate's overlay, and cuts each + out by the game's depth (or by colour when there is no depth). +- Once the turn has come round, the frames nearest evenly spaced points of that one turn are saved. +- While it turns, the game's screen-space contact shadows, bloom, lens flare and dirt, film grain, + chromatic aberration and motion blur are switched off with `reskate.settings.set`. For a skater, + screen-space reflections and ambient occlusion are switched off too. All of them are put back + afterwards. + +The files are saved in the plugin's data folder, +`Mods/.reskate_plugin_data/fit-check/fit-checks/`, as `-