diff --git a/CMakeLists.txt b/CMakeLists.txt index f284285c..8926f15e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -401,6 +401,15 @@ if(DINGOSDK_BUILD_XRAY_TESTS) target_link_libraries(dingosdk_xray_skeleton_tests PRIVATE dingosdk_game_archives dingosdk_logging) add_test(NAME xray_skeleton COMMAND dingosdk_xray_skeleton_tests "${DINGOSDK_TEST_GAME_ROOT}") endif() +option(DINGOSDK_BUILD_CAPTURE_TESTS "Build screen capture regression tests" OFF) +if(DINGOSDK_BUILD_CAPTURE_TESTS AND TARGET dingosdk_capture) + enable_testing() + # Matting, resizing, the card, GIF / APNG / WebM files that decode back to what went in, and a + # recording of a turning board found, cut out and saved (reskate.capture). + add_executable(dingosdk_capture_tests Extension/Rendering/Capture/Test/capture_tests.cpp) + target_link_libraries(dingosdk_capture_tests PRIVATE dingosdk_capture) + add_test(NAME capture COMMAND dingosdk_capture_tests "${PROJECT_SOURCE_DIR}/External/fonts") +endif() option(DINGOSDK_BUILD_PLUGIN_TESTS "Build plugin host regression tests" OFF) if(DINGOSDK_BUILD_PLUGIN_TESTS AND WIN32) enable_testing() diff --git a/Extension/Console/commands.cpp b/Extension/Console/commands.cpp index f6906e04..53fa0532 100644 --- a/Extension/Console/commands.cpp +++ b/Extension/Console/commands.cpp @@ -227,6 +227,7 @@ const Commands &game_commands() { register_multiplayer_commands(*result); register_perf_commands(*result); register_item_commands(*result); + register_capture_commands(*result); register_trainer_commands(*result); register_replay_keyframes_commands(*result); register_plugin_commands(*result); diff --git a/Extension/Console/commands.h b/Extension/Console/commands.h index 27c1f7ab..aa0826fd 100644 --- a/Extension/Console/commands.h +++ b/Extension/Console/commands.h @@ -25,6 +25,8 @@ void register_perf_commands(Commands &); // perf_commands.cpp: starts `perf memory`'s trace with the game when RESKATE_MEMORY_TRACE asks for it. void start_memory_trace_from_environment() noexcept; void register_item_commands(Commands &); +// Extension/Rendering/Capture/capture_commands.cpp: `capture depth`, research for reskate.capture. +void register_capture_commands(Commands &); void register_trainer_commands(Commands &); void register_replay_keyframes_commands(Commands &); void register_plugin_commands(Commands &); diff --git a/Extension/Customization/item_browser.cpp b/Extension/Customization/item_browser.cpp index e723c8dd..8501a610 100644 --- a/Extension/Customization/item_browser.cpp +++ b/Extension/Customization/item_browser.cpp @@ -221,6 +221,20 @@ Rows read_rows(const md::Context& context, md::Value items, const Layout& layout return rows; } +// Every item's title as the grids showed it this session (a mod's item has none in the game's own +// catalog), for plugins (reskate.outfit.title). +struct Titles { + std::mutex mutex; + std::map> by_key; +}; +Titles& titles() { static auto* value = new Titles; return *value; } +void remember_title(const std::string& key, const std::string& title) { + if (key.empty() || title.empty()) return; + auto& t = titles(); + std::lock_guard lock(t.mutex); + if (t.by_key.size() < 65536) t.by_key.insert_or_assign(key, title); +} + // The catalog item behind a tile's item model. Empty key: the tile does not show a cosmetic. Item resolve(const md::Context& context, std::uint64_t content) { Item item; @@ -236,6 +250,7 @@ Item resolve(const md::Context& context, std::uint64_t content) { item.key = md::string(md::read(at + context.member(data.type, data_key).offset), 255); item.title = md::string(md::read(at + context.member(data.type, data_title).offset), 255); item.category = md::string(md::read(at + context.member(data.type, data_category).offset), 96); + remember_title(item.key, item.title); const auto& catalogs = content_cache::catalogs(); item.modded = catalogs.available && !catalogs.items.contains(folded(item.key)); return item; @@ -551,6 +566,12 @@ void set_search(std::string text) { sh.search_pending = true; } void set_filter(Filter filter) noexcept { shared().filter.store(static_cast(filter)); } +std::string known_title(std::string_view key) { + auto& t = titles(); + std::lock_guard lock(t.mutex); + const auto found = t.by_key.find(key); + return found == t.by_key.end() ? std::string() : found->second; +} const overlay::ItemBrowserHost* overlay_host() noexcept { static const overlay::ItemBrowserHost host{ [] { @@ -675,6 +696,7 @@ void follow_focus(State& s, Shared& sh, std::uintptr_t base) { if (item == grid->native.end() || item->key == next.focused_key) return; next.focused_title = item->title; next.focused_key = item->key; + next.focused_category = item->category; next.focused_favorite = s.favorites.keys.contains(item->key); } std::lock_guard shared_lock(sh.mutex); @@ -786,10 +808,13 @@ void update(std::uintptr_t base) noexcept { next.shown = static_cast(active->shown.size()) - (std::min)(blanks, static_cast(active->shown.size())); next.total = static_cast(active->native.size()) - blanks; for (const auto& item : active->native) next.favorites += s.favorites.keys.contains(item.key); + for (const auto& item : active->native) + if (!item.key.empty()) next.grid_keys.push_back(item.key); } if (focused) { next.focused_title = focused->title; next.focused_key = focused->key; + next.focused_category = focused->category; next.focused_favorite = s.favorites.keys.contains(focused->key); } std::lock_guard shared_lock(sh.mutex); diff --git a/Extension/Customization/item_browser.h b/Extension/Customization/item_browser.h index e0c085e3..c2946295 100644 --- a/Extension/Customization/item_browser.h +++ b/Extension/Customization/item_browser.h @@ -3,6 +3,7 @@ #include #include #include +#include // Favorites, search and filters for the skater's item grids (Apparel, Accessories, Board, Tattoos, // Gestures, Appearance). The grids stay the game's own: their tile rows are reordered and filtered in @@ -15,7 +16,8 @@ struct View { std::string search; Filter filter{}; unsigned shown{}, total{}, favorites{}; // of the grid that has the focus - std::string focused_title, focused_key; + std::string focused_title, focused_key, focused_category; + std::vector grid_keys; // every item of the grid that has the focus bool focused_favorite{}; }; View view(); @@ -24,6 +26,8 @@ void toggle_focused_favorite() noexcept; void set_search(std::string text); void set_filter(Filter filter) noexcept; void cycle_filter() noexcept; +// An item's title as an item grid showed it this session (mods' items too), or empty. Any thread. +std::string known_title(std::string_view key); // The table the overlay's search bar and keys use (process lifetime). const overlay::ItemBrowserHost* overlay_host() noexcept; // Console research aids; their answers go to the log. diff --git a/Extension/Customization/worn_items.cpp b/Extension/Customization/worn_items.cpp new file mode 100644 index 00000000..4b45229a --- /dev/null +++ b/Extension/Customization/worn_items.cpp @@ -0,0 +1,144 @@ +#include "worn_items.h" +#include "item_browser.h" +#include "local_customization.h" +#include "local_customization_runtime.h" +#include "Engine/Core/Platform/memory.h" +#include "Engine/Game/Build/addresses.h" +#include "Engine/Game/Build/20260929/engine.h" +#include "Engine/Game/Build/20260929/skater_slot_override.h" +#include +#include +#include +#include +#include + +// What the skater wears, read from the game's own presets: ported from Fit Check (pull request 326, +// turnaround.cpp), for plugins (reskate.outfit.worn, board, title). +namespace dingosdk::worn_items { +namespace { +// The board's own recipe in a skater preset (its template), and its slots, in the card's order. Read +// from the game's presets with Fit Check's `turnaround parts`, 2026-10-20. +constexpr std::uint32_t board_template = 0x5e61aaef; +struct BoardSlot { std::uint32_t slot; const char* label; }; +constexpr BoardSlot board_slots[] = { + {0x66071628, "Deck"}, {0x755d5ee0, "Grip tape"}, {0x976f18c4, "Grip cutout"}, {0x0add98be, "Trucks"}, {0xc83bb94e, "Wheels"}}; + +// An item's name as the game shows it: its catalog title, the title an item grid showed (a mod's +// item has no catalog entry), or its key made readable the way the game does for a mod's item. +} // namespace +std::string item_title(const std::string& asset) { + auto title = profile::item_display_metadata(asset, Json::object()).value("title", ""); + if (title.empty()) title = item_browser::known_title(asset); + if (title.empty()) { + title = asset.starts_with("Own_") || asset.starts_with("own_") ? asset.substr(4) : asset; + std::replace(title.begin(), title.end(), '_', ' '); + } + return title; +} + +namespace { +// The selected skater preset (the native getter skater_slot_override.cpp uses), or -1. Game thread. +std::int32_t selected_preset(std::uintptr_t base, std::uintptr_t manager) { + namespace slot = addr::skater_slot_override; + std::array bytes{}; + if (!memory::read_bytes(base + slot::native_selected, bytes.data(), bytes.size()) || bytes != slot::native_selected_prefix) return -1; + return reinterpret_cast(base + slot::native_selected)(manager); +} + +// The board the skater rides now: the board recipe of the selected preset (the first when none is +// selected), its parts in the card's order, empty slots left out. Game thread. +} // namespace +std::vector worn_board() { + std::vector parts; + const auto base = reinterpret_cast(GetModuleHandleW(nullptr)); + std::uintptr_t manager{}, begin{}, end{}; + if (!memory::read(base + addr::engine::loadout_manager, manager) || !manager || !memory::read(manager + 0x18, begin) || + !memory::read(manager + 0x20, end) || begin >= end || (end - begin) % 0x48 || (end - begin) / 0x48 > 10) + return parts; + const auto count = static_cast((end - begin) / 0x48); + auto preset = selected_preset(base, manager); + if (preset < 0 || preset >= count) preset = 0; + profile::CosmeticLoadout loadout; + std::vector native; + if (!profile_runtime::read_cosmetic_recipes(reinterpret_cast(begin + static_cast(preset) * 0x48), loadout, native)) + return parts; + for (const auto& recipe : loadout.recipes) { + if (recipe.template_key != board_template) continue; + for (const auto& wanted : board_slots) + for (const auto& item : recipe.items) { + if (item.slot != wanted.slot || item.asset.empty()) continue; + parts.push_back({wanted.label, item.asset, item_title(item.asset)}); + } + break; + } + return parts; +} + +// What the skater wears now: every item of the selected preset +// but the board's, labelled by its catalog category (the game's names for its grids: "tops", +// "headgear"...), clothes and accessories only (not the body, hair, tattoos or gestures), in the +// order they are worn from head to foot. Game thread. +namespace { +struct OutfitKind { const char* word; const char* label; int order; }; +constexpr OutfitKind outfit_kinds[] = { + {"head", "Headwear", 0}, {"hat", "Headwear", 0}, {"cap", "Headwear", 0}, {"beanie", "Headwear", 0}, {"helmet", "Headwear", 0}, + {"eye", "Eyewear", 1}, {"glass", "Eyewear", 1}, {"mask", "Face", 1}, + {"neck", "Neckwear", 2}, {"jewel", "Jewellery", 2}, {"earring", "Earrings", 2}, + {"costume", "Costume", 3}, {"outfit", "Outfit", 3}, + {"outer", "Outerwear", 4}, {"jacket", "Outerwear", 4}, {"top", "Top", 5}, {"shirt", "Top", 5}, {"hood", "Top", 5}, + {"glove", "Gloves", 6}, {"wrist", "Wrist", 6}, {"watch", "Wrist", 6}, {"bag", "Bag", 7}, {"backpack", "Bag", 7}, + {"bottom", "Bottoms", 8}, {"pant", "Bottoms", 8}, {"short", "Bottoms", 8}, {"leg", "Bottoms", 8}, + {"sock", "Socks", 9}, {"foot", "Shoes", 10}, {"shoe", "Shoes", 10}}; +constexpr const char* not_outfit[] = {"skin", "face_", "body", "hair", "beard", "brow", "makeup", "eyecolo", "eye_colo", + "tattoo", "gesture", "emote", "voice", "bkobject", "board", "deck", "truck", "wheel", "grip", + "locomotion", "lipcolo", "mole", "stubble", "freckle", "scar", "nose", "mouth", "teeth"}; +std::string folded(std::string_view text) { + std::string out(text); + for (auto& c : out) c = static_cast(std::tolower(static_cast(c))); + return out; +} +} // namespace +std::vector worn_outfit() { + std::vector parts; + const auto base = reinterpret_cast(GetModuleHandleW(nullptr)); + std::uintptr_t manager{}, begin{}, end{}; + if (!memory::read(base + addr::engine::loadout_manager, manager) || !manager || !memory::read(manager + 0x18, begin) || + !memory::read(manager + 0x20, end) || begin >= end || (end - begin) % 0x48 || (end - begin) / 0x48 > 10) + return parts; + const auto count = static_cast((end - begin) / 0x48); + auto preset = selected_preset(base, manager); + if (preset < 0 || preset >= count) preset = 0; + profile::CosmeticLoadout loadout; + std::vector native; + if (!profile_runtime::read_cosmetic_recipes(reinterpret_cast(begin + static_cast(preset) * 0x48), loadout, native)) + return parts; + const auto& known = profile_runtime::cosmetic_runtime().items; + std::vector> found; + for (const auto& recipe : loadout.recipes) { + if (recipe.template_key == board_template) continue; + for (const auto& item : recipe.items) { + if (item.asset.empty()) continue; + const auto entry = known.find(item.asset); + const auto category = folded(entry != known.end() ? entry->second.category : std::string()); + const auto has = [&](const char* word) { return category.find(word) != std::string::npos; }; + // The body's own parts are "cust..." (custhead, custeye, custnoses, custbodies, custmoles...). + if (category.empty() || category.starts_with("cust") || std::any_of(std::begin(not_outfit), std::end(not_outfit), has)) + continue; + const auto kind = std::find_if(std::begin(outfit_kinds), std::end(outfit_kinds), [&](const OutfitKind& k) { return has(k.word); }); + std::string label; + int order = 20; + if (kind != std::end(outfit_kinds)) label = kind->label, order = kind->order; + else { + // A category this does not know: its name made readable. + label = entry->second.category; + std::replace(label.begin(), label.end(), '_', ' '); + if (!label.empty()) label[0] = static_cast(std::toupper(static_cast(label[0]))); + } + found.push_back({order, {label, item.asset, item_title(item.asset)}}); + } + } + std::stable_sort(found.begin(), found.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + for (auto& entry : found) parts.push_back(std::move(entry.second)); + return parts; +} +} // namespace dingosdk::worn_items diff --git a/Extension/Customization/worn_items.h b/Extension/Customization/worn_items.h new file mode 100644 index 00000000..bb83de86 --- /dev/null +++ b/Extension/Customization/worn_items.h @@ -0,0 +1,19 @@ +#pragma once +#include +#include + +// What the skater wears now, read from the selected skater preset (game thread): the clothes and +// accessories by the kind of item, from head to foot, and the board's parts; an item's title as the +// game shows it. For plugins (reskate.outfit.worn, board, title); from Fit Check (pull request 326). +namespace dingosdk::worn_items { +struct Worn { + std::string slot; // "Headwear", "Top", ... "Shoes"; or "Deck", "Grip tape", "Grip cutout", "Trucks", "Wheels" + std::string item; // its key + std::string title; // as the game shows it +}; +std::vector worn_outfit(); +std::vector worn_board(); +// Its catalog title, the title an item grid showed this session (a mod's item has no catalog +// entry), or its key made readable the way the game does for a mod's item. +std::string item_title(const std::string& asset); +} // namespace dingosdk::worn_items diff --git a/Extension/Plugins/Test/plugin_tests.cpp b/Extension/Plugins/Test/plugin_tests.cpp index f7907539..7e3cde78 100644 --- a/Extension/Plugins/Test/plugin_tests.cpp +++ b/Extension/Plugins/Test/plugin_tests.cpp @@ -503,14 +503,74 @@ struct FakeGame final : Game { Track track() override { return now_playing; } std::string music(std::string_view command) override { calls.push_back("music " + std::string(command)); return {}; } Progress progress() override { return {true, {{"historic", "Historic", 4}}, {{"c1", "line", "historic", 2, 1, 3}}, 1500, 9000, 2}; } - std::optional setting(std::string_view name) override { return name == "Render.Fov" ? std::optional("90") : std::nullopt; } - std::vector setting_names() override { return {"Render.Fov"}; } + std::optional setting(std::string_view name) override { + if (std::find(render_settings.begin(), render_settings.end(), name) != render_settings.end()) return "true"; + return name == "Render.Fov" ? std::optional("90") : std::nullopt; + } + std::vector setting_names() override { + auto names = render_settings; + names.insert(names.begin(), "Render.Fov"); + return names; + } std::string bind(std::string_view plugin, std::string_view action, std::uint32_t combo) override { binds[std::string(plugin) + "/" + std::string(action)] = combo; return {}; } std::vector> take_presses() override { return std::exchange(presses, {}); } void release(std::string_view plugin) override { released.emplace_back(plugin); entity_clear(plugin); } + // The game menu, its item grid, what is worn, and a screen capture of something turning once + // every `turning` frames (reskate.capture, room.turn, menu.game/grid, outfit.board). + GameMenu menu_now; + std::optional grid_now; + std::vector board_now, outfit_now; + std::vector render_settings; // more settings, each on + CaptureStatus capture_now{"idle"}; + double turning = 100; + std::vector capture_starts; + std::vector capture_saves; + float turned{}; + std::string room_turn(std::string_view, float value, int stick) override { + if (value != 0 && !menu_now.open) return "the game's menu is not open"; + turned = value; + calls.push_back(std::format("turn {:.2f} {}", value, stick)); + return {}; + } + GameMenu game_menu() override { return menu_now; } + std::optional menu_grid() override { return grid_now; } + std::vector board_parts() override { return board_now; } + std::vector outfit_worn() override { return outfit_now; } + std::string capture_start(std::string_view plugin, const CaptureStart& request) override { + capture_starts.push_back(request); + capture_now = {"recording"}; + calls.push_back(std::format("capture start {}", plugin)); + return {}; + } + void capture_stop(std::string_view) override { + if (capture_now.state == "recording") capture_now.state = "stopped"; + } + void capture_discard(std::string_view plugin) override { + capture_now = {"idle"}; + calls.push_back(std::format("capture discard {}", plugin)); + } + CaptureStatus capture_status(std::string_view) override { return capture_now; } + float capture_compare(std::string_view, std::uint32_t a, std::uint32_t b) override { + if (a >= capture_now.taken || b >= capture_now.taken) return -1; + return 100 * static_cast(std::abs(std::sin(3.14159265358979 * (double(a) - double(b)) / turning))); + } + std::vector capture_kept(std::string_view) override { + std::vector out; + for (std::uint32_t n = 0; n < capture_now.taken; ++n) out.push_back(n); + return out; + } + std::string capture_save(std::string_view, const CaptureSave& request) override { + capture_saves.push_back(request); + capture_now = {"saving", capture_now.taken, capture_now.taken}; + return {}; + } + std::string files_reveal(std::string_view plugin, std::string_view folder) override { + calls.push_back(std::format("reveal {} {}", plugin, folder)); + return {}; + } bool called(std::string_view part) const { return std::any_of(calls.begin(), calls.end(), [&](const auto& c) { return c.find(part) != std::string::npos; }); } @@ -2672,6 +2732,92 @@ void plugins_add_throwdown_modes() { // The throwdown kit: a plugin registers a mode, publishes its documents (parsed into the store the // native UI reads), hears what the native UI posts, holds the native flag, keeps data files, opens a // dialog; and everything goes when it stops. +// Fit Check as a mod (contrib/plugins/fit-check): its own checks pass, and V on the Board screen +// turns the board until it has come round, eases the stick off when it turns too fast for the frames +// wanted, and saves evenly spaced frames of one turn with the board's parts on its card, the +// browsed deck in place of the worn one; the game's bloom is off while it turns. +void fit_check_turns_the_board_and_saves_it() { + TestEnvironment environment; + std::filesystem::remove_all(scratch / "storage" / "fit-check"); + auto& game = environment.fake; + const auto directory = scratch / "mods" / "fit-check"; + std::filesystem::remove_all(directory); + std::filesystem::copy(std::filesystem::path(RESKATE_SOURCE_DIR) / "contrib" / "plugins" / "fit-check", directory, + std::filesystem::copy_options::recursive); + Host host(environment, Side::client); + host.apply({{"fit-check", directory, "1", check_package_files(directory)}}); + running(host, "fit-check", environment); + check(host.run_console("fitcheck test") && environment.printed("fitcheck: 10 passed, 0 failed") && !environment.printed("FAIL"), + "Fit Check's own checks pass:\n" + printed_lines(environment) + environment.all()); + check(game.binds.contains("fit-check/fitcheck"), "V is bound"); + + // V in the world does nothing; on the Board screen, with a deck browsed, it turns the board. + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.capture_starts.empty(), "Not out of the menu"); + game.menu_now = {true, false, {}}; + game.grid_now = Game::MenuGrid{"deck_b", "Deck B", "Decks", {"deck_a", "deck_b"}, 2, 2}; + game.board_now = {{"Deck", "deck_a", "Deck A"}, {"Wheels", "wheels_a", "Wheels A"}}; + game.render_settings = {"PostProcess.BloomEnable", "Render.ShadowsEnable"}; + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.capture_starts.size() == 1 && game.capture_starts[0].menu && game.capture_starts[0].left == 0.655f && + game.capture_starts[0].skip == 45 && game.capture_starts[0].height == 640, + "The board's area of the menu is captured\n" + environment.all()); + check(game.called("setting fit-check PostProcess.BloomEnable false") && !game.called("Render.ShadowsEnable"), + "Bloom is off while it turns, the shadow maps left\n" + environment.all()); + ticks(host, environment, 1); + check(game.called("turn 0.50 2"), "The right stick is held half over\n" + environment.all()); + // Its banner over the game, and its page saying what it is doing. + environment.panels_open = true; + ticks(host, environment, 10); + const auto banner = std::any_of(environment.layers.begin(), environment.layers.end(), [](const plugin_ui::Layer& layer) { + return std::any_of(layer.shapes.begin(), layer.shapes.end(), [](const auto& s) { return s.text.starts_with("RECORDING FIT CHECK"); }); + }); + check(banner, "The banner shows while it records\n" + environment.all()); + check(environment.panels.size() == 1 && environment.panels[0].title == "Fit Check" && environment.panels[0].widgets.size() > 8, + std::format("Its page is built ({} panels, {} widgets)\n", environment.panels.size(), + environment.panels.empty() ? 0 : environment.panels[0].widgets.size()) + environment.all()); + environment.panels_open = false; + + // Round in 100 frames, too few for 120 (4 s at 30): slower, and again. + game.capture_now.taken = 130; + ticks(host, environment, 1); + check(game.capture_starts.size() == 2 && game.capture_saves.empty(), "Too fast: started again\n" + environment.all()); + + // Round in 130: its 120 evenly spaced frames saved, the card with the browsed deck. + game.turning = 130; + game.capture_now.taken = 150; + ticks(host, environment, 1); + check(game.called("turn 0.46 2"), "The stick eased off to 0.5 x 0.92\n" + environment.all()); + check(game.capture_saves.size() == 1, "Saved once it came round\n" + environment.all()); + const auto& saved = game.capture_saves[0]; + check(saved.frames.size() == 120 && saved.frames[0] == 0 && saved.frames[60] == 65 && saved.format == "gif" && saved.size == 512 && + !saved.background && saved.solid, + "Evenly spaced frames of one turn, as a transparent GIF"); + check(saved.path.starts_with("fit-checks/Deck-B-") && saved.path.ends_with(".gif"), "named after the deck: " + saved.path); + check(saved.card_title == "Board" && saved.card.size() == 2 && saved.card[0].second == "Deck B" && saved.card[1].second == "Wheels A", + "The card has the browsed deck and the worn wheels"); + check(game.called("setting fit-check PostProcess.BloomEnable none") && game.turned == 0, "Bloom back and the stick let go\n" + environment.all()); + game.capture_now = {"saved", 150, 150, true, "cut out by depth", saved.path}; + ticks(host, environment, 1); + check(environment.logged("[fit-check] Saved Deck-B-") && environment.logged(".gif in the Fit Checks folder (cut out by depth)."), + "And it says where\n" + environment.all()); + + check(game.called("capture discard fit-check"), "Its pictures are let go once saved"); + game.calls.clear(); + // V on the Skaters screen with no grid: the skater; V again stops it. + game.grid_now.reset(); + game.menu_now = {true, true, {}}; + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.capture_starts.size() == 3 && game.capture_starts.back().left == 0.27f, "The skater's area\n" + environment.all()); + game.presses = {{"fit-check", "fitcheck"}}; + ticks(host, environment, 1); + check(game.called("capture discard fit-check") && game.turned == 0, "V again stops it\n" + environment.all()); + check(host.run_console("fitcheck open") && game.called("reveal fit-check fit-checks"), "The folder is shown"); +} + void plugins_drive_the_throwdown_kit() { namespace kit = dingosdk::throwdowns::kit; TestEnvironment environment; @@ -3298,6 +3444,7 @@ int main() { grind_exits_pick_the_side(); smokey_throwdowns_register_and_play_one_up(); plugins_drive_the_throwdown_kit(); + fit_check_turns_the_board_and_saves_it(); the_throwdowns_page_lists_the_stock_cards_and_plugin_modes(); plugins_add_throwdown_modes(); reskate_hostile_is_stopped_or_refused_every_time(); diff --git a/Extension/Plugins/plugin_api.cpp b/Extension/Plugins/plugin_api.cpp index f0b8b216..39b465c0 100644 --- a/Extension/Plugins/plugin_api.cpp +++ b/Extension/Plugins/plugin_api.cpp @@ -710,6 +710,7 @@ void Api::install(Plugin& plugin, lua_State* L) { install_world_api(plugin, L); install_services_api(plugin, L); install_kit_api(plugin, L); + install_capture_api(plugin, L); readonly(); lua_setglobal(L, "reskate"); lua_pushcfunction(L, ApiFunctions::print, "print"); diff --git a/Extension/Plugins/plugin_api.h b/Extension/Plugins/plugin_api.h index 90e26494..d7b0aea3 100644 --- a/Extension/Plugins/plugin_api.h +++ b/Extension/Plugins/plugin_api.h @@ -23,6 +23,9 @@ void install_services_api(Plugin& plugin, lua_State* L); // The throwdown kit's documents and native flag, the plugin's data files, overlay dialogs and the // skater's push (with its wait and reset) and hold (plugin_api_throwdowns.cpp). void install_kit_api(Plugin& plugin, lua_State* L); +// Screen captures, the game menu's preview turned and read, what the skater wears, and the data +// folder shown (plugin_api_capture.cpp). +void install_capture_api(Plugin& plugin, lua_State* L); // ---- adding to the reskate table --------------------------------------------------------------- // A new part of the API is a file of `static int name(lua_State*)` functions and an install_*_api diff --git a/Extension/Plugins/plugin_api_capture.cpp b/Extension/Plugins/plugin_api_capture.cpp new file mode 100644 index 00000000..08e138cd --- /dev/null +++ b/Extension/Plugins/plugin_api_capture.cpp @@ -0,0 +1,314 @@ +#include "plugin_api.h" +#include "plugin_game.h" +#include "lua.h" +#include "lualib.h" +#include +#include +#include + +// reskate.capture (the screen as the game draws it, saved as a GIF, APNG, PNG or WebM), the game +// menu's preview turned (reskate.room.turn) and read (reskate.menu.game, reskate.menu.grid), what the +// skater wears (reskate.outfit.worn, board, title) and reskate.files.reveal: from Fit Check (pull +// request 326), whose turn finding and card are now a plugin's (contrib/plugins/fit-check). +namespace dingosdk::plugins { +namespace { +int result(lua_State* L, const std::string& why) { + lua_pushboolean(L, why.empty()); + if (why.empty()) return 1; + lua_pushlstring(L, why.data(), why.size()); + return 2; +} +void push_text(lua_State* L, const std::string& text, const char* field) { + lua_pushlstring(L, text.data(), text.size()); + lua_setfield(L, -2, field); +} +// An option of a table at `index`: a number within [least, most] (raises otherwise), or `fallback`. +double number_option(lua_State* L, int index, const char* name, double fallback, double least, double most) { + if (lua_isnoneornil(L, index)) return fallback; + lua_getfield(L, index, name); + double value = fallback; + if (!lua_isnil(L, -1)) { + value = luaL_checknumber(L, -1); + if (!(value >= least && value <= most)) luaL_error(L, "%s is %g to %g", name, least, most); + } + lua_pop(L, 1); + return value; +} +bool flag_option(lua_State* L, int index, const char* name, bool fallback) { + if (lua_isnoneornil(L, index)) return fallback; + lua_getfield(L, index, name); + const bool value = lua_isnil(L, -1) ? fallback : lua_toboolean(L, -1) != 0; + lua_pop(L, 1); + return value; +} +std::string text_option(lua_State* L, int index, const char* name, const char* fallback) { + if (lua_isnoneornil(L, index)) return fallback; + lua_getfield(L, index, name); + std::string value = fallback; + if (!lua_isnil(L, -1)) value = luaL_checkstring(L, -1); + lua_pop(L, 1); + return value; +} +// A per-second budget; false past it. +bool within(double now, double& second, int& count, int most) { + if (now - second >= 1000) { + second = now; + count = 0; + } + return ++count <= most; +} +} // namespace + +struct CaptureApi { + static Game& game(Plugin& plugin) { return plugin.services_.game(); } + static Plugin& capturing(lua_State* L, const char* what) { + auto& plugin = plugin_of(L); + need(L, plugin, "capture", what); + return plugin; + } + + // start({area = {left, top, right, bottom}, frame = "screen"|"menu", depth, height, skip, frames, seconds}) + static int start(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.start"); + Game::CaptureStart request; + if (!lua_isnoneornil(L, 1)) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_getfield(L, 1, "area"); + if (!lua_isnil(L, -1)) { + luaL_checktype(L, -1, LUA_TTABLE); + float edges[4]{}; + for (int i = 0; i < 4; ++i) { + lua_rawgeti(L, -1, i + 1); + const double v = luaL_checknumber(L, -1); + if (!(v >= 0 && v <= 1)) luaL_error(L, "an area is {left, top, right, bottom}, each 0 to 1"); + edges[i] = static_cast(v); + lua_pop(L, 1); + } + if (!(edges[2] > edges[0] && edges[3] > edges[1])) luaL_error(L, "an area's right is past its left, and its bottom past its top"); + request.left = edges[0], request.top = edges[1], request.right = edges[2], request.bottom = edges[3]; + } + lua_pop(L, 1); + const auto frame = text_option(L, 1, "frame", "screen"); + if (frame != "screen" && frame != "menu") luaL_error(L, "frame is \"screen\" or \"menu\""); + request.menu = frame == "menu"; + request.depth = flag_option(L, 1, "depth", true); + request.height = static_cast(number_option(L, 1, "height", 0, 0, 4096)); + if (request.height && request.height < 64) luaL_error(L, "height is 0 (as on screen) or 64 to 4096"); + request.skip = static_cast(number_option(L, 1, "skip", 0, 0, 600)); + request.most = static_cast(number_option(L, 1, "frames", 2400, 1, 20000)); + request.seconds = number_option(L, 1, "seconds", 60, 1, 120); + } + if (!within(plugin.services_.now_ms(), plugin.control_second_, plugin.controls_, 4)) return result(L, "asked more than 4 times in a second"); + return result(L, game(plugin).capture_start(plugin.id(), request)); + } + static int stop(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.stop"); + game(plugin).capture_stop(plugin.id()); + return 0; + } + static int discard(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.discard"); + game(plugin).capture_discard(plugin.id()); + return 0; + } + static int state(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.state"); + const auto now = game(plugin).capture_status(plugin.id()); + lua_createtable(L, 0, 6); + push_text(L, now.state, "state"); + lua_pushinteger(L, static_cast(now.taken)); + lua_setfield(L, -2, "taken"); + lua_pushinteger(L, static_cast(now.kept)); + lua_setfield(L, -2, "kept"); + lua_pushboolean(L, now.depth); + lua_setfield(L, -2, "depth"); + if (!now.message.empty()) push_text(L, now.message, "message"); + if (!now.file.empty()) push_text(L, now.file, "file"); + return 1; + } + static std::uint32_t frame_number(lua_State* L, int index) { + const double n = luaL_checknumber(L, index); + if (!(n >= 1 && n <= 20000) || n != std::floor(n)) luaL_argerror(L, index, "a frame is a whole number from 1"); + return static_cast(n) - 1; + } + // compare(a, b): how unlike two frames taken look, 0 to 255; nil when either is not known. + static int compare(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.compare"); + const float d = game(plugin).capture_compare(plugin.id(), frame_number(L, 1), frame_number(L, 2)); + if (d < 0) lua_pushnil(L); + else lua_pushnumber(L, d); + return 1; + } + static int kept(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.kept"); + const auto list = game(plugin).capture_kept(plugin.id()); + lua_createtable(L, static_cast(list.size()), 0); + for (std::size_t i = 0; i < list.size(); ++i) { + lua_pushinteger(L, static_cast(list[i]) + 1); + lua_rawseti(L, -2, static_cast(i + 1)); + } + return 1; + } + // save(path, {frames, format, fps, size, background, cutout, solid, card = {title, lines}}) + static int save(lua_State* L) { + auto& plugin = capturing(L, "reskate.capture.save"); + need(L, plugin, "files", "reskate.capture.save"); + Game::CaptureSave request; + request.path = data_path(L, 1, false); + luaL_checktype(L, 2, LUA_TTABLE); + lua_getfield(L, 2, "frames"); + luaL_checktype(L, -1, LUA_TTABLE); + const int count = lua_objlen(L, -1); + if (count < 1 || count > 360) luaL_error(L, "a capture saves 1 to 360 frames"); + for (int i = 1; i <= count; ++i) { + lua_rawgeti(L, -1, i); + request.frames.push_back(frame_number(L, lua_gettop(L))); + lua_pop(L, 1); + } + lua_pop(L, 1); + request.format = text_option(L, 2, "format", "gif"); + if (request.format != "gif" && request.format != "apng" && request.format != "png" && request.format != "webm") + luaL_error(L, "format is \"gif\", \"apng\", \"png\" or \"webm\""); + if (request.format == "png" && count != 1) luaL_error(L, "a png is one frame"); + request.fps = static_cast(number_option(L, 2, "fps", 30, 1, 60)); + request.size = static_cast(number_option(L, 2, "size", 512, 64, 2048)); + lua_getfield(L, 2, "background"); + if (lua_isnumber(L, -1)) { + const double c = lua_tonumber(L, -1); + if (!(c >= 0 && c <= 0xFFFFFF) || c != std::floor(c)) luaL_error(L, "background is 0xRRGGBB, or \"transparent\""); + request.background = static_cast(c); + } else if (!lua_isnil(L, -1) && std::string(luaL_checkstring(L, -1)) != "transparent") + luaL_error(L, "background is 0xRRGGBB, or \"transparent\""); + lua_pop(L, 1); + request.cutout = text_option(L, 2, "cutout", "auto"); + if (request.cutout != "auto" && request.cutout != "depth" && request.cutout != "colour" && request.cutout != "none") + luaL_error(L, "cutout is \"auto\", \"depth\", \"colour\" or \"none\""); + request.solid = flag_option(L, 2, "solid", true); + lua_getfield(L, 2, "card"); + if (!lua_isnil(L, -1)) { + luaL_checktype(L, -1, LUA_TTABLE); + const int card = lua_gettop(L); + request.card_title = text_option(L, card, "title", ""); + if (request.card_title.size() > 32) luaL_error(L, "a card's title is at most 32 bytes"); + lua_getfield(L, card, "lines"); + luaL_checktype(L, -1, LUA_TTABLE); + const int lines = lua_objlen(L, -1); + if (lines > 16) luaL_error(L, "a card has at most 16 lines"); + for (int i = 1; i <= lines; ++i) { + lua_rawgeti(L, -1, i); + luaL_checktype(L, -1, LUA_TTABLE); + lua_rawgeti(L, -1, 1); + lua_rawgeti(L, -2, 2); + std::string label = luaL_checkstring(L, -2), value = luaL_checkstring(L, -1); + if (label.size() > 64 || value.size() > 64) luaL_error(L, "a card's line is {label, value}, each at most 64 bytes"); + request.card.emplace_back(std::move(label), std::move(value)); + lua_pop(L, 3); + } + lua_pop(L, 1); + } + lua_pop(L, 1); + if (!within(plugin.services_.now_ms(), plugin.files_second_, plugin.file_calls_, 64)) return result(L, "more than 64 file calls in a second"); + return result(L, game(plugin).capture_save(plugin.id(), request)); + } + + // room.turn(value, stick): the game menu's preview turned by a stand-in pad's stick, held 300 ms. + static int room_turn(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "room", "reskate.room.turn"); + const double value = luaL_checknumber(L, 1); + if (!(value >= -1 && value <= 1)) luaL_argerror(L, 1, "-1 to 1"); + const std::string stick = luaL_optstring(L, 2, "right_x"); + static constexpr const char* sticks[] = {"left_x", "left_y", "right_x", "right_y"}; + const auto at = std::find_if(std::begin(sticks), std::end(sticks), [&](const char* s) { return stick == s; }); + if (at == std::end(sticks)) luaL_argerror(L, 2, "\"left_x\", \"left_y\", \"right_x\" or \"right_y\""); + return result(L, game(plugin).room_turn(plugin.id(), static_cast(value), static_cast(at - std::begin(sticks)))); + } + static int menu_game(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "menu", "reskate.menu.game"); + const auto menu = game(plugin).game_menu(); + lua_createtable(L, 0, 3); + lua_pushboolean(L, menu.open); + lua_setfield(L, -2, "open"); + lua_pushboolean(L, menu.open && menu.skaters); + lua_setfield(L, -2, "skaters"); + lua_createtable(L, static_cast(menu.buttons.size()), 0); + for (std::size_t i = 0; i < menu.buttons.size() && i < 64; ++i) { + lua_pushlstring(L, menu.buttons[i].data(), menu.buttons[i].size()); + lua_rawseti(L, -2, static_cast(i + 1)); + } + lua_setfield(L, -2, "buttons"); + return 1; + } + static int menu_grid(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "menu", "reskate.menu.grid"); + const auto grid = game(plugin).menu_grid(); + if (!grid) { + lua_pushnil(L); + return 1; + } + lua_createtable(L, 0, 6); + if (!grid->focused.empty()) push_text(L, grid->focused, "focused"); + if (!grid->title.empty()) push_text(L, grid->title, "title"); + if (!grid->category.empty()) push_text(L, grid->category, "category"); + lua_createtable(L, static_cast(std::min(grid->items.size(), 4000)), 0); + for (std::size_t i = 0; i < grid->items.size() && i < 4000; ++i) { + lua_pushlstring(L, grid->items[i].data(), grid->items[i].size()); + lua_rawseti(L, -2, static_cast(i + 1)); + } + lua_setfield(L, -2, "items"); + lua_pushinteger(L, static_cast(grid->shown)); + lua_setfield(L, -2, "shown"); + lua_pushinteger(L, static_cast(grid->total)); + lua_setfield(L, -2, "total"); + return 1; + } + static int push_worn(lua_State* L, const std::vector& list) { + lua_createtable(L, static_cast(list.size()), 0); + for (std::size_t i = 0; i < list.size(); ++i) { + lua_createtable(L, 0, 3); + push_text(L, list[i].slot, "slot"); + push_text(L, list[i].item, "item"); + push_text(L, list[i].title, "title"); + lua_rawseti(L, -2, static_cast(i + 1)); + } + return 1; + } + static int outfit_worn(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "outfit", "reskate.outfit.worn"); + return push_worn(L, game(plugin).outfit_worn()); + } + static int outfit_board(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "outfit", "reskate.outfit.board"); + return push_worn(L, game(plugin).board_parts()); + } + static int outfit_title(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "outfit", "reskate.outfit.title"); + const auto key = checked_text(L, 1, 255, "an item's key"); + const auto title = game(plugin).item_title(key); + lua_pushlstring(L, title.data(), title.size()); + return 1; + } + static int files_reveal(lua_State* L) { + auto& plugin = plugin_of(L); + need(L, plugin, "files", "reskate.files.reveal"); + const auto folder = data_path(L, 1, true); + if (!within(plugin.services_.now_ms(), plugin.reveal_second_, plugin.reveals_, 1)) return result(L, "once a second"); + return result(L, game(plugin).files_reveal(plugin.id(), folder)); + } + static void install(lua_State* L) { + add_functions(L, "capture", {{"start", start}, {"stop", stop}, {"discard", discard}, {"state", state}, {"compare", compare}, + {"kept", kept}, {"save", save}}); + add_functions(L, "room", {{"turn", room_turn}}); + add_functions(L, "menu", {{"game", menu_game}, {"grid", menu_grid}}); + add_functions(L, "outfit", {{"worn", outfit_worn}, {"board", outfit_board}, {"title", outfit_title}}); + add_functions(L, "files", {{"reveal", files_reveal}}); + } +}; + +void install_capture_api(Plugin&, lua_State* L) { CaptureApi::install(L); } +} // namespace dingosdk::plugins diff --git a/Extension/Plugins/plugin_game.h b/Extension/Plugins/plugin_game.h index 8df8e719..a76cca0e 100644 --- a/Extension/Plugins/plugin_game.h +++ b/Extension/Plugins/plugin_game.h @@ -378,6 +378,58 @@ public: std::string state, why; }; virtual RoomState room_state(std::string_view /*plugin*/) { return {"idle", {}}; } + // The game menu's preview (the skater on the Skaters page, the board on a Board tab) turned by a + // stick of a stand-in pad: `value` -1..1 of the stick held for 300 ms (call it every tick), 0 lets + // go; `stick` 0 left x, 1 left y, 2 right x, 3 right y. Only while the game's menu is open. + virtual std::string room_turn(std::string_view /*plugin*/, float /*value*/, int /*stick*/) { return unavailable; } + // The game's menu: whether it is open, whether it shows the Skaters page, and the navigations of + // the open page's buttons (how the game names where each goes: "CASPresetPurchase", ...). + struct GameMenu { + bool open{}, skaters{}; + std::vector buttons; + }; + virtual GameMenu game_menu() { return {}; } + // The customization grid that has the focus (Extension/Customization/item_browser.h), when one does. + struct MenuGrid { + std::string focused, title, category; // the highlighted item: key, title, catalog category + std::vector items; // every item's key in the grid + unsigned shown{}, total{}; + }; + virtual std::optional menu_grid() { return std::nullopt; } + // ---- capture: the screen as the game draws it (Extension/Rendering/Capture) --------------------- + // One plugin records at a time; frames are numbered from 0 as they are taken. + struct CaptureStart { + float left{}, top{}, right{1}, bottom{1}; + bool menu{}; // fractions of the game's menu frame (at most 21:9), else of the screen + bool depth{true}; + std::uint32_t height{}, skip{}, most{2400}; + double seconds{60}; + }; + virtual std::string capture_start(std::string_view /*plugin*/, const CaptureStart&) { return unavailable; } + virtual void capture_stop(std::string_view /*plugin*/) {} + virtual void capture_discard(std::string_view /*plugin*/) {} + struct CaptureStatus { + std::string state; // "idle", "recording", "stopped", "saving", "saved", "failed" + std::uint32_t taken{}, kept{}; + bool depth{}; + std::string message, file; // file: its path in the plugin's data folder + }; + // Another plugin's capture reads as idle. + virtual CaptureStatus capture_status(std::string_view /*plugin*/) { return {"idle"}; } + virtual float capture_compare(std::string_view /*plugin*/, std::uint32_t, std::uint32_t) { return -1; } + virtual std::vector capture_kept(std::string_view /*plugin*/) { return {}; } + struct CaptureSave { + std::vector frames; + std::string format{"gif"}; // "gif", "apng", "png", "webm" + std::uint32_t fps{30}, size{512}; + std::optional background; // 0xRRGGBB; none: transparent + std::string cutout{"auto"}; // "auto", "depth", "colour", "none" + bool solid{true}; + std::string card_title; + std::vector> card; // label, value + std::string path; // in the plugin's data folder + }; + virtual std::string capture_save(std::string_view /*plugin*/, const CaptureSave&) { return unavailable; } // Game buttons kept from the game while asked (overlay::hide_game_buttons/hold_game_buttons), // and all game input paused for half a second at a time. virtual void game_buttons(std::string_view /*plugin*/, std::uint16_t /*hidden*/, std::uint16_t /*held*/, bool /*paused*/) {} @@ -440,6 +492,8 @@ public: virtual std::string files_read(std::string_view /*plugin*/, std::string_view /*path*/, std::string& /*text*/) { return unavailable; } virtual std::string files_write(std::string_view /*plugin*/, std::string_view /*path*/, std::string_view /*text*/) { return unavailable; } virtual std::string files_remove(std::string_view /*plugin*/, std::string_view /*path*/) { return unavailable; } + // The data folder (or a folder in it) opened in the player's file manager. + virtual std::string files_reveal(std::string_view /*plugin*/, std::string_view /*folder*/) { return unavailable; } // The same, for a file up to `most` bytes (an animation clip's, say). virtual std::string files_read_bytes(std::string_view plugin, std::string_view path, std::string& out, std::size_t /*most*/) { return files_read(plugin, path, out); @@ -605,6 +659,17 @@ public: virtual std::string outfit_find(std::string_view) { return {}; } virtual std::vector outfit_slots() { return {}; } virtual std::string outfit_status() { return unavailable; } + // What the skater wears now (the selected preset): `slot` the kind of item ("Headwear", "Top", ..., + // "Shoes", in the order worn from head to foot; clothes and accessories only), or the board's part + // ("Deck", "Grip tape", "Grip cutout", "Trucks", "Wheels"). + struct WornItem { + std::string slot, item, title; + }; + virtual std::vector outfit_worn() { return {}; } + virtual std::vector board_parts() { return {}; } + // An item's title as the game shows it (its catalog's, the one an item grid showed, or its key + // made readable). + virtual std::string item_title(std::string_view key) { return std::string(key); } // ---- board: what is drawn of the local skateboard (Extension/Skater/board_pose.h) ------------------ struct BoardBone { Vec3 position{}; diff --git a/Extension/Plugins/plugin_game_runtime.cpp b/Extension/Plugins/plugin_game_runtime.cpp index bb93335b..186a7c83 100644 --- a/Extension/Plugins/plugin_game_runtime.cpp +++ b/Extension/Plugins/plugin_game_runtime.cpp @@ -61,6 +61,11 @@ #include "Extension/UI/Overlay/screen_look.h" #include "Extension/UI/Overlay/scripted_pad.h" #include "Extension/World/character_room.h" +#include "Extension/Rendering/Capture/capture_runtime.h" +#include "Extension/Customization/item_browser.h" +#include "Extension/Customization/worn_items.h" +#include "Extension/Multiplayer/Hud/game_ui_state.h" +#include #include #include #include @@ -1102,6 +1107,100 @@ public: skater_animation::show_in_room(false); character_room::leave(); } + std::string room_turn(std::string_view, float value, int stick) override { + if (value != 0 && !multiplayer::sample_game_ui_state(base_).in_menu) return "the game's menu is not open"; + overlay::hold_stick(stick, value, 300); + return {}; + } + GameMenu game_menu() override { + GameMenu menu; + menu.open = multiplayer::sample_game_ui_state(base_).in_menu; + if (!menu.open) return menu; + const auto buttons = character_room::menu_buttons(base_); + menu.skaters = character_room::is_skaters_page(buttons); + std::size_t start = 0; + while (start < buttons.size() && menu.buttons.size() < 64) { + const auto end = std::min(buttons.find(' ', start), buttons.size()); + if (end > start) menu.buttons.push_back(buttons.substr(start, end - start)); + start = end + 1; + } + return menu; + } + std::optional menu_grid() override { + const auto view = item_browser::view(); + if (!view.open) return std::nullopt; + return MenuGrid{view.focused_key, view.focused_title, view.focused_category, view.grid_keys, view.shown, view.total}; + } + // ---- capture: one plugin records at a time (capture_runtime.h) -------------------------------- + std::string capture_start(std::string_view plugin, const CaptureStart& request) override { + auto& recorder = capture::recorder(); + const auto now = recorder.status().state; + const bool busy = now == capture::State::recording || now == capture::State::saving; + if (busy && !capture_owner_.empty() && capture_owner_ != plugin) return capture_owner_ + " is capturing"; + capture::RecordRequest r; + r.area = {request.left, request.top, request.right, request.bottom, request.menu}; + r.depth = request.depth; + r.height = request.height; + r.skip = request.skip; + r.most = request.most; + r.seconds = request.seconds; + std::string why; + if (!recorder.start(r, why)) return why; + capture_owner_ = plugin; + capture_file_.clear(); + return {}; + } + void capture_stop(std::string_view plugin) override { + if (capture_owner_ == plugin) capture::recorder().stop(); + } + void capture_discard(std::string_view plugin) override { + // While it saves, the save goes on and is still this plugin's to read. + if (capture_owner_ != plugin || capture::recorder().status().state == capture::State::saving) return; + capture::recorder().discard(); + capture_owner_.clear(); + } + CaptureStatus capture_status(std::string_view plugin) override { + if (capture_owner_ != plugin) return {"idle"}; + static constexpr const char* names[] = {"idle", "recording", "stopped", "saving", "saved", "failed"}; + const auto s = capture::recorder().status(); + CaptureStatus out{names[static_cast(s.state)], s.taken, s.kept, s.depth, s.message, {}}; + if (s.state == capture::State::saved) out.file = capture_file_; + return out; + } + float capture_compare(std::string_view plugin, std::uint32_t a, std::uint32_t b) override { + return capture_owner_ == plugin ? capture::recorder().compare(a, b) : -1.0f; + } + std::vector capture_kept(std::string_view plugin) override { + return capture_owner_ == plugin ? capture::recorder().kept() : std::vector{}; + } + std::string capture_save(std::string_view plugin, const CaptureSave& request) override { + if (capture_owner_ != plugin) return "nothing was recorded"; + std::filesystem::path root, file; + if (auto why = data_file(plugin, request.path, root, file); !why.empty()) return why; + std::error_code error; + if (std::filesystem::is_directory(file, error)) return "a folder is there"; + capture::SaveRequest r; + r.frames = request.frames; + r.format = request.format == "apng" ? capture::Format::apng : request.format == "png" ? capture::Format::png + : request.format == "webm" ? capture::Format::webm : capture::Format::gif; + r.fps = request.fps; + r.size = request.size; + if (request.background) + r.background = capture::Rgb{static_cast(*request.background >> 16), static_cast(*request.background >> 8), + static_cast(*request.background)}; + r.cutout = request.cutout == "depth" ? capture::Cutout::depth : request.cutout == "colour" ? capture::Cutout::colour + : request.cutout == "none" ? capture::Cutout::none : capture::Cutout::automatic; + r.solid = request.solid; + r.card_title = request.card_title; + for (const auto& [label, value] : request.card) r.card.push_back({label, value}); + r.fonts = capture::menu_fonts(); + r.file = file; + std::string why; + capture::recorder().stop(); + if (!capture::recorder().save(std::move(r), why)) return why; + capture_file_ = request.path; + return {}; + } RoomState room_state(std::string_view plugin) override { using P = character_room::Phase; const auto phase = character_room::phase(); @@ -1376,7 +1475,31 @@ public: return why; } bool frozen() const noexcept { return !freezers_.empty(); } + std::vector outfit_worn() override { + std::vector out; + for (auto& w : worn_items::worn_outfit()) out.push_back({std::move(w.slot), std::move(w.item), std::move(w.title)}); + return out; + } + std::vector board_parts() override { + std::vector out; + for (auto& w : worn_items::worn_board()) out.push_back({std::move(w.slot), std::move(w.item), std::move(w.title)}); + return out; + } + std::string item_title(std::string_view key) override { return worn_items::item_title(std::string(key)); } + std::string files_reveal(std::string_view plugin, std::string_view folder) override { + std::filesystem::path root, at; + if (auto why = data_file(plugin, folder, root, at); !why.empty()) return why; + std::error_code error; + std::filesystem::create_directories(at, error); + if (error || !std::filesystem::is_directory(at, error)) return "not a folder"; + const auto done = reinterpret_cast(ShellExecuteW(nullptr, L"open", at.c_str(), nullptr, nullptr, SW_SHOWNORMAL)); + return done > 32 ? std::string{} : std::string("the folder could not be opened"); + } void release(std::string_view plugin) override { + if (capture_owner_ == plugin) { + capture::recorder().discard(); + capture_owner_.clear(); + } if (hold_owner_ == plugin) (void)skater_hold(plugin, false); plugin_feed::release_hidden(std::string(plugin)); world_entities::clear(plugin); @@ -1564,6 +1687,7 @@ private: } std::uintptr_t base_; + std::string capture_owner_, capture_file_; // the plugin whose capture it is, and where it saved to std::string camera_holder_, follower_, camera_status_; std::array camera_pose_{}; float camera_fov_{}; diff --git a/Extension/Plugins/plugin_manifest.cpp b/Extension/Plugins/plugin_manifest.cpp index 4d47b9fc..914fb012 100644 --- a/Extension/Plugins/plugin_manifest.cpp +++ b/Extension/Plugins/plugin_manifest.cpp @@ -15,13 +15,13 @@ namespace { // throwdown kit (modes, documents, the native flag); "variants" choosing which of its own mod's // variants is in the game (reskate.package.choose). A manifest's "resident" list and "variants" are // the merge's (Engine/Vfs/mod_list.h ResidentAssets, Variants), not the plugin host's. -constexpr std::array permissions{ +constexpr std::array permissions{ "console", "storage", "overlay", "menu", "ui.screen", "input", "hud", "events.tricks", "events.skater", "skater.read", "skater.teleport", "skater.control", "skater.animation", "trainer", "camera", "world.raycast", "world.objects", "world.entities", "world.environment", "world.ai", "travel", "session.read", "session.skaters", "session.nametags", "net.session", "net.http", "voice.control", "chat.read", "chat.send", "chat.local", "server", "throwdowns", "replay", "music", "progression.read", "settings.read", "audio.ui", "audio", "outfit", "board", "mods.read", - "events.physics", "game.speed", "files", "variants", "settings.write", "room"}; + "events.physics", "game.speed", "files", "variants", "settings.write", "room", "capture"}; } bool known_permission(std::string_view name) noexcept { diff --git a/Extension/Plugins/plugin_vm.h b/Extension/Plugins/plugin_vm.h index 5a52a752..12bef132 100644 --- a/Extension/Plugins/plugin_vm.h +++ b/Extension/Plugins/plugin_vm.h @@ -263,6 +263,7 @@ private: friend struct WorldApi; friend struct ServicesApi; friend struct KitApi; + friend struct CaptureApi; struct Command { std::string help; int function{}; }; struct Timer { double due{}, every{}; int function{}; }; using Clock = std::chrono::steady_clock; @@ -359,8 +360,8 @@ private: int controls_{}, edits_{}, environments_{}, voices_{}, musics_{}; double travel_at_{-1e9}, chat_at_{-1e9}, local_chat_second_{}, server_second_{}, host_at_{-1e9}, wipeout_at_{-1e9}; int local_chats_{}, server_calls_{}; - double files_second_{}, dialog_second_{}; - int file_calls_{}, dialog_calls_{}; + double files_second_{}, dialog_second_{}, reveal_second_{}; + int file_calls_{}, dialog_calls_{}, reveals_{}; std::optional> trainer_point_; std::set> binds_; std::vector screens_; diff --git a/Extension/Rendering/Capture/Test/capture_tests.cpp b/Extension/Rendering/Capture/Test/capture_tests.cpp new file mode 100644 index 00000000..171df409 --- /dev/null +++ b/Extension/Rendering/Capture/Test/capture_tests.cpp @@ -0,0 +1,438 @@ +#include "Extension/Rendering/Capture/capture_encode.h" +#include "Extension/Rendering/Capture/capture_matte.h" +#include "Extension/Rendering/Capture/capture_panel.h" +#include "Extension/Rendering/Capture/capture_recorder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Captures without the game (reskate.capture): matting, resizing, the card, the file formats, and a +// recording of a turning board or skater found, cut out and saved. +// Run with the folder of the menu fonts (External/fonts) as the first argument to test the card too, +// and a folder as the second to save a picture of it there. +namespace { +using namespace dingosdk::capture; +int failures = 0; + +void expect(bool condition, const char* what) { + if (condition) return; + ++failures; + std::printf("FAIL: %s\n", what); +} + +std::uint32_t be32(const std::uint8_t* p) { return std::uint32_t{p[0]} << 24 | std::uint32_t{p[1]} << 16 | std::uint32_t{p[2]} << 8 | p[3]; } + +// The chunk types of a PNG, in order. +std::vector chunks(const std::vector& png) { + std::vector out; + for (std::size_t at = 8; at + 12 <= png.size();) { + const auto size = be32(png.data() + at); + out.emplace_back(reinterpret_cast(png.data() + at + 4), 4); + at += 12 + size; + } + return out; +} + +// A board-ish test picture: an opaque rounded bar with a soft edge, a see-through middle band and a +// fully transparent background. +Image board(std::uint32_t size, float turn) { + Image image(size, size); + const auto n = static_cast(size); + const float c = n / 2.0f, half_w = n * 0.18f * std::abs(std::cos(turn)) + 2, half_h = n * 0.42f; + for (std::uint32_t y = 0; y < size; ++y) + for (std::uint32_t x = 0; x < size; ++x) { + const float fx = static_cast(x) + 0.5f, fy = static_cast(y) + 0.5f; + const float dx = std::abs(fx - c) - half_w, dy = std::abs(fy - c) - half_h; + const float edge = std::clamp(0.5f - std::max(dx, dy), 0.0f, 1.0f); + auto* p = image.at(x, y); + p[0] = static_cast(40 + y * 200 / size); + p[1] = 140; + p[2] = static_cast(220 - x * 100 / size); + const bool band = std::abs(fy - c) < n * 0.05f; + p[3] = static_cast(std::lround(edge * (band ? 120 : 255))); + } + return image; +} + +// `image` over a background, then "graded" the way the game's post-processing shifts both captures. +std::vector composite(const Image& image, std::array background) { + std::vector out(image.rgba.size()); + for (std::size_t i = 0; i < image.rgba.size(); i += 4) { + const float a = image.rgba[i + 3] / 255.0f; + for (int c = 0; c < 3; ++c) { + const float v = image.rgba[i + c] * a + background[c] * (1 - a); + out[i + 2 - c] = static_cast(std::lround(std::clamp(v * 0.9f + 6.0f, 0.0f, 255.0f))); // BGRA + } + out[i + 3] = 255; + } + return out; +} + +void test_matte() { + const auto truth = board(96, 0.3f); + const auto dark = composite(truth, {10, 12, 14}), light = composite(truth, {235, 236, 240}); + const PixelView d{dark.data(), 96, 96, 96 * 4, true}, l{light.data(), 96, 96, 96 * 4, true}; + const Rect all{0, 0, 96, 96}; + const auto backgrounds = measure_backgrounds(d, l, all); + expect(std::abs(backgrounds.dark[0] - (10 * 0.9f + 6)) < 1 && std::abs(backgrounds.light[2] - (240 * 0.9f + 6)) < 1, + "backgrounds are measured from the edge"); + const auto cut = matte_pair(d, l, all, backgrounds, 0.0f); + int worst_alpha = 0, worst_colour = 0; + for (std::uint32_t y = 0; y < 96; ++y) + for (std::uint32_t x = 0; x < 96; ++x) { + const auto* want = truth.at(x, y); + const auto* got = cut.at(x, y); + worst_alpha = std::max(worst_alpha, std::abs(want[3] - got[3])); + if (want[3] > 60) + for (int c = 0; c < 3; ++c) + worst_colour = std::max(worst_colour, std::abs(static_cast(want[c] * 0.9f + 6) - got[c])); + } + expect(worst_alpha <= 3, "matting recovers alpha, soft edges and the see-through band included"); + expect(worst_colour <= 6, "matting recovers the board's colour without the background in it"); + expect(content_bounds(cut).width < 96, "the cut-out has transparent space around the board"); + + const auto same = matte_pair(d, d, all, measure_backgrounds(d, d, all)); + expect(same.at(0, 0)[3] == 255, "identical captures are taken as opaque, not empty"); +} + +// A whole turn of a board with a black grip over a dark radial vignette, the Board screen's worst +// case: where the board has been, the backdrop must come back see-through, not as a dark ghost. +void test_backdrop_key(bool light_both_sides) { + constexpr std::uint32_t w = 300, h = 500, turn_frames = 120; + std::vector frames, truth; + for (std::uint32_t f = 0; f < turn_frames; ++f) { + const float angle = 6.2831853f * static_cast(f) / turn_frames; + const float half_w = 55 * std::abs(std::cos(angle)) + 3, half_h = 200; + const bool grip = std::cos(angle) < 0; + Image frame(w, h), want(w, h); + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const float fx = static_cast(x) + 0.5f, fy = static_cast(y) + 0.5f; + const float rx = (fx - 120) / 300, ry = (fy - 230) / 400; + // A vignette no simple surface fits: a soft spot, a kink and a slow ripple. + const float bg = 24 + 30 * std::exp(-(rx * rx + ry * ry) * 5) + 8 * std::max(0.0f, 0.4f - std::abs(rx)) + + 5 * std::sin(fy / 37); + const float dx = std::abs(fx - 150) - half_w, dy = std::abs(fy - 250) - half_h; + const float cover = std::clamp(0.5f - std::max(dx, dy), 0.0f, 1.0f); + // A white deck with a black grip; or one light on both sides (a white grip), which covers + // the middle for most of the turn: there the backdrop is seen least. + const float back = light_both_sides ? 200.0f : 12.0f; + const float deck[3] = {grip ? back : 226.0f, grip ? back : 220.0f, grip ? back + 1 : 208.0f}; + auto* p = frame.at(x, y); + for (int c = 0; c < 3; ++c) p[c] = static_cast(std::lround(deck[c] * cover + (bg + static_cast(c)) * (1 - cover))); + p[3] = 255; + want.at(x, y)[3] = static_cast(std::lround(cover * 255)); + } + frames.push_back(std::move(frame)); + truth.push_back(std::move(want)); + } + key_against_backdrop(frames); + std::uint64_t ghost = 0, lost = 0, background = 0, board = 0; + for (std::uint32_t f = 0; f < turn_frames; ++f) + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto want = truth[f].at(x, y)[3], got = frames[f].at(x, y)[3]; + if (want == 0) ++background, ghost += got > 64; + if (want == 255) ++board, lost += got < 192; + } + std::printf("backdrop key (%s): %.3f%% ghost, %.3f%% lost\n", light_both_sides ? "light both sides" : "black grip", 100.0 * static_cast(ghost) / static_cast(background), + 100.0 * static_cast(lost) / static_cast(board)); + expect(ghost * 1000 < background, "where the board has been comes back see-through"); + expect(lost * 100 < board, "a black grip on the dark backdrop stays board"); +} + +void test_geometry() { + const auto square = padded_square({10, 20, 30, 50}, 5, 200, 100); + expect(square.width == 60 && square.height == 60, "padded_square grows and squares"); + expect(square.x + square.width <= 200 && square.y + square.height <= 100, "padded_square stays inside"); + const auto u = united({0, 0, 10, 10}, {20, 5, 5, 20}); + expect(u.x == 0 && u.y == 0 && u.width == 25 && u.height == 25, "united covers both"); + const auto small = fitted(board(200, 0), 50); + expect(small.width == 50 && small.height == 50, "fitted keeps a square square"); + expect(small.at(25, 2)[3] == 0 && small.at(25, 15)[3] > 200, "fitted keeps the background clear and the board solid"); + // Premultiplied filtering: an opaque red pixel next to transparent green does not turn olive. + Image edge(2, 1); + edge.rgba = {255, 0, 0, 255, 0, 255, 0, 0}; + const auto halved = fitted(edge, 1); + expect(halved.at(0, 0)[0] == 255 && halved.at(0, 0)[1] == 0 && halved.at(0, 0)[3] == 128, "fitted does not bleed transparent colour"); + const auto flat = over_matte(halved, {0, 0, 255}); + expect(flat.at(0, 0)[3] == 255 && flat.at(0, 0)[0] == 128 && flat.at(0, 0)[2] == 127, "over_matte flattens"); +} + +void test_png() { + Animation animation; + animation.fps = 24; + for (int i = 0; i < 6; ++i) animation.frames.push_back(board(64, static_cast(i) * 0.5f)); + const auto apng = encode_apng(animation); + expect(apng.size() > 8 && std::memcmp(apng.data(), "\x89PNG\r\n\x1a\n", 8) == 0, "APNG signature"); + const auto types = chunks(apng); + const auto count = [&](const char* t) { return std::count(types.begin(), types.end(), std::string(t)); }; + expect(types.front() == "IHDR" && types[1] == "acTL" && types.back() == "IEND", "APNG chunk order"); + expect(count("fcTL") == 6 && count("IDAT") == 1 && count("fdAT") == 5, "APNG has one control and data chunk per frame"); + const auto still = encode_png(animation.frames.front()); + expect(chunks(still).size() == 3, "PNG is IHDR, IDAT, IEND"); +} + +// Decodes a GIF's first frame's LZW data back to indices, to check the encoder's code-size steps. +std::vector first_frame_indices(const std::vector& gif, std::size_t pixels) { + std::size_t at = 13 + 768; + while (at < gif.size() && gif[at] == 0x21) { // extensions + at += 2; + while (gif[at]) at += gif[at] + 1u; + ++at; + } + at += 10; // image descriptor + const int min_bits = gif[at++]; + std::vector data; + while (gif[at]) { + data.insert(data.end(), gif.begin() + static_cast(at + 1), gif.begin() + static_cast(at + 1 + gif[at])); + at += gif[at] + 1u; + } + const int clear = 1 << min_bits, end = clear + 1; + std::vector> table; + const auto reset = [&] { + table.assign(static_cast(end) + 1, {}); + for (int i = 0; i < clear; ++i) table[static_cast(i)] = {static_cast(i)}; + }; + reset(); + int bits = min_bits + 1, previous = -1; + std::size_t bit = 0; + std::vector out; + while (bit + static_cast(bits) <= data.size() * 8 && out.size() < pixels) { + int code = 0; + for (int k = 0; k < bits; ++k, ++bit) code |= ((data[bit / 8] >> (bit % 8)) & 1) << k; + if (code == clear) { reset(), bits = min_bits + 1, previous = -1; continue; } + if (code == end) break; + std::vector entry; + if (code < static_cast(table.size())) entry = table[static_cast(code)]; + else if (previous >= 0) entry = table[static_cast(previous)], entry.push_back(table[static_cast(previous)][0]); + else return {}; + out.insert(out.end(), entry.begin(), entry.end()); + if (previous >= 0 && table.size() < 4096) { + auto added = table[static_cast(previous)]; + added.push_back(entry[0]); + table.push_back(std::move(added)); + } + if (static_cast(table.size()) >= (1 << bits) && bits < 12) ++bits; + previous = code; + } + return out; +} + +void test_gif() { + // Noise fills the LZW table several times over, so clears and every code width get used. + Image noise(300, 200); + std::uint32_t seed = 7; + for (std::size_t i = 0; i < noise.rgba.size(); i += 4) { + seed = seed * 1664525u + 1013904223u; + noise.rgba[i] = static_cast(seed >> 24), noise.rgba[i + 1] = static_cast(seed >> 16); + noise.rgba[i + 2] = static_cast(seed >> 8), noise.rgba[i + 3] = (seed >> 7) & 1 ? 255 : 0; + } + Animation animation; + animation.frames = {noise, noise}; + const auto gif = encode_gif(animation, {.fringe = {0, 0, 0}, .dither = false}); + expect(std::memcmp(gif.data(), "GIF89a", 6) == 0 && gif.back() == 0x3b, "GIF header and trailer"); + const auto indices = first_frame_indices(gif, 300 * 200); + expect(indices.size() == 300 * 200, "GIF LZW data decodes to every pixel"); + bool transparent_right = indices.size() == 300 * 200; + for (std::size_t i = 0; transparent_right && i < indices.size(); ++i) + transparent_right = (indices[i] == 255) == (noise.rgba[i * 4 + 3] < 128); + expect(transparent_right, "GIF cuts out exactly the transparent pixels"); + + Animation timing; + timing.fps = 30; + timing.frames.assign(3, board(16, 0)); + const auto timed = encode_gif(timing); + int total = 0; + for (std::size_t i = 0; i + 5 < timed.size(); ++i) + if (timed[i] == 0x21 && timed[i + 1] == 0xf9) total += timed[i + 4] | timed[i + 5] << 8; + expect(total == 10, "GIF delays add up to the clip's length"); +} + +void test_webm() { + Animation animation; + animation.fps = 30; + for (int i = 0; i < 4; ++i) animation.frames.push_back(board(48, static_cast(i) * 0.4f)); + const auto webm = encode_webm(animation); + const auto has = [&](std::initializer_list bytes) { + return std::search(webm.begin(), webm.end(), bytes.begin(), bytes.end()) != webm.end(); + }; + expect(webm.size() > 4 && webm[0] == 0x1a && webm[1] == 0x45 && webm[2] == 0xdf && webm[3] == 0xa3, "WebM starts with an EBML header"); + expect(has({'V', '_', 'V', 'P', '8'}), "WebM track is VP8"); + expect(has({0x53, 0xc0}), "a see-through WebM marks its track AlphaMode"); + expect(has({0x9d, 0x01, 0x2a}), "WebM frames are VP8 key frames"); + Animation opaque; + opaque.frames = {over_matte(board(48, 0), {0, 0, 0})}; + const auto flat = encode_webm(opaque); + const std::array alpha_mode{0x53, 0xc0}; + expect(std::search(flat.begin(), flat.end(), alpha_mode.begin(), alpha_mode.end()) == flat.end(), + "an opaque WebM has no alpha"); +} + +void test_panel(const std::filesystem::path& fonts, const char* out) { + const auto load = [&](const char* name) { + std::ifstream in(fonts / name, std::ios::binary); + return std::vector(std::istreambuf_iterator(in), {}); + }; + const auto brush = load("PermanentMarker-Regular.ttf"), heading = load("Montserrat-ExtraBold.ttf"), body = load("Montserrat-SemiBold.ttf"); + const std::vector parts{{"Deck", "ReSkate Scrolling Pattern"}, + {"Grip tape", "A grip tape with a name long enough that it has to wrap onto a second line and then some more"}, + {"Trucks", "Silver"}, {"Wheels", ""}}; + const auto card = render_parts_panel("Board", parts, 512, {heading, body, brush}); + expect(card.height == 512 && card.width > 300, "the parts card is the height asked for"); + expect(card.at(0, 0)[3] < 40 && card.at(card.width / 2, card.height - 4)[3] > 200, "the card has rounded corners and a solid body"); + int bright = 0; + for (std::size_t i = 0; i < card.rgba.size(); i += 4) bright += card.rgba[i] > 200; + expect(bright > 500, "the card has text on it"); + const auto shown = side_by_side(fitted(board(300, 0.2f), 512), card); + expect(shown.width == 512 + card.width, "side_by_side lays the card beside the board"); + if (out) { // a folder to save the card in, to look at it + const auto png = encode_png(shown); + std::ofstream(std::filesystem::path(out) / "parts_card.png", std::ios::binary).write(reinterpret_cast(png.data()), static_cast(png.size())); + } +} +} // namespace + +// A stand-in for the game: a 1280x720 frame with a board that turns a little every frame on the right +// of the menu (or a skater in the middle, swaying as they stand), a piece of UI that never moves, and +// the game's depth when asked for. +struct FakeGame { + double turn{}, spin{1.0 / 37.3}; + bool skater{}, with_depth{}; + unsigned drawn{}; + std::vector last_; + std::vector frame() { + turn += spin; + ++drawn; + auto now = draw(); + // As the game's temporal smoothing does, a frame keeps a trace of the last one. + if (!last_.empty()) + for (std::size_t i = 0; i < now.size(); ++i) now[i] = static_cast((now[i] * 3 + last_[i] + 2) / 4); + last_ = now; + return now; + } + float centre_x() const { return 1280 * (skater ? 0.5f : 0.76f) + (skater ? 3.0f * std::sin(static_cast(drawn) * 0.21f) : 0.0f); } + float half_width() const { return 60 * std::abs(std::cos(static_cast(turn) * 6.2831853f)) + 3; } + std::vector draw() const { + constexpr std::uint32_t w = 1280, h = 720; + std::vector out(std::size_t{w} * h * 4); + const float cx = centre_x(), cy = 720 * 0.57f, half_w = half_width(), half_h = 230; + const bool back = std::cos(static_cast(turn) * 6.2831853f) < 0; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const float fx = static_cast(x) + 0.5f, fy = static_cast(y) + 0.5f; + const float bg = 28.0f + 10.0f * fy / h; + float rgb[3] = {bg, bg, bg + 4}; + if (x > 860 && x < 900 && y > 140 && y < 170) rgb[0] = rgb[1] = rgb[2] = 200; // UI that never moves + const float dx = std::abs(fx - cx) - half_w, dy = std::abs(fy - cy) - half_h; + const float cover = std::clamp(0.5f - std::max(dx, dy), 0.0f, 1.0f); + const float deck[3] = {back ? 12.0f : 40.0f, back ? 12.0f : 150.0f + 80 * fy / h, back ? 13.0f : 220.0f}; + for (int c = 0; c < 3; ++c) rgb[c] = deck[c] * cover + rgb[c] * (1 - cover); + auto* p = out.data() + (std::size_t{y} * w + x) * 4; + for (int c = 0; c < 3; ++c) p[2 - c] = static_cast(std::clamp(rgb[c] * 0.92f + 5, 0.0f, 255.0f)); + p[3] = 255; + } + return out; + } + // The game's depth for the frame drawn last: the board a quarter as far as the backdrop. + std::vector depth() const { + constexpr std::uint32_t w = 1280, h = 720; + const float cx = centre_x(), cy = 720 * 0.57f, half_w = half_width(), half_h = 230; + std::vector out(std::size_t{w} * h, 1.0f / 400); + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) + if (std::abs(static_cast(x) + 0.5f - cx) < half_w && std::abs(static_cast(y) + 0.5f - cy) < half_h) + out[std::size_t{y} * w + x] = 1.0f / 100; + return out; + } +}; + +// A recording of the board (or skater) turning, as a plugin makes one: its frames compared to find +// the turn, the frames evenly spaced through it saved. +void test_recording(const std::filesystem::path& fonts, Format format, bool skater, bool with_depth, std::uint32_t frames) { + const auto load = [&](const char* name) { + std::ifstream in(fonts / name, std::ios::binary); + return std::vector(std::istreambuf_iterator(in), {}); + }; + static const auto brush = load("PermanentMarker-Regular.ttf"), heading = load("Montserrat-ExtraBold.ttf"), + body = load("Montserrat-SemiBold.ttf"); + FakeGame game; + game.skater = skater; + game.with_depth = with_depth; + Recorder recorder; + RecordRequest request; + request.area = skater ? Area{0.27f, 0.14f, 0.735f, 0.985f, true} : Area{0.655f, 0.185f, 0.86f, 0.985f, true}; + request.depth = with_depth; + request.height = 480; + request.skip = 3; + request.most = 120; + std::string why; + expect(recorder.start(request, why), "a recording starts"); + for (int i = 0; i < 400 && recorder.status().state == State::recording; ++i) { + const bool wanted = recorder.wants_frame(); + const auto pixels = game.frame(); + if (!wanted) continue; + const auto depth = with_depth ? game.depth() : std::vector{}; + recorder.take_frame({pixels.data(), 1280, 720, 1280 * 4, true, with_depth ? reinterpret_cast(depth.data()) : nullptr, + 1280 * 4}); + } + const auto taken = recorder.status(); + expect(taken.state == State::stopped && taken.taken == 120, "it stops by itself at the frames asked for"); + expect(taken.depth == with_depth, "it says whether every frame was cut out by depth"); + // Comes round: about 37 frames on it looks as it started, half way it does not. + const float round = recorder.compare(0, 37), half = recorder.compare(0, 19); + expect(round >= 0 && half > 3 * round + 1, "a whole turn on, a frame looks like the first again; half way it does not"); + expect(recorder.compare(0, 500) < 0, "a frame not taken is not compared"); + const auto kept = recorder.kept(); + expect(kept.size() == 120 && kept.front() == 0 && kept.back() == 119, "every frame is kept while there is room"); + SaveRequest save; + for (std::uint32_t j = 0; j < frames; ++j) save.frames.push_back(static_cast(std::lround(37.3 * j / frames))); + save.format = format; + save.size = 256; + save.solid = !skater; + save.card_title = skater ? "Outfit" : "Board"; + save.card = {{"Deck", "Test Deck"}, {"Wheels", "Round"}}; + save.fonts = {heading, body, brush}; + save.file = std::filesystem::temp_directory_path() / "capture_tests" / (std::string("turn.") + extension(format)); + expect(recorder.save(save, why), "it saves"); + std::string second; + expect(!recorder.save(save, second), "one save at a time"); + for (int i = 0; i < 600 && recorder.status().state == State::saving; ++i) std::this_thread::sleep_for(std::chrono::milliseconds(50)); + const auto saved = recorder.status(); + if (saved.state != State::saved) + std::printf("recording (%s, %s, %s): %s\n", extension(format), skater ? "skater" : "board", with_depth ? "depth" : "colour", saved.message.c_str()); + expect(saved.state == State::saved, "the recording is saved"); + std::error_code error; + expect(std::filesystem::file_size(saved.file, error) > 1000, "its file is written"); + if (with_depth) expect(saved.message.find("by depth") != std::string::npos, "cut out by the game's depth when it had one"); + else expect(saved.message.find("by colour") != std::string::npos, "else by colour"); + recorder.discard(); + expect(recorder.status().state == State::idle && recorder.kept().empty(), "discarded, nothing is held"); +} + +int main(int argc, char** argv) { + test_matte(); + test_backdrop_key(false); + test_backdrop_key(true); + test_geometry(); + test_png(); + test_gif(); + test_webm(); + if (argc > 1) { + test_panel(argv[1], argc > 2 ? argv[2] : nullptr); + test_recording(argv[1], Format::apng, false, false, 16); + test_recording(argv[1], Format::gif, false, true, 24); + test_recording(argv[1], Format::webm, true, false, 16); + test_recording(argv[1], Format::png, true, true, 1); + } else std::printf("skipped: the card and recordings (pass the fonts folder)\n"); + if (!failures) std::printf("capture tests passed\n"); + return failures ? 1 : 0; +} diff --git a/Extension/Rendering/Capture/capture_commands.cpp b/Extension/Rendering/Capture/capture_commands.cpp new file mode 100644 index 00000000..c7f89cc0 --- /dev/null +++ b/Extension/Rendering/Capture/capture_commands.cpp @@ -0,0 +1,48 @@ +#include "Extension/Console/commands.h" +#include "Extension/UI/Overlay/overlay.h" +#include +#include +#include +#include + +namespace dingosdk::console { +namespace { +std::filesystem::path game_folder() { + std::wstring path(MAX_PATH, L'\0'); + for (;;) { + const auto length = GetModuleFileNameW(nullptr, path.data(), static_cast(path.size())); + if (length < path.size()) { + path.resize(length); + break; + } + path.resize(path.size() * 2); + } + return std::filesystem::path(path).parent_path(); +} +} // namespace + +// Research aids for screen captures (reskate.capture), from Fit Check's `fitcheck depth`. +void register_capture_commands(Commands& registry) { + auto arguments = argument("verb", Type::text, true); + arguments.complete = [](const Model&, auto) { return std::vector{"depth"}; }; + auto command = action("capture", + "Research aids for plugins' screen captures (reskate.capture): depth (the game's depth buffers over the next frames, " + "as pictures with depth-probe.txt in Captures\\depth beside Skate.exe). Answers go to the log.", + Group::console, {std::move(arguments)}); + command.run = [](const Model&, const Values& values, const Output& out) { + std::istringstream words(values.empty() ? std::string{} : std::get(values[0])); + std::string verb; + words >> verb; + if (verb == "depth") { + const auto folder = game_folder() / L"Captures" / L"depth"; + std::error_code error; + std::filesystem::create_directories(folder, error); + overlay::request_depth_probe(folder.c_str()); + out("Taking the game's depth buffers over the next frames: depth-probe.txt and depth-*.pgm go to Captures\\depth."); + } else { + out("capture depth"); + } + }; + registry.add(std::move(command)); +} +} // namespace dingosdk::console diff --git a/Extension/Rendering/Capture/capture_encode.h b/Extension/Rendering/Capture/capture_encode.h new file mode 100644 index 00000000..51740d20 --- /dev/null +++ b/Extension/Rendering/Capture/capture_encode.h @@ -0,0 +1,38 @@ +#pragma once +#include "capture_image.h" +#include +#include + +// File formats for a screen capture (reskate.capture). Every frame of an animation is the same size. +namespace dingosdk::capture { +struct Animation { + std::vector frames; + std::uint32_t fps{30}; +}; + +// A still PNG (RGBA). +std::vector encode_png(const Image& image); +// An animated PNG that loops forever. Keeps full 8-bit alpha; flatten the frames over a matte +// (over_matte) first for an opaque one. +std::vector encode_apng(const Animation& animation); + +struct GifOptions { + // GIF transparency is all or nothing. Pixels at least half opaque are kept, blended over this + // colour so their soft edge does not go black; the rest are cut out. Pick something close to + // the page the GIF will sit on. + Rgb fringe{40, 40, 40}; + // A light, fixed (ordered) dither, which does not crawl between frames the way error diffusion does. + bool dither{true}; +}; +// An animated GIF that loops forever: one 256-colour palette for every frame. +// GIF timing is in hundredths of a second, so the frame rate is kept on average, at most 50. +std::vector encode_gif(const Animation& animation, const GifOptions& options = {}); + +struct WebmOptions { + int quality{12}; // VP8 quantizer index for the colour, 0 (best) .. 127 + int alpha_quality{4}; // and for the transparency, which shows every block edge, so finer +}; +// A WebM with VP8 video; frames with any transparency carry it as VP8 alpha (Chrome, Firefox and +// ffmpeg's libvpx decoder show it). WebM has no loop flag: players loop it themselves. +std::vector encode_webm(const Animation& animation, const WebmOptions& options = {}); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_gif.cpp b/Extension/Rendering/Capture/capture_gif.cpp new file mode 100644 index 00000000..6cffadeb --- /dev/null +++ b/Extension/Rendering/Capture/capture_gif.cpp @@ -0,0 +1,229 @@ +#include "capture_encode.h" +#include +#include +#include + +// Animated GIF (GIF89a) for captures: one median-cut palette shared by every frame. +namespace dingosdk::capture { +namespace { +constexpr int bins = 32 * 32 * 32; // 5 bits a channel + +std::uint32_t key_of(int r, int g, int b) noexcept { + return static_cast((r >> 3) << 10 | (g >> 3) << 5 | (b >> 3)); +} + +// 4x4 Bayer matrix centred on zero, in steps of one 5-bit level (8 of 255) across its range. +constexpr std::array bayer{0, 8, 2, 10, 12, 4, 14, 6, 3, 11, 1, 9, 15, 7, 13, 5}; +int dither_offset(std::uint32_t x, std::uint32_t y) noexcept { return bayer[(y & 3) * 4 + (x & 3)] / 2 - 4; } + +struct Histogram { + std::vector count = std::vector(bins); + std::vector> sum = std::vector>(bins); +}; + +struct Box { + std::vector keys; // occupied bins + std::uint64_t weight{}; +}; + +// The bin's channel `c` (0 r, 1 g, 2 b), 0..31. +int channel(std::uint32_t key, int c) noexcept { return static_cast(key >> (10 - c * 5)) & 31; } + +std::vector median_cut(const Histogram& h, std::size_t colours) { + std::vector boxes(1); + for (std::uint32_t k = 0; k < bins; ++k) + if (h.count[k]) boxes[0].keys.push_back(k), boxes[0].weight += h.count[k]; + if (boxes[0].keys.empty()) return {Rgb{0, 0, 0}}; + while (boxes.size() < colours) { + // Split the box with the most pixels times its widest spread. + std::size_t pick = boxes.size(); + std::uint64_t best = 0; + int best_channel = 0; + for (std::size_t i = 0; i < boxes.size(); ++i) { + if (boxes[i].keys.size() < 2) continue; + for (int c = 0; c < 3; ++c) { + int lo = 31, hi = 0; + for (const auto k : boxes[i].keys) lo = std::min(lo, channel(k, c)), hi = std::max(hi, channel(k, c)); + const auto score = boxes[i].weight * static_cast(hi - lo); + if (score > best) best = score, pick = i, best_channel = c; + } + } + if (pick == boxes.size()) break; // nothing left to split + auto& box = boxes[pick]; + std::sort(box.keys.begin(), box.keys.end(), + [c = best_channel](auto a, auto b) { return channel(a, c) < channel(b, c); }); + // Split at the weighted median. + std::uint64_t running = 0; + std::size_t cut = 1; + for (; cut < box.keys.size(); ++cut) { + running += h.count[box.keys[cut - 1]]; + if (running * 2 >= box.weight) break; + } + cut = std::clamp(cut, 1, box.keys.size() - 1); + Box other; + other.keys.assign(box.keys.begin() + static_cast(cut), box.keys.end()); + box.keys.resize(cut); + box.weight = 0; + for (const auto k : box.keys) box.weight += h.count[k]; + for (const auto k : other.keys) other.weight += h.count[k]; + boxes.push_back(std::move(other)); + } + std::vector palette; + for (const auto& box : boxes) { + std::array total{}; + for (const auto k : box.keys) + for (int c = 0; c < 3; ++c) total[c] += h.sum[k][c]; + Rgb colour{}; + for (int c = 0; c < 3; ++c) colour[c] = static_cast((total[c] + box.weight / 2) / box.weight); + palette.push_back(colour); + } + return palette; +} + +// Variable-length LZW (GIF's flavour) of 8-bit indices. +std::vector lzw(const std::vector& indices) { + constexpr int min_bits = 8, clear = 1 << min_bits, end = clear + 1; + std::vector out; + std::uint32_t bit_buffer = 0; + int bit_count = 0, code_bits = min_bits + 1, next = end + 1; + const auto emit = [&](int code) { + bit_buffer |= static_cast(code) << bit_count; + bit_count += code_bits; + while (bit_count >= 8) out.push_back(static_cast(bit_buffer)), bit_buffer >>= 8, bit_count -= 8; + }; + std::vector table(4096 * 256, -1); // (prefix code, next index) -> code + const auto reset = [&] { + std::fill(table.begin(), table.end(), static_cast(-1)); + code_bits = min_bits + 1; + next = end + 1; + }; + emit(clear); + if (indices.empty()) { + emit(end); + } else { + int prefix = indices[0]; + for (std::size_t i = 1; i < indices.size(); ++i) { + const auto byte = indices[i]; + const auto slot = static_cast(prefix) * 256 + byte; + if (table[slot] >= 0) { prefix = table[slot]; continue; } + emit(prefix); + if (next < 4096) { + table[slot] = static_cast(next++); + if (next > (1 << code_bits) && code_bits < 12) ++code_bits; + } else { + emit(clear); + reset(); + } + prefix = byte; + } + emit(prefix); + emit(end); + } + if (bit_count) out.push_back(static_cast(bit_buffer)); + return out; +} + +void put16(std::vector& out, std::uint32_t v) { + out.push_back(static_cast(v)); + out.push_back(static_cast(v >> 8)); +} +} // namespace + +std::vector encode_gif(const Animation& animation, const GifOptions& options) { + if (animation.frames.empty() || animation.frames.front().empty()) throw std::invalid_argument("No frames to save."); + const auto width = animation.frames.front().width, height = animation.frames.front().height; + if (width > 65535 || height > 65535) throw std::invalid_argument("Too large for a GIF."); + for (const auto& frame : animation.frames) + if (frame.width != width || frame.height != height) throw std::invalid_argument("The frames are not all the same size."); + + // Every kept pixel, its soft edge blended over the fringe colour. + const auto kept = [&](const std::uint8_t* p, Rgb& colour) { + if (p[3] < 128) return false; + for (int c = 0; c < 3; ++c) + colour[c] = static_cast((p[c] * p[3] + options.fringe[c] * (255 - p[3]) + 127) / 255); + return true; + }; + Histogram histogram; + bool transparent = false; + for (const auto& frame : animation.frames) + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + Rgb colour; + if (!kept(frame.at(x, y), colour)) { transparent = true; continue; } + const auto k = key_of(colour[0], colour[1], colour[2]); + ++histogram.count[k]; + for (int c = 0; c < 3; ++c) histogram.sum[k][c] += colour[c]; + } + auto palette = median_cut(histogram, transparent ? 255 : 256); + const auto colours = palette.size(); // the rest of the table is padding, never matched + const auto transparent_index = static_cast(colours); + // Nearest palette colour of each bin, worked out when first needed. + std::vector nearest(bins, -1); + const auto map = [&](int r, int g, int b) { + const auto k = key_of(r, g, b); + if (nearest[k] < 0) { + const int cr = (r & ~7) + 4, cg = (g & ~7) + 4, cb = (b & ~7) + 4; + int best = 0, best_distance = INT32_MAX; + for (std::size_t i = 0; i < colours; ++i) { + const int dr = palette[i][0] - cr, dg = palette[i][1] - cg, db = palette[i][2] - cb; + // Weighted roughly by how much the eye cares about each. + const int d = 3 * dr * dr + 4 * dg * dg + 2 * db * db; + if (d < best_distance) best_distance = d, best = static_cast(i); + } + nearest[k] = static_cast(best); + } + return static_cast(nearest[k]); + }; + + std::vector out = {'G', 'I', 'F', '8', '9', 'a'}; + put16(out, width); + put16(out, height); + out.push_back(0xf7); // global colour table of 256 entries + out.push_back(0); + out.push_back(0); + palette.resize(256, Rgb{0, 0, 0}); + for (const auto& colour : palette) out.insert(out.end(), colour.begin(), colour.end()); + // Loop forever (NETSCAPE2.0). + out.insert(out.end(), {0x21, 0xff, 0x0b, 'N', 'E', 'T', 'S', 'C', 'A', 'P', 'E', '2', '.', '0', 0x03, 0x01, 0x00, 0x00, 0x00}); + + const auto fps = std::clamp(animation.fps, 1, 50); + std::uint64_t elapsed = 0; // hundredths of a second written so far + std::vector indices(std::size_t{width} * height); + for (std::size_t f = 0; f < animation.frames.size(); ++f) { + const auto& frame = animation.frames[f]; + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + Rgb colour; + auto& index = indices[std::size_t{y} * width + x]; + if (!kept(frame.at(x, y), colour)) { index = transparent_index; continue; } + const int offset = options.dither ? dither_offset(x, y) : 0; + index = map(std::clamp(colour[0] + offset, 0, 255), std::clamp(colour[1] + offset, 0, 255), + std::clamp(colour[2] + offset, 0, 255)); + } + const auto due = (f + 1) * 100 / fps; + const auto delay = static_cast(std::max(2, due - elapsed)); + elapsed += delay; + // Graphic control: clear to the background between frames when anything is see-through. + out.insert(out.end(), {0x21, 0xf9, 0x04, static_cast(transparent ? 0x09 : 0x04)}); + put16(out, delay); + out.push_back(transparent ? transparent_index : 0); + out.push_back(0); + out.push_back(0x2c); + put16(out, 0); + put16(out, 0); + put16(out, width); + put16(out, height); + out.push_back(0); + out.push_back(8); // LZW minimum code size + const auto data = lzw(indices); + for (std::size_t i = 0; i < data.size(); i += 255) { + const auto n = std::min(255, data.size() - i); + out.push_back(static_cast(n)); + out.insert(out.end(), data.begin() + static_cast(i), data.begin() + static_cast(i + n)); + } + out.push_back(0); + } + out.push_back(0x3b); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_image.cpp b/Extension/Rendering/Capture/capture_image.cpp new file mode 100644 index 00000000..568e9833 --- /dev/null +++ b/Extension/Rendering/Capture/capture_image.cpp @@ -0,0 +1,171 @@ +#include "capture_image.h" +#include + +namespace dingosdk::capture { +namespace { +// Per-pixel weights of one axis of an area (box) resample from `from` to `to` samples. +struct Tap { std::uint32_t first{}; std::vector weights; }; +std::vector area_taps(std::uint32_t from, std::uint32_t to) { + std::vector taps(to); + const double scale = static_cast(from) / to; + for (std::uint32_t i = 0; i < to; ++i) { + auto& tap = taps[i]; + if (scale <= 1.0) { + // Enlarging: linear between the two nearest source samples. + const double centre = (i + 0.5) * scale - 0.5; + const double floor = std::floor(centre); + const auto left = static_cast(floor); + const auto t = static_cast(centre - floor); + const auto a = static_cast(std::clamp(left, 0, from - 1)); + const auto b = static_cast(std::clamp(left + 1, 0, from - 1)); + tap.first = a; + tap.weights.assign(b - a + 1, 0.0f); + tap.weights[0] += 1.0f - t; + tap.weights[b - a] += t; + continue; + } + const double start = i * scale, end = (i + 1) * scale; + tap.first = static_cast(std::floor(start)); + const auto last = std::min(from - 1, static_cast(std::ceil(end)) - 1); + float total = 0; + for (auto s = tap.first; s <= last; ++s) { + const auto w = static_cast(std::min(end, s + 1.0) - std::max(start, s)); + tap.weights.push_back(w); + total += w; + } + for (auto& w : tap.weights) w /= total; + } + return taps; +} +} // namespace + +Image over_matte(const Image& image, Rgb matte) { + Image out = image; + for (std::size_t i = 0; i < out.rgba.size(); i += 4) { + const unsigned a = out.rgba[i + 3]; + for (int c = 0; c < 3; ++c) + out.rgba[i + c] = static_cast((out.rgba[i + c] * a + matte[c] * (255 - a) + 127) / 255); + out.rgba[i + 3] = 255; + } + return out; +} + +Rect content_bounds(const Image& image, std::uint8_t threshold) { + std::uint32_t x0 = image.width, y0 = image.height, x1 = 0, y1 = 0; + for (std::uint32_t y = 0; y < image.height; ++y) + for (std::uint32_t x = 0; x < image.width; ++x) + if (image.at(x, y)[3] >= threshold) { + x0 = std::min(x0, x), y0 = std::min(y0, y); + x1 = std::max(x1, x + 1), y1 = std::max(y1, y + 1); + } + if (x0 >= x1 || y0 >= y1) return {}; + return {x0, y0, x1 - x0, y1 - y0}; +} + +Rect united(Rect a, Rect b) noexcept { + if (a.empty()) return b; + if (b.empty()) return a; + const auto x0 = std::min(a.x, b.x), y0 = std::min(a.y, b.y); + const auto x1 = std::max(a.x + a.width, b.x + b.width), y1 = std::max(a.y + a.height, b.y + b.height); + return {x0, y0, x1 - x0, y1 - y0}; +} + +Rect padded_square(Rect rect, std::uint32_t margin, std::uint32_t width, std::uint32_t height) noexcept { + if (rect.empty() || !width || !height) return {}; + const auto side = std::min({std::max(rect.width, rect.height) + margin * 2, width, height}); + const auto centre_x = rect.x + rect.width / 2, centre_y = rect.y + rect.height / 2; + const auto place = [side](std::uint32_t centre, std::uint32_t limit) { + const auto half = side / 2; + const auto start = centre > half ? centre - half : 0u; + return std::min(start, limit - side); + }; + return {place(centre_x, width), place(centre_y, height), side, side}; +} + +Image square_around(const Image& image, Rect content, std::uint32_t margin) { + if (content.empty()) return {}; + const auto side = std::max(content.width, content.height) + margin * 2; + // The square's corner, which may lie outside the image (left of it, above it). + const auto x0 = static_cast(content.x) + static_cast(content.width / 2) - static_cast(side / 2); + const auto y0 = static_cast(content.y) + static_cast(content.height / 2) - static_cast(side / 2); + Image out(side, side); + for (std::uint32_t y = 0; y < side; ++y) + for (std::uint32_t x = 0; x < side; ++x) { + const auto sx = x0 + x, sy = y0 + y; + if (sx >= 0 && sy >= 0 && sx < image.width && sy < image.height) + std::copy_n(image.at(static_cast(sx), static_cast(sy)), 4, out.at(x, y)); + } + return out; +} + +Image cropped(const Image& image, Rect rect) { + Image out(rect.width, rect.height); + for (std::uint32_t y = 0; y < rect.height; ++y) + for (std::uint32_t x = 0; x < rect.width; ++x) { + const auto sx = rect.x + x, sy = rect.y + y; + if (sx < image.width && sy < image.height) std::copy_n(image.at(sx, sy), 4, out.at(x, y)); + } + return out; +} + +Image resized(const Image& image, std::uint32_t w, std::uint32_t h) { + if (image.empty() || !w || !h) return {}; + const auto columns = area_taps(image.width, w), rows = area_taps(image.height, h); + // Horizontal pass into premultiplied floats, then vertical. + std::vector wide(std::size_t{w} * image.height * 4); + for (std::uint32_t y = 0; y < image.height; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto& tap = columns[x]; + float sum[4]{}; + for (std::size_t k = 0; k < tap.weights.size(); ++k) { + const auto* p = image.at(tap.first + static_cast(k), y); + const float a = p[3] / 255.0f, weight = tap.weights[k]; + for (int c = 0; c < 3; ++c) sum[c] += p[c] * a * weight; + sum[3] += p[3] * weight; + } + std::copy_n(sum, 4, wide.data() + (std::size_t{y} * w + x) * 4); + } + Image out(w, h); + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto& tap = rows[y]; + float sum[4]{}; + for (std::size_t k = 0; k < tap.weights.size(); ++k) { + const auto* p = wide.data() + ((tap.first + k) * w + x) * 4; + for (int c = 0; c < 4; ++c) sum[c] += p[c] * tap.weights[k]; + } + auto* d = out.at(x, y); + const float alpha = std::clamp(sum[3], 0.0f, 255.0f); + d[3] = static_cast(std::lround(alpha)); + for (int c = 0; c < 3; ++c) + d[c] = d[3] ? static_cast(std::clamp(std::lround(sum[c] * 255.0f / alpha), 0L, 255L)) : 0; + } + return out; +} + +Image fitted(const Image& image, std::uint32_t size) { + if (image.empty() || !size) return {}; + const double scale = static_cast(size) / std::max(image.width, image.height); + const auto w = std::max(1, static_cast(std::lround(image.width * scale))); + const auto h = std::max(1, static_cast(std::lround(image.height * scale))); + const auto small = resized(image, w, h); + Image out(size, size); + const auto left = (size - w) / 2, top = (size - h) / 2; + for (std::uint32_t y = 0; y < h; ++y) std::copy_n(small.at(0, y), std::size_t{w} * 4, out.at(left, top + y)); + return out; +} + +Image side_by_side(const Image& a, const Image& b) { + if (a.empty()) return b; + if (b.empty()) return a; + Image out(a.width + b.width, std::max(a.height, b.height)); + const auto place = [&out](const Image& part, std::uint32_t left) { + const auto top = (out.height - part.height) / 2; + for (std::uint32_t y = 0; y < part.height; ++y) + std::copy_n(part.at(0, y), std::size_t{part.width} * 4, out.at(left, top + y)); + }; + place(a, 0); + place(b, a.width); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_image.h b/Extension/Rendering/Capture/capture_image.h new file mode 100644 index 00000000..c759f633 --- /dev/null +++ b/Extension/Rendering/Capture/capture_image.h @@ -0,0 +1,75 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +// The pictures a capture is made of: straight (not premultiplied) 8-bit RGBA, rows top to +// bottom with no padding. Kept free of Windows and the game so the encoders and matting can be +// tested anywhere (Extension/Rendering/Capture/Test). +namespace dingosdk::capture { +using Rgb = std::array; + +struct Image { + std::uint32_t width{}, height{}; + std::vector rgba; // width * height * 4 + + Image() = default; + Image(std::uint32_t w, std::uint32_t h) : width(w), height(h), rgba(std::size_t{w} * h * 4, 0) {} + bool empty() const noexcept { return !width || !height; } + std::uint8_t* at(std::uint32_t x, std::uint32_t y) noexcept { return rgba.data() + (std::size_t{y} * width + x) * 4; } + const std::uint8_t* at(std::uint32_t x, std::uint32_t y) const noexcept { + return rgba.data() + (std::size_t{y} * width + x) * 4; + } +}; + +// A rectangle of pixels in a captured frame. +struct Rect { + std::uint32_t x{}, y{}, width{}, height{}; + bool empty() const noexcept { return !width || !height; } +}; + +// Rows of 4-byte pixels as read back from the GPU: `pitch` bytes apart, channel order given by `bgra`. +struct PixelView { + const std::uint8_t* data{}; + std::uint32_t width{}, height{}, pitch{}; + bool bgra{}; + // The game's depth for the same frame, when there is one: 32-bit floats a pixel, reversed (1 + // near, 0 at the far plane), `depth_pitch` bytes a row, the frame's size. + const std::uint8_t* depth{}; + std::uint32_t depth_pitch{}; + float depth_at(std::uint32_t x, std::uint32_t y) const noexcept { + float v; + std::memcpy(&v, depth + std::size_t{y} * depth_pitch + std::size_t{x} * 4, 4); + return v; + } + std::array rgb(std::uint32_t x, std::uint32_t y) const noexcept { + const auto* p = data + std::size_t{y} * pitch + std::size_t{x} * 4; + return bgra ? std::array{p[2], p[1], p[0]} : std::array{p[0], p[1], p[2]}; + } +}; + +// `image` flattened over `matte`: every pixel opaque. +Image over_matte(const Image& image, Rgb matte); +// The smallest rectangle holding every pixel at least `threshold` opaque, or an empty one. +Rect content_bounds(const Image& image, std::uint8_t threshold = 8); +// The union of two rectangles (an empty one counts as nothing). +Rect united(Rect a, Rect b) noexcept; +// `rect` grown by `margin` on every side and made square, kept inside `width` x `height`. +Rect padded_square(Rect rect, std::uint32_t margin, std::uint32_t width, std::uint32_t height) noexcept; +// A square around `content`, `margin` wider on every side, centred on it: it may reach past the +// image, and what is outside is transparent (a board taller than the area it was taken from). +Image square_around(const Image& image, Rect content, std::uint32_t margin); +// The part of `image` inside `rect`; pixels outside the image are transparent. +Image cropped(const Image& image, Rect rect); +// `image` resized to exactly `width` x `height` (filtered as fitted is). +Image resized(const Image& image, std::uint32_t width, std::uint32_t height); +// `image` resized to fit inside `size` x `size` (keeping its shape, centred, transparent around it), +// filtered in premultiplied alpha so edges do not pick up the colour of transparent pixels. +Image fitted(const Image& image, std::uint32_t size); +// `b` placed to the right of `a`, both centred vertically on a canvas as tall as the taller. +Image side_by_side(const Image& a, const Image& b); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_matte.cpp b/Extension/Rendering/Capture/capture_matte.cpp new file mode 100644 index 00000000..a3062aa4 --- /dev/null +++ b/Extension/Rendering/Capture/capture_matte.cpp @@ -0,0 +1,226 @@ +#include "capture_matte.h" +#include +#include +#include +#include + +namespace dingosdk::capture { +Backgrounds measure_backgrounds(const PixelView& dark, const PixelView& light, Rect area) noexcept { + // The median of each channel along the edge: a stray UI element on it does not shift it. + Backgrounds out; + if (area.empty()) return out; + std::array, 6> counts{}; + std::uint64_t total = 0; + const auto add = [&](std::uint32_t x, std::uint32_t y) { + const auto d = dark.rgb(x, y), l = light.rgb(x, y); + for (std::size_t c = 0; c < 3; ++c) ++counts[c][d[c]], ++counts[3 + c][l[c]]; + ++total; + }; + const auto right = area.x + area.width - 1, bottom = area.y + area.height - 1; + for (auto x = area.x; x <= right; ++x) add(x, area.y), add(x, bottom); + for (auto y = area.y + 1; y < bottom; ++y) add(area.x, y), add(right, y); + for (std::size_t k = 0; k < 6; ++k) { + std::uint64_t seen = 0; + std::size_t value = 0; + while (value < 255 && (seen += counts[k][value]) * 2 < total) ++value; + (k < 3 ? out.dark : out.light)[k % 3] = static_cast(value); + } + return out; +} + +Image matte_pair(const PixelView& dark, const PixelView& light, Rect area, const Backgrounds& backgrounds, + float floor) { + Image out(area.width, area.height); + std::array step{}; + float step_power = 0; + for (int c = 0; c < 3; ++c) { + step[c] = backgrounds.light[c] - backgrounds.dark[c]; + step_power += step[c] * step[c]; + } + // Backgrounds that look the same tell nothing apart: everything is taken as opaque. + const bool usable = step_power > 3 * 16.0f * 16.0f; + floor = std::clamp(floor, 0.0f, 0.45f); + for (std::uint32_t y = 0; y < area.height; ++y) + for (std::uint32_t x = 0; x < area.width; ++x) { + const auto d = dark.rgb(area.x + x, area.y + y), l = light.rgb(area.x + x, area.y + y); + auto* p = out.at(x, y); + float seen = 0; // how much of the background shows: 0 opaque .. 1 empty + if (usable) { + // Least squares over the three channels of (light - dark) = seen * step. + float dot = 0; + for (int c = 0; c < 3; ++c) dot += (static_cast(l[c]) - d[c]) * step[c]; + seen = std::clamp((dot / step_power - floor) / (1.0f - 2.0f * floor), 0.0f, 1.0f); + } + const float alpha = 1.0f - seen; + if (alpha <= 0.0f) { p[0] = p[1] = p[2] = p[3] = 0; continue; } + for (int c = 0; c < 3; ++c) { + // The board's own colour, from both captures with their background taken away. + const float a = d[c] - seen * backgrounds.dark[c], b = l[c] - seen * backgrounds.light[c]; + p[c] = static_cast(std::clamp(std::lround((a + b) * 0.5f / alpha), 0L, 255L)); + } + p[3] = static_cast(std::lround(alpha * 255.0f)); + } + return out; +} +// Without a background of its own to swap, the board is keyed out against the backdrop, found from +// the turn itself. The backdrop keeps still and, across any one row, changes only gently (the Board +// screen's vignette); the board covers part of each row at most. So: +// 1. each pixel's steady colour over the turn (the one it keeps most of the time) is a candidate; +// 2. along each row, a candidate is the backdrop when it is close to the row's typical candidate: +// one the board covers for most of the turn (a white deck, a white grip in the middle) has a +// board colour for its steady one, and is not; +// 3. where a row has no backdrop candidate the backdrop is carried across from those either side; +// 4. the backdrop at a pixel is then the colour it shows, in any frame, close to that estimate. +// An exact backdrop lets the key be tight, so a black grip against the dark backdrop still reads as +// board. Matting (a stage that swaps the background) is better still; this is for when there is none. +void key_against_backdrop(std::vector& frames, bool solid_rows) { + if (frames.empty()) return; + const auto raw = frames; + const auto w = frames.front().width, h = frames.front().height; + if (w < 8 || h < 8) return; + const auto luma = [](const std::uint8_t* p) { return (77 * p[0] + 150 * p[1] + 29 * p[2]) >> 8; }; + Image plate(w, h); + std::vector values(frames.size()), sorted(frames.size()); + std::vector> candidate(w); + std::vector has(w), backdrop(w); + std::vector> previous_row; + for (std::uint32_t y = 0; y < h; ++y) { + // 1. Steady colours along the row. + for (std::uint32_t x = 0; x < w; ++x) { + for (std::size_t f = 0; f < frames.size(); ++f) values[f] = luma(frames[f].at(x, y)); + sorted = values; + std::nth_element(sorted.begin(), sorted.begin() + static_cast(sorted.size() / 2), sorted.end()); + const int middle = sorted[sorted.size() / 2]; + std::array sum{}; + std::size_t steady = 0; + for (std::size_t f = 0; f < frames.size(); ++f) + if (std::abs(values[f] - middle) <= 4) { + ++steady; + for (std::size_t c = 0; c < 3; ++c) sum[c] += frames[f].at(x, y)[c]; + } + has[x] = steady * 100 >= frames.size() * 35; + for (std::size_t c = 0; c < 3; ++c) candidate[x][c] = has[x] ? static_cast(sum[c] / static_cast(steady)) : 0.0f; + } + // 2. The row's typical candidate, and those close to it. Close to it locally too: compared + // with the median of the candidates within half the row each way, so a vignette's slope across + // the row is followed. + const std::uint32_t reach = std::max(4, w / 2); + std::array, 3> window; + for (std::uint32_t x = 0; x < w; ++x) { + backdrop[x] = false; + if (!has[x]) continue; + const auto from = x > reach ? x - reach : 0u, to = std::min(w, x + reach + 1); + for (auto& v : window) v.clear(); + for (auto k = from; k < to; ++k) + if (has[k]) + for (std::size_t c = 0; c < 3; ++c) window[c].push_back(candidate[k][c]); + float off = 0; + for (std::size_t c = 0; c < 3; ++c) { + auto& v = window[c]; + std::nth_element(v.begin(), v.begin() + static_cast(v.size() / 2), v.end()); + off = std::max(off, std::abs(candidate[x][c] - v[v.size() / 2])); + } + backdrop[x] = off <= 12.0f; + } + // 3. Carried across the gaps from the backdrop either side (or from the row above). + std::vector> estimate(w); + std::int64_t left = -1; + for (std::uint32_t x = 0; x <= w; ++x) { + if (x < w && !backdrop[x]) continue; + const auto right = static_cast(x); + for (auto k = left + 1; k < right; ++k) { + const auto i = static_cast(k); + if (left < 0 && right >= static_cast(w)) { + estimate[i] = previous_row.empty() ? std::array{} : previous_row[i]; + } else if (left < 0) { + estimate[i] = candidate[static_cast(right)]; + } else if (right >= static_cast(w)) { + estimate[i] = candidate[static_cast(left)]; + } else { + const float t = static_cast(k - left) / static_cast(right - left); + for (std::size_t c = 0; c < 3; ++c) + estimate[i][c] = candidate[static_cast(left)][c] + + (candidate[static_cast(right)][c] - candidate[static_cast(left)][c]) * t; + } + } + if (x < w) estimate[x] = candidate[x], left = right; + } + previous_row = estimate; + // 4. What each pixel shows that is close to its estimate: the backdrop, exactly. + for (std::uint32_t x = 0; x < w; ++x) { + std::array sum{}; + std::size_t fits = 0; + for (const auto& frame : frames) { + const auto* p = frame.at(x, y); + float off = 0; + for (std::size_t c = 0; c < 3; ++c) off = std::max(off, std::abs(static_cast(p[c]) - estimate[x][c])); + if (off > 8.0f) continue; + ++fits; + for (std::size_t c = 0; c < 3; ++c) sum[c] += p[c]; + } + auto* b = plate.at(x, y); + for (std::size_t c = 0; c < 3; ++c) + b[c] = static_cast(std::clamp(std::lround(fits >= 2 ? sum[c] / static_cast(fits) : estimate[x][c]), 0L, 255L)); + } + } + for (auto& frame : frames) + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + auto* p = frame.at(x, y); + const auto* b = plate.at(x, y); + int distance = 0; + for (std::size_t c = 0; c < 3; ++c) distance = std::max(distance, std::abs(p[c] - b[c])); + const float alpha = std::clamp((static_cast(distance) - 3.0f) / 9.0f, 0.0f, 1.0f); + if (alpha <= 0) { p[0] = p[1] = p[2] = p[3] = 0; continue; } + for (std::size_t c = 0; c < 3; ++c) + p[c] = static_cast(std::clamp(std::lround((p[c] - (1 - alpha) * b[c]) / alpha), 0L, 255L)); + p[3] = static_cast(std::lround(alpha * 255)); + } + // Part of what turns can match the backdrop and key out as a hole in it: a dark joint or shirt, + // part of a deck's graphic. A pocket it closes all round is filled from the frame. A board is + // solid, so any pocket in it is board; a skater's bigger pockets are real backdrop (between an + // arm and the body). (Not filled across each row: the backdrop between a deck and its wheels is + // open above and below the truck, and stays clear.) + for (std::size_t f = 0; f < frames.size(); ++f) { + auto& frame = frames[f]; + const std::size_t n = std::size_t{w} * h; + std::vector outside(n); + std::vector todo; + std::size_t opaque = 0; + const auto clear = [&](std::size_t p) { return frame.rgba[p * 4 + 3] < 128; }; + for (std::size_t p = 0; p < n; ++p) { + if (!clear(p)) { ++opaque; continue; } + const auto x = p % w, y = p / w; + if (x == 0 || y == 0 || x + 1 == w || y + 1 == h) outside[p] = 1, todo.push_back(static_cast(p)); + } + const auto spread = [&](std::vector& mark, std::vector& stack, std::vector* region) { + while (!stack.empty()) { + const auto p = stack.back(); + stack.pop_back(); + if (region) region->push_back(p); + const auto x = p % w, y = p / w; + const std::uint32_t next[4] = {x > 0 ? p - 1 : p, x + 1 < w ? p + 1 : p, y > 0 ? p - w : p, y + 1 < h ? p + w : p}; + for (const auto q : next) + if (!mark[q] && clear(q)) mark[q] = 1, stack.push_back(q); + } + }; + spread(outside, todo, nullptr); + const auto pocket_most = std::max(16, solid_rows ? opaque / 2 : opaque * 3 / 100); + std::vector region; + for (std::size_t p = 0; p < n; ++p) { + if (outside[p] || !clear(p)) continue; + region.clear(); + outside[p] = 1; + todo.push_back(static_cast(p)); + spread(outside, todo, ®ion); + if (region.size() > pocket_most) continue; + for (const auto q : region) { + auto* px = frame.rgba.data() + std::size_t{q} * 4; + std::copy_n(raw[f].rgba.data() + std::size_t{q} * 4, 3, px); + px[3] = 255; + } + } + } +} + +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_matte.h b/Extension/Rendering/Capture/capture_matte.h new file mode 100644 index 00000000..f6db48f8 --- /dev/null +++ b/Extension/Rendering/Capture/capture_matte.h @@ -0,0 +1,29 @@ +#pragma once +#include "capture_image.h" + +// Transparency from two captures of the same still picture, one over a dark background and one over +// a light one ("difference matting"). Where the board covers the background both captures agree; +// where the background shows through they differ by the full background step; part-covered pixels +// (antialiased edges, the holo's see-through bits) differ by part of it. That gives soft edges the +// game's backbuffer cannot: it has no alpha of its own. +// +// The backgrounds do not have to be pure black and white. The game tone-maps and grades them, so +// what they actually look like on screen is measured (`Backgrounds`) and used instead. +namespace dingosdk::capture { +struct Backgrounds { + std::array dark{}, light{}; // the empty background's colour in each capture, 0..255 +}; +// The typical (median) colour of the pixels along the edge of `area` in each capture: the background, as long +// as the board is not touching the edge of the area. +Backgrounds measure_backgrounds(const PixelView& dark, const PixelView& light, Rect area) noexcept; +// The board cut out of `area`: straight RGBA, colour taken from the dark capture. +// `floor` is how much difference (0..1 of the background step) is still counted as fully opaque, to +// keep the noise of the game's dithering and film grain from leaving the board faintly see-through. +Image matte_pair(const PixelView& dark, const PixelView& light, Rect area, const Backgrounds& backgrounds, + float floor = 0.04f); +// Without a second background to compare against: the board keyed out of every frame of a whole +// turn (opaque captures of the same area) against the backdrop the turn reveals. See the .cpp. +// `solid_rows`: what turns is solid (a board), so any hole it closes all round is part of it. For a +// skater only small ones are: their arms and legs have backdrop between them. +void key_against_backdrop(std::vector& frames, bool solid_rows = true); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_panel.cpp b/Extension/Rendering/Capture/capture_panel.cpp new file mode 100644 index 00000000..05358585 --- /dev/null +++ b/Extension/Rendering/Capture/capture_panel.cpp @@ -0,0 +1,206 @@ +#include "capture_panel.h" +#include +#include +#include + +// stb_truetype comes with Dear ImGui; imgui_draw.cpp keeps its copy private (STBTT_STATIC), so +// this file compiles its own. +#define STBTT_STATIC +#define STB_TRUETYPE_IMPLEMENTATION +#if defined(_MSC_VER) +#pragma warning(push, 0) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wsign-conversion" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wdouble-promotion" +#endif +#include +#if defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +namespace dingosdk::capture { +namespace { +constexpr Rgb card{18, 20, 24}, white{245, 246, 248}, grey{150, 156, 168}, accent{40, 120, 255}; +constexpr std::uint8_t card_alpha = 230; + +// Code points of UTF-8 text; bad bytes come out as U+FFFD. +std::u32string decode(std::string_view text) { + std::u32string out; + for (std::size_t i = 0; i < text.size();) { + const auto b = static_cast(text[i]); + const int extra = b < 0x80 ? 0 : (b >> 5) == 6 ? 1 : (b >> 4) == 14 ? 2 : (b >> 3) == 30 ? 3 : -1; + if (extra < 0 || i + static_cast(extra) >= text.size() + (extra ? 0 : 1)) { + out.push_back(0xfffd), ++i; + continue; + } + char32_t c = extra ? b & (0x3f >> extra) : b; + bool ok = true; + for (int k = 1; k <= extra; ++k) { + const auto next = static_cast(text[i + static_cast(k)]); + if ((next & 0xc0) != 0x80) { ok = false; break; } + c = c << 6 | (next & 0x3f); + } + out.push_back(ok ? c : 0xfffd); + i += ok ? static_cast(extra) + 1 : 1; + } + return out; +} + +// Straight-alpha "over": `colour` at `coverage` (0..255) onto the pixel. +void blend(Image& image, std::int64_t x, std::int64_t y, Rgb colour, unsigned coverage) { + if (x < 0 || y < 0 || x >= image.width || y >= image.height || !coverage) return; + auto* p = image.at(static_cast(x), static_cast(y)); + const float a = static_cast(coverage) / 255.0f, under = static_cast(p[3]) / 255.0f, out = a + under * (1 - a); + for (int c = 0; c < 3; ++c) + p[c] = static_cast(std::lround((colour[c] * a + p[c] * under * (1 - a)) / out)); + p[3] = static_cast(std::lround(out * 255)); +} + +struct Font { + stbtt_fontinfo info{}; + explicit Font(std::span file) { + if (file.empty() || !stbtt_InitFont(&info, file.data(), stbtt_GetFontOffsetForIndex(file.data(), 0))) + throw std::runtime_error("A menu font could not be read."); + } + float scale(float pixels) const { return stbtt_ScaleForPixelHeight(&info, pixels); } + float ascent(float pixels) const { + int a = 0, d = 0, g = 0; + stbtt_GetFontVMetrics(&info, &a, &d, &g); + return static_cast(a) * scale(pixels); + } + float width(std::u32string_view text, float pixels) const { + const float s = scale(pixels); + float x = 0; + for (std::size_t i = 0; i < text.size(); ++i) { + int advance = 0, bearing = 0; + stbtt_GetCodepointHMetrics(&info, static_cast(text[i]), &advance, &bearing); + x += static_cast(advance) * s; + if (i + 1 < text.size()) + x += static_cast(stbtt_GetCodepointKernAdvance(&info, static_cast(text[i]), static_cast(text[i + 1]))) * s; + } + return x; + } + // Draws `text` with its top at `top`. + void draw(Image& image, std::u32string_view text, float left, float top, float pixels, Rgb colour) const { + const float s = scale(pixels), baseline = top + ascent(pixels); + float x = left; + std::vector glyph; + for (std::size_t i = 0; i < text.size(); ++i) { + const int code = static_cast(text[i]); + const float shift = x - std::floor(x); + int x0 = 0, y0 = 0, x1 = 0, y1 = 0; + stbtt_GetCodepointBitmapBoxSubpixel(&info, code, s, s, shift, 0, &x0, &y0, &x1, &y1); + const int w = x1 - x0, h = y1 - y0; + if (w > 0 && h > 0) { + glyph.assign(static_cast(w) * static_cast(h), 0); + stbtt_MakeCodepointBitmapSubpixel(&info, glyph.data(), w, h, w, s, s, shift, 0, code); + const auto gx = static_cast(std::floor(x)) + x0; + const auto gy = static_cast(std::lround(baseline)) + y0; + for (int row = 0; row < h; ++row) + for (int col = 0; col < w; ++col) + blend(image, gx + col, gy + row, colour, glyph[static_cast(row * w + col)]); + } + int advance = 0, bearing = 0; + stbtt_GetCodepointHMetrics(&info, code, &advance, &bearing); + x += static_cast(advance) * s; + if (i + 1 < text.size()) x += static_cast(stbtt_GetCodepointKernAdvance(&info, code, static_cast(text[i + 1]))) * s; + } + } +}; + +// `text` broken at spaces into at most `max_lines` lines of `width`; the last one cut short with "...". +std::vector wrapped(const Font& font, std::u32string text, float pixels, float width, std::size_t max_lines) { + std::vector lines; + while (!text.empty() && lines.size() < max_lines) { + if (font.width(text, pixels) <= width) { lines.push_back(text); break; } + if (lines.size() + 1 == max_lines) { + while (!text.empty() && font.width(text + U"...", pixels) > width) text.pop_back(); + while (!text.empty() && text.back() == U' ') text.pop_back(); + lines.push_back(text + U"..."); + break; + } + // The longest run of words that fits; a single word too long for a line is split anywhere. + std::size_t cut = 0; + for (std::size_t i = 1; i <= text.size(); ++i) { + if (font.width(std::u32string_view(text).substr(0, i), pixels) > width) break; + if (i == text.size() || text[i] == U' ') cut = i; + } + if (!cut) { + cut = 1; + while (cut < text.size() && font.width(std::u32string_view(text).substr(0, cut + 1), pixels) <= width) ++cut; + } + lines.push_back(text.substr(0, cut)); + text.erase(0, cut); + while (!text.empty() && text.front() == U' ') text.erase(0, 1); + } + return lines; +} + +// Coverage (0..1) of a rounded rectangle at a pixel centre. +float rounded(float px, float py, float w, float h, float radius) { + const float dx = std::max({radius - px, px - (w - radius), 0.0f}); + const float dy = std::max({radius - py, py - (h - radius), 0.0f}); + return std::clamp(radius - std::sqrt(dx * dx + dy * dy) + 0.5f, 0.0f, 1.0f); +} +} // namespace + +Image render_parts_panel(std::string_view title, std::span parts, std::uint32_t height, const PanelFonts& fonts) { + if (height < 64) throw std::invalid_argument("The parts card is too small."); + const Font brush(fonts.brush), heading(fonts.heading), body(fonts.body); + const auto H = static_cast(height); + const auto width = static_cast(std::lround(H * 0.72f)); + const float pad = H * 0.07f, inner = static_cast(width) - pad * 2; + + // Lay the card out at full size; if it does not fit, again with everything smaller. + struct Line { const Font* font; std::u32string text; float x, y, size; Rgb colour; }; + std::vector lines; + float scale = 1.0f, bottom = 0; + for (int attempt = 0; attempt < 8; ++attempt) { + lines.clear(); + const float title_size = H * 0.11f * scale, label_size = H * 0.042f * scale, value_size = H * 0.058f * scale; + float y = pad; + auto title_text = decode(title); + while (!title_text.empty() && brush.width(title_text, title_size) > inner) title_text.pop_back(); + lines.push_back({&brush, title_text, pad, y, title_size, white}); + y += title_size * 1.05f + H * 0.035f * scale; // room for the accent rule + for (const auto& part : parts) { + std::string label = part.slot; + for (auto& c : label) c = static_cast(std::toupper(static_cast(c))); + lines.push_back({&body, decode(label), pad, y, label_size, grey}); + y += label_size * 1.25f; + const auto name = part.name.empty() ? std::string("None") : part.name; + for (auto& text : wrapped(heading, decode(name), value_size, inner, 2)) { + lines.push_back({&heading, std::move(text), pad, y, value_size, white}); + y += value_size * 1.1f; + } + y += H * 0.03f * scale; + } + bottom = y; + if (bottom <= H - pad * 0.5f) break; + scale *= 0.88f; + } + + Image out(width, height); + const float radius = H * 0.04f; + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + const float cover = rounded(static_cast(x) + 0.5f, static_cast(y) + 0.5f, static_cast(width), H, radius); + auto* p = out.at(x, y); + p[0] = card[0], p[1] = card[1], p[2] = card[2]; + p[3] = static_cast(std::lround(cover * card_alpha)); + } + // The accent rule under the title. + const auto& title_line = lines.front(); + const float rule_y = title_line.y + title_line.size * 1.05f + H * 0.008f; + const float rule_w = std::min(inner, std::max(H * 0.12f, brush.width(title_line.text, title_line.size))); + for (auto y = static_cast(rule_y); y < static_cast(rule_y + std::max(2.0f, H * 0.008f)); ++y) + for (auto x = static_cast(pad); x < static_cast(pad + rule_w); ++x) blend(out, x, y, accent, 255); + for (const auto& line : lines) line.font->draw(out, line.text, line.x, line.y, line.size, line.colour); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_panel.h b/Extension/Rendering/Capture/capture_panel.h new file mode 100644 index 00000000..9ee8d54f --- /dev/null +++ b/Extension/Rendering/Capture/capture_panel.h @@ -0,0 +1,23 @@ +#pragma once +#include "capture_image.h" +#include +#include + +// The optional card drawn beside what a capture shows: a heading, then each +// part's slot and item name, in the menu's fonts. +namespace dingosdk::capture { +struct Part { + std::string slot; // "Deck", "Grip tape", "Trucks", "Wheels", ... + std::string name; // the item's title as the game shows it (UTF-8) +}; +// TrueType font files (the menu's: ReSkate builds them in as FONT_HEADING / FONT_BODY / FONT_BRUSH). +struct PanelFonts { + std::span heading; // the item names + std::span body; // the slot labels + std::span brush; // the title, in the game's brush lettering +}; +// A card `height` pixels tall (and about three quarters as wide), on a dark, mostly opaque rounded +// panel, so it reads over a transparent background as well as a matte. +Image render_parts_panel(std::string_view title, std::span parts, std::uint32_t height, + const PanelFonts& fonts); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_png.cpp b/Extension/Rendering/Capture/capture_png.cpp new file mode 100644 index 00000000..1b534711 --- /dev/null +++ b/Extension/Rendering/Capture/capture_png.cpp @@ -0,0 +1,150 @@ +#include "capture_encode.h" +#include +#include +#include +#include +#include + +// PNG and APNG (https://www.w3.org/TR/png-3/#apng-frame-control) for captures. +namespace dingosdk::capture { +namespace { +std::uint32_t png_crc(std::span bytes, std::uint32_t crc = 0) noexcept { + static const auto table = [] { + std::array t{}; + for (std::uint32_t n = 0; n < 256; ++n) { + auto c = n; + for (int k = 0; k < 8; ++k) c = c & 1 ? 0xedb88320u ^ (c >> 1) : c >> 1; + t[n] = c; + } + return t; + }(); + crc = ~crc; + for (const auto b : bytes) crc = table[(crc ^ b) & 0xff] ^ (crc >> 8); + return ~crc; +} + +struct Writer { + std::vector out; + void u32(std::vector& to, std::uint32_t v) { + to.insert(to.end(), {static_cast(v >> 24), static_cast(v >> 16), + static_cast(v >> 8), static_cast(v)}); + } + void u16(std::vector& to, std::uint16_t v) { + to.insert(to.end(), {static_cast(v >> 8), static_cast(v)}); + } + void chunk(std::string_view type, std::span data) { + u32(out, static_cast(data.size())); + const auto start = out.size(); + out.insert(out.end(), type.begin(), type.end()); + out.insert(out.end(), data.begin(), data.end()); + u32(out, png_crc({out.data() + start, out.size() - start})); + } +}; + +// The image's rows, each with the PNG filter that leaves the smallest sum of magnitudes (the usual +// heuristic), compressed with zlib. +std::vector image_data(const Image& image) { + const std::size_t stride = std::size_t{image.width} * 4; + std::vector filtered; + filtered.reserve((stride + 1) * image.height); + std::vector zero(stride, 0), candidate(stride), best(stride); + for (std::uint32_t y = 0; y < image.height; ++y) { + const auto* row = image.rgba.data() + y * stride; + const auto* up = y ? row - stride : zero.data(); + std::uint64_t best_cost = ~0ull; + std::uint8_t best_filter = 0; + for (std::uint8_t filter = 0; filter < 5; ++filter) { + std::uint64_t cost = 0; + for (std::size_t i = 0; i < stride; ++i) { + const int a = i >= 4 ? row[i - 4] : 0, b = up[i], c = i >= 4 ? up[i - 4] : 0; + int predicted = 0; + switch (filter) { + case 1: predicted = a; break; + case 2: predicted = b; break; + case 3: predicted = (a + b) / 2; break; + case 4: { + const int p = a + b - c, pa = std::abs(p - a), pb = std::abs(p - b), pc = std::abs(p - c); + predicted = pa <= pb && pa <= pc ? a : pb <= pc ? b : c; + break; + } + default: break; + } + candidate[i] = static_cast(row[i] - predicted); + cost += static_cast(std::abs(static_cast(candidate[i]))); + } + if (cost < best_cost) best_cost = cost, best_filter = filter, best.swap(candidate); + } + filtered.push_back(best_filter); + filtered.insert(filtered.end(), best.begin(), best.end()); + } + std::size_t size = 0; + void* packed = tdefl_compress_mem_to_heap(filtered.data(), filtered.size(), &size, TDEFL_WRITE_ZLIB_HEADER | 256); + if (!packed) throw std::runtime_error("PNG compression failed."); + std::vector out(static_cast(packed), static_cast(packed) + size); + mz_free(packed); + return out; +} + +void header(Writer& w, const Image& image) { + static constexpr std::uint8_t signature[] = {0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}; + w.out.assign(std::begin(signature), std::end(signature)); + std::vector ihdr; + w.u32(ihdr, image.width); + w.u32(ihdr, image.height); + ihdr.insert(ihdr.end(), {8, 6, 0, 0, 0}); // 8-bit RGBA, deflate, adaptive filters, not interlaced + w.chunk("IHDR", ihdr); +} +} // namespace + +std::vector encode_png(const Image& image) { + if (image.empty()) throw std::invalid_argument("The picture is empty."); + Writer w; + header(w, image); + w.chunk("IDAT", image_data(image)); + w.chunk("IEND", {}); + return std::move(w.out); +} + +std::vector encode_apng(const Animation& animation) { + if (animation.frames.empty() || animation.frames.front().empty()) throw std::invalid_argument("No frames to save."); + const auto& first = animation.frames.front(); + for (const auto& frame : animation.frames) + if (frame.width != first.width || frame.height != first.height) + throw std::invalid_argument("The frames are not all the same size."); + Writer w; + header(w, first); + std::vector actl; + w.u32(actl, static_cast(animation.frames.size())); + w.u32(actl, 0); // loop forever + w.chunk("acTL", actl); + std::uint32_t sequence = 0; + const auto fps = static_cast(std::clamp(animation.fps, 1, 120)); + for (std::size_t i = 0; i < animation.frames.size(); ++i) { + const auto& frame = animation.frames[i]; + std::vector fctl; + w.u32(fctl, sequence++); + w.u32(fctl, frame.width); + w.u32(fctl, frame.height); + w.u32(fctl, 0); + w.u32(fctl, 0); + w.u16(fctl, 1); // delay: 1 / fps seconds + w.u16(fctl, fps); + // Each frame is whole: clear what the last one left (dispose to background) and replace, + // not blend, so see-through parts of the board stay see-through. + fctl.insert(fctl.end(), {1, 0}); + w.chunk("fcTL", fctl); + auto data = image_data(frame); + if (!i) { + w.chunk("IDAT", data); + continue; + } + std::vector fdat; + fdat.reserve(data.size() + 4); + w.u32(fdat, sequence++); + fdat.insert(fdat.end(), data.begin(), data.end()); + w.chunk("fdAT", fdat); + } + w.chunk("IEND", {}); + return std::move(w.out); +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_recorder.cpp b/Extension/Rendering/Capture/capture_recorder.cpp new file mode 100644 index 00000000..8a1945d7 --- /dev/null +++ b/Extension/Rendering/Capture/capture_recorder.cpp @@ -0,0 +1,465 @@ +#include "capture_recorder.h" +#include "capture_matte.h" +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::capture { +namespace { +// `area` of a frame, copied out in the frame's own channel order. +std::vector copy_area(const PixelView& frame, Rect area) { + std::vector out(std::size_t{area.width} * area.height * 4); + for (std::uint32_t y = 0; y < area.height; ++y) + std::copy_n(frame.data + std::size_t{area.y + y} * frame.pitch + std::size_t{area.x} * 4, std::size_t{area.width} * 4, + out.data() + std::size_t{y} * area.width * 4); + return out; +} + +Image opaque_image(const PixelView& view) { + Image out(view.width, view.height); + for (std::uint32_t y = 0; y < view.height; ++y) + for (std::uint32_t x = 0; x < view.width; ++x) { + const auto rgb = view.rgb(x, y); + auto* p = out.at(x, y); + p[0] = rgb[0], p[1] = rgb[1], p[2] = rgb[2], p[3] = 255; + } + return out; +} +} // namespace + +// What turns, by the game's depth: nearer than the backdrop behind it. The backdrop's distance for +// each row is that of the area's outer columns (the backdrop, or the floor where the row meets it: +// the camera is level, so the floor is about as far all along a row), and what is 8% nearer +// than that is the board or skater. The game's UI is never in the depth, and neither are a colour's +// likeness to the backdrop or the floor's reflection. Into `picture`'s alpha; false when the depth +// shows nothing there. +bool depth_mask(const PixelView& frame, Rect area, Image& picture) { + if (!frame.depth || picture.width != area.width || picture.height != area.height) return false; + const auto w = area.width, h = area.height; + const auto margin = std::max(2, w * 6 / 100); + std::vector edge; + float behind = 0; + std::size_t found = 0; + for (std::uint32_t y = 0; y < h; ++y) { + edge.clear(); + for (std::uint32_t x = 0; x < w; x = x + 1 == margin ? w - margin : x + 1) { + const float d = frame.depth_at(area.x + x, area.y + y); + if (d > 1e-7f) edge.push_back(1.0f / d); + } + if (!edge.empty()) { + std::nth_element(edge.begin(), edge.begin() + static_cast(edge.size() / 2), edge.end()); + behind = edge[edge.size() / 2]; + } + for (std::uint32_t x = 0; x < w; ++x) { + const float d = frame.depth_at(area.x + x, area.y + y); + const bool in_front = d > 1e-7f && (behind <= 0 || 1.0f / d < behind * 0.92f); + picture.at(x, y)[3] = in_front ? 255 : 0; + found += in_front; + } + } + // The soles: where the feet meet the floor the depth cannot tell them from it. Carried down from + // the lowest part of the skater in each column while the colour is unlike the floor's along that + // row (the area's outer columns again), a little way at most. + std::vector> floor_colour(h); + std::vector channel; + for (std::uint32_t y = 0; y < h; ++y) + for (int c = 0; c < 3; ++c) { + channel.clear(); + for (std::uint32_t x = 0; x < w; x = x + 1 == margin ? w - margin : x + 1) channel.push_back(picture.at(x, y)[c]); + std::nth_element(channel.begin(), channel.begin() + static_cast(channel.size() / 2), channel.end()); + floor_colour[y][static_cast(c)] = channel[channel.size() / 2]; + } + const auto reach = std::max(2, h / 25); + for (std::uint32_t x = margin; x + margin < w; ++x) { + std::uint32_t lowest = h; + for (std::uint32_t y = h; y-- > 0;) + if (picture.at(x, y)[3]) { lowest = y; break; } + if (lowest >= h) continue; + for (std::uint32_t y = lowest + 1; y < h && y <= lowest + reach; ++y) { + auto* p = picture.at(x, y); + int off = 0; + for (int c = 0; c < 3; ++c) off = std::max(off, std::abs(p[c] - floor_colour[y][static_cast(c)])); + if (off <= 24) break; + p[3] = 255; + } + } + return found > 64; +} + +// Keeps only what turned, and says where it is. What moves over the turn (opaque and unlike the +// first frame) is the subject; what is solid in any frame, changes at all, and is joined to that is +// the rest of it (the nose and tail sit on the axis the board turns about and change little); +// everything else in the area is the game's UI, and is cleared from every frame. A rim two pixels wide is kept round it +// for the soft edges. +// The pixels the game drew the same in every frame, as captured (before the backdrop is keyed out, +// which can shade a still pixel differently from frame to frame): the game's UI (a prompt, its +// leader line) stays exactly as it is, while every part of what turns, its nose and tail too, +// takes the light differently as it goes round. +std::vector unchanging(const std::vector& frames) { + const auto& first = frames.front(); + std::vector low(first.rgba), high(first.rgba), out(first.rgba.size() / 4); + for (const auto& frame : frames) + for (std::size_t i = 0; i < frame.rgba.size(); ++i) + low[i] = std::min(low[i], frame.rgba[i]), high[i] = std::max(high[i], frame.rgba[i]); + for (std::size_t p = 0; p < out.size(); ++p) { + int spread = 0; + for (std::size_t c = 0; c < 3; ++c) spread = std::max(spread, high[p * 4 + c] - low[p * 4 + c]); + out[p] = spread <= 3; + } + return out; +} + +Rect keep_moving(std::vector& frames, const std::vector& fixed) { + const auto& first = frames.front(); + const auto w = first.width, h = first.height; + std::vector moved(std::size_t{w} * h), solid(std::size_t{w} * h); + for (const auto& frame : frames) + for (std::size_t p = 0, i = 0; p < moved.size(); ++p, i += 4) { + const auto alpha = frame.rgba[i + 3]; + if (alpha >= 64 && !fixed[p]) solid[p] = 1; + if (moved[p] || fixed[p] || alpha < 16) continue; + int change = 0; + for (std::size_t c = 0; c < 4; ++c) change = std::max(change, std::abs(frame.rgba[i + c] - first.rgba[i + c])); + if (change > 24) moved[p] = 1; + } + // Flood from what moved over what is solid, one piece at a time: each piece's size (in pixels + // that moved) and bounds. + std::vector piece(moved.size()); // 0: none, else the piece's number + 1 + struct Piece { std::size_t moved{}; std::uint32_t x0{}, y0{}, x1{}, y1{}; }; + std::vector pieces; + std::vector todo; + for (std::uint32_t start = 0; start < moved.size(); ++start) { + if (!moved[start] || piece[start]) continue; + pieces.push_back({0, w, h, 0, 0}); + const auto label = static_cast(pieces.size()); + auto& info = pieces.back(); + piece[start] = label; + todo.push_back(start); + while (!todo.empty()) { + const auto p = todo.back(); + todo.pop_back(); + const auto x = static_cast(p % w), y = static_cast(p / w); + info.moved += moved[p]; + info.x0 = std::min(info.x0, static_cast(x)), info.x1 = std::max(info.x1, static_cast(x + 1)); + info.y0 = std::min(info.y0, static_cast(y)), info.y1 = std::max(info.y1, static_cast(y + 1)); + for (std::int64_t dy = -1; dy <= 1; ++dy) + for (std::int64_t dx = -1; dx <= 1; ++dx) { + const auto nx = x + dx, ny = y + dy; + if (nx < 0 || ny < 0 || nx >= w || ny >= h) continue; + const auto q = static_cast(ny * w + nx); + if (!piece[q] && (solid[q] || moved[q])) piece[q] = label, todo.push_back(q); + } + } + } + // What turns is the biggest piece, with any piece centred within its bounds (a board held + // apart from the hand); pieces elsewhere are the scene's own movement (a light, its haze). + std::vector kept(moved.size()); + if (!pieces.empty()) { + const auto biggest = static_cast(std::max_element(pieces.begin(), pieces.end(), [](const Piece& l, const Piece& r) { + return l.moved < r.moved; + }) - pieces.begin()); + const auto& big = pieces[biggest]; + std::vector keep(pieces.size()); + for (std::size_t i = 0; i < pieces.size(); ++i) { + const auto cx = (pieces[i].x0 + pieces[i].x1) / 2, cy = (pieces[i].y0 + pieces[i].y1) / 2; + keep[i] = i == biggest || (cx >= big.x0 && cx < big.x1 && cy >= big.y0 && cy < big.y1); + } + for (std::size_t p = 0; p < kept.size(); ++p) kept[p] = piece[p] && keep[piece[p] - 1]; + } + // The rim (of what changes: never a still prompt touching the edge). + for (int pass = 0; pass < 2; ++pass) { + auto grown = kept; + const auto add = [&](std::size_t q) { if (!fixed[q]) grown[q] = 1; }; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + if (!kept[std::size_t{y} * w + x]) continue; + if (x > 0) add(std::size_t{y} * w + x - 1); + if (x + 1 < w) add(std::size_t{y} * w + x + 1); + if (y > 0) add(std::size_t{y - 1} * w + x); + if (y + 1 < h) add(std::size_t{y + 1} * w + x); + } + kept = std::move(grown); + } + std::uint32_t x0 = w, y0 = h, x1 = 0, y1 = 0; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) + if (kept[std::size_t{y} * w + x] && solid[std::size_t{y} * w + x]) + x0 = std::min(x0, x), y0 = std::min(y0, y), x1 = std::max(x1, x + 1), y1 = std::max(y1, y + 1); + for (auto& frame : frames) + for (std::size_t p = 0; p < kept.size(); ++p) + if (!kept[p]) std::fill_n(frame.rgba.data() + p * 4, 4, std::uint8_t{0}); + if (x1 <= x0 || y1 <= y0) return {}; + return {x0, y0, x1 - x0, y1 - y0}; +} +// A 24x48 grey thumbnail of a frame's area, for telling how far round the board is and whether it +// has stopped. +std::vector thumbnail(const PixelView& frame, Rect area) { + constexpr std::uint32_t tw = 24, th = 48; + std::vector out(tw * th); + for (std::uint32_t ty = 0; ty < th; ++ty) + for (std::uint32_t tx = 0; tx < tw; ++tx) { + const auto x0 = tx * area.width / tw, x1 = std::max(x0 + 1, (tx + 1) * area.width / tw); + const auto y0 = ty * area.height / th, y1 = std::max(y0 + 1, (ty + 1) * area.height / th); + float sum = 0; + for (auto y = y0; y < y1; y += 2) + for (auto x = x0; x < x1; x += 2) { + const auto rgb = frame.rgb(area.x + x, area.y + y); + sum += 0.299f * rgb[0] + 0.587f * rgb[1] + 0.114f * rgb[2]; + } + out[ty * tw + tx] = sum / static_cast(((x1 - x0 + 1) / 2) * ((y1 - y0 + 1) / 2)); + } + return out; +} +float unlike(const std::vector& a, const std::vector& b) noexcept { + float sum = 0; + for (std::size_t i = 0; i < a.size() && i < b.size(); ++i) sum += std::abs(a[i] - b[i]); + return sum / static_cast(std::max(a.size(), 1)); +} +Rect area_on(const Area& a, std::uint32_t width, std::uint32_t height) noexcept { + const double frame_width = a.menu ? std::min(width, height * 21.0 / 9.0) : width; + const double frame_left = (width - frame_width) / 2; + const auto clamp01 = [](float v) { return std::clamp(static_cast(v), 0.0, 1.0); }; + const auto x0 = static_cast(frame_left + frame_width * clamp01(a.left)); + const auto x1 = static_cast(frame_left + frame_width * clamp01(a.right)); + const auto y0 = static_cast(height * clamp01(a.top)), y1 = static_cast(height * clamp01(a.bottom)); + if (x1 <= x0 || y1 <= y0) return {}; + return {x0, y0, std::min(x1, width) - x0, std::min(y1, height) - y0}; +} + +const char* extension(Format format) noexcept { + switch (format) { + case Format::apng: + case Format::png: return "png"; + case Format::webm: return "webm"; + default: return "gif"; + } +} + +bool Recorder::start(const RecordRequest& request, std::string& why) { + if (saving_.load()) { + why = "a capture is being saved"; + return false; + } + std::lock_guard lock(mutex_); + request_ = request; + request_.most = std::clamp(request.most, 1, 20000); + area_ = {}; + skip_ = request.skip; + stride_ = 1; + started_ = std::chrono::steady_clock::now(); + pictures_.clear(), kept_at_.clear(), kept_depth_.clear(), thumbnails_.clear(); + status_ = {State::recording, 0, 0, false, {}, {}}; + return true; +} + +void Recorder::stop() noexcept { + std::lock_guard lock(mutex_); + if (status_.state == State::recording) status_.state = State::stopped; +} + +void Recorder::discard() noexcept { + if (saving_.load()) return; + std::lock_guard lock(mutex_); + pictures_.clear(), pictures_.shrink_to_fit(); + kept_at_.clear(), kept_depth_.clear(), thumbnails_.clear(), thumbnails_.shrink_to_fit(); + status_ = {}; +} + +Status Recorder::status() const { + std::lock_guard lock(mutex_); + return status_; +} + +float Recorder::compare(std::uint32_t a, std::uint32_t b) const { + std::lock_guard lock(mutex_); + if (a >= thumbnails_.size() || b >= thumbnails_.size()) return -1; + return unlike(thumbnails_[a], thumbnails_[b]); +} + +std::vector Recorder::kept() const { + std::lock_guard lock(mutex_); + return kept_at_; +} + +bool Recorder::active() const noexcept { + std::lock_guard lock(mutex_); + return status_.state == State::recording; +} + +void Recorder::fail(std::string message) { + status_.state = State::failed; + status_.message = std::move(message); +} + +bool Recorder::wants_frame() noexcept { + std::lock_guard lock(mutex_); + if (status_.state != State::recording) return false; + // However it goes, a recording lets go within its time. + if (std::chrono::steady_clock::now() - started_ > std::chrono::duration(request_.seconds)) { + status_.state = State::stopped; + status_.message = "the recording reached its time"; + return false; + } + if (skip_) { + --skip_; + return false; + } + return true; +} + +void Recorder::take_frame(const PixelView& frame) noexcept { + try { + Rect area; + RecordRequest request; + { + std::lock_guard lock(mutex_); + if (status_.state != State::recording) return; + if (!frame.data || !frame.width || !frame.height) { + fail("the game's picture could not be read"); + return; + } + if (area_.empty()) { + area_ = area_on(request_.area, frame.width, frame.height); + if (area_.empty()) { + fail("the area is not on the screen"); + return; + } + } + area = area_; + request = request_; + } + // The heavy part outside the lock: the area copied, cut out, and made the size it is kept at. + auto small = thumbnail(frame, area); + const auto copy = copy_area(frame, area); + auto picture = opaque_image({copy.data(), area.width, area.height, area.width * 4, frame.bgra}); + const bool by_depth = request.depth && depth_mask(frame, area, picture); + if (!by_depth) + for (std::size_t i = 3; i < picture.rgba.size(); i += 4) picture.rgba[i] = 255; + if (request.height && request.height < picture.height) + picture = resized(picture, std::max(1, picture.width * request.height / picture.height), request.height); + const auto most = std::clamp(kept_bytes / std::max(picture.rgba.size(), 1), 60, most_kept); + std::lock_guard lock(mutex_); + if (status_.state != State::recording) return; + const auto n = static_cast(thumbnails_.size()); + thumbnails_.push_back(std::move(small)); + if (n % stride_ == 0) { + kept_at_.push_back(n); + kept_depth_.push_back(by_depth); + pictures_.push_back(std::move(picture)); + if (pictures_.size() > most) { // keep every other one, and from now on every other + std::size_t w = 0; + for (std::size_t r = 0; r < pictures_.size(); r += 2) + pictures_[w] = std::move(pictures_[r]), kept_depth_[w] = kept_depth_[r], kept_at_[w++] = kept_at_[r]; + pictures_.resize(w), kept_at_.resize(w), kept_depth_.resize(w); + stride_ *= 2; + } + } + status_.taken = n + 1; + status_.kept = static_cast(pictures_.size()); + status_.depth = !kept_depth_.empty() && std::all_of(kept_depth_.begin(), kept_depth_.end(), [](bool d) { return d; }); + if (status_.taken >= request_.most) status_.state = State::stopped; + } catch (const std::exception& error) { + std::lock_guard lock(mutex_); + fail(error.what()); + } catch (...) { + std::lock_guard lock(mutex_); + fail("the frame could not be kept"); + } +} + +bool Recorder::save(SaveRequest request, std::string& why) { + if (request.frames.empty() || request.frames.size() > most_saved) { + why = "a capture saves 1 to " + std::to_string(most_saved) + " frames"; + return false; + } + if (saving_.exchange(true)) { + why = "a capture is being saved"; + return false; + } + { + std::lock_guard lock(mutex_); + if (pictures_.empty()) { + saving_.store(false); + why = "nothing was recorded"; + return false; + } + status_.state = State::saving; + status_.message.clear(); + status_.file.clear(); + } + std::thread([this, request = std::move(request)]() mutable { encode(std::move(request)); }).detach(); + return true; +} + +void Recorder::encode(SaveRequest o) noexcept { + try { + std::vector frames; + bool by_depth{}; + { + std::lock_guard lock(mutex_); + // Each frame asked for: the nearest one kept. + for (const auto wanted : o.frames) { + std::size_t best = 0; + for (std::size_t k = 1; k < kept_at_.size(); ++k) + if (std::llabs(static_cast(kept_at_[k]) - wanted) < std::llabs(static_cast(kept_at_[best]) - wanted)) best = k; + frames.push_back(pictures_[best]); + } + by_depth = std::all_of(kept_depth_.begin(), kept_depth_.end(), [](bool d) { return d; }); + } + if (o.cutout == Cutout::depth && !by_depth) throw std::runtime_error("not every frame had the game's depth"); + const bool depth_cut = o.cutout == Cutout::depth || (o.cutout == Cutout::automatic && by_depth); + const bool colour_cut = o.cutout == Cutout::colour || (o.cutout == Cutout::automatic && !by_depth); + if (!depth_cut) + for (auto& frame : frames) + for (std::size_t i = 3; i < frame.rgba.size(); i += 4) frame.rgba[i] = 255; + Rect through{0, 0, frames.front().width, frames.front().height}; + if (o.cutout != Cutout::none) { + const auto fixed = depth_cut ? std::vector(frames.front().rgba.size() / 4) : unchanging(frames); + if (colour_cut) key_against_backdrop(frames, o.solid); + // A still cut out by depth: the depth alone says what it is (nothing moves in one frame). + if (depth_cut && frames.size() < 2) through = content_bounds(frames.front()); + else through = keep_moving(frames, fixed); + if (through.empty()) throw std::runtime_error("nothing that moved was found in the area"); + } + const auto margin = std::max(through.width, through.height) / 25 + 2; + const auto size = std::clamp(o.size, 64, 2048); + std::optional card; + if (!o.card.empty() && !o.fonts.heading.empty()) card = render_parts_panel(o.card_title, o.card, size, o.fonts); + Animation animation; + animation.fps = std::clamp(o.fps, 1, 60); + for (auto& frame : frames) { + auto shown = fitted(square_around(frame, through, margin), size); + frame = {}; + if (card) shown = side_by_side(shown, *card); + if (o.background) shown = over_matte(shown, *o.background); + animation.frames.push_back(std::move(shown)); + } + std::vector bytes; + switch (o.format) { + case Format::apng: bytes = encode_apng(animation); break; + case Format::png: bytes = encode_png(animation.frames.front()); break; + case Format::webm: bytes = encode_webm(animation, {.quality = 1, .alpha_quality = 0}); break; // the highest VP8 has + default: bytes = encode_gif(animation, {.fringe = o.background.value_or(Rgb{24, 25, 28}), .dither = true}); break; + } + std::filesystem::create_directories(o.file.parent_path()); + std::ofstream out(o.file, std::ios::binary); + out.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (!out) throw std::runtime_error("the file could not be written"); + std::lock_guard lock(mutex_); + status_.state = State::saved; + status_.file = o.file; + status_.message = depth_cut ? "cut out by depth" : colour_cut ? "cut out by colour" : "not cut out"; + } catch (const std::exception& error) { + std::lock_guard lock(mutex_); + fail(std::string("not saved: ") + error.what()); + } catch (...) { + std::lock_guard lock(mutex_); + fail("not saved"); + } + saving_.store(false); +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_recorder.h b/Extension/Rendering/Capture/capture_recorder.h new file mode 100644 index 00000000..a2bacfdc --- /dev/null +++ b/Extension/Rendering/Capture/capture_recorder.h @@ -0,0 +1,117 @@ +#pragma once +#include "capture_encode.h" +#include "capture_panel.h" +#include +#include +#include +#include +#include +#include +#include + +// A recording of part of the screen as the game draws it, for reskate.capture: every presented frame +// of an area, read back before ReSkate's own UI goes on (the overlay's frame grab), cut out by the +// game's depth where it has one, with a small grey thumbnail of each frame to compare them by; then +// chosen frames saved as a GIF, APNG, PNG or WebM, on a thread of its own. Ported from Fit Check +// (pull request 326's turnaround_session.cpp), whose turn finding is now the plugin's. No Windows or +// game code: the overlay hands frames to take_frame (render thread), the plugin side asks the rest +// (any thread). +namespace dingosdk::capture { +// Where on screen: fractions of the whole screen, or of the game's menu frame (`menu`: the whole +// width up to 21:9, a 21:9 frame in the middle of anything wider), left to right and top to bottom. +struct Area { + float left{}, top{}, right{1}, bottom{1}; + bool menu{}; +}; +Rect area_on(const Area& area, std::uint32_t width, std::uint32_t height) noexcept; + +enum class Format { gif, apng, png, webm }; +const char* extension(Format format) noexcept; + +struct RecordRequest { + Area area; + bool depth{true}; // cut each frame out by the game's depth where it has one + std::uint32_t height{}; // each kept picture this tall at most (0: as on screen) + std::uint32_t skip{}; // presented frames let go first + std::uint32_t most{2400}; // frames taken at most; then it stops by itself + double seconds{60}; // and after this long +}; +// Pictures and bytes held at once, at most: past them every other kept frame is let go, and from +// then on only every other frame taken is kept (and so on). +inline constexpr std::size_t most_kept = 480; +inline constexpr std::size_t kept_bytes = std::size_t{1000} << 20; +inline constexpr std::uint32_t most_saved = 360; // frames in one saved file +inline constexpr std::uint32_t thumbnail_width = 24, thumbnail_height = 48; + +enum class State { idle, recording, stopped, saving, saved, failed }; +struct Status { + State state{State::idle}; + std::uint32_t taken{}, kept{}; + bool depth{}; // every kept frame was cut out by the game's depth + std::string message; // what went wrong, or what was saved + std::filesystem::path file; +}; + +enum class Cutout { automatic, depth, colour, none }; +struct SaveRequest { + std::vector frames; // frames taken (from 0), each saved as the nearest one kept + Format format{Format::gif}; + std::uint32_t fps{30}; + std::uint32_t size{512}; // the square what was recorded is fitted into, in pixels + std::optional background; // flattened over it; none: transparent + Cutout cutout{Cutout::automatic}; // automatic: by depth when every frame had it, else by colour + bool solid{true}; // a hole what turns closes all round is part of it (a board), or only a small one (a skater) + std::string card_title; // a card beside it: this title, then each line (none without lines) + std::vector card; + PanelFonts fonts; // for the card; must outlive the save + std::filesystem::path file; // where it is written (its folder is made) +}; + +class Recorder { +public: + // A new recording, letting go of the last one; false (and why) while one is being saved. + bool start(const RecordRequest& request, std::string& why); + // No more frames; what was taken is kept for compare, kept and save. + void stop() noexcept; + // Lets go of everything taken. + void discard() noexcept; + Status status() const; + // How unlike two frames taken look: the mean difference of their thumbnails' grey levels + // (0..255), or a negative number when either is not known. + float compare(std::uint32_t a, std::uint32_t b) const; + // The frames still held, by the number they were taken as, in order. + std::vector kept() const; + // Saves on a thread of its own (the recording stops first); false (and why) when there is + // nothing to save or a save is going on. + bool save(SaveRequest request, std::string& why); + + // Render thread, every presented frame. + bool active() const noexcept; + bool wants_frame() noexcept; + void take_frame(const PixelView& frame) noexcept; + +private: + void fail(std::string message); + void encode(SaveRequest request) noexcept; + + mutable std::mutex mutex_; + Status status_; + RecordRequest request_; + Rect area_; + std::uint32_t skip_{}, stride_{1}; + std::chrono::steady_clock::time_point started_{}; + std::vector pictures_; + std::vector kept_at_; + std::vector kept_depth_; + std::vector> thumbnails_; // every frame taken + std::atomic saving_{false}; +}; + +// The pieces of saving, for the tests: what turns cut out by depth into `picture`'s alpha (false when +// the depth shows nothing), a frame's grey thumbnail, and only what moved across the frames kept. +bool depth_mask(const PixelView& frame, Rect area, Image& picture); +std::vector thumbnail(const PixelView& frame, Rect area); +float unlike(const std::vector& a, const std::vector& b) noexcept; +std::vector unchanging(const std::vector& frames); +Rect keep_moving(std::vector& frames, const std::vector& fixed); +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_runtime.cpp b/Extension/Rendering/Capture/capture_runtime.cpp new file mode 100644 index 00000000..ac4cde8c --- /dev/null +++ b/Extension/Rendering/Capture/capture_runtime.cpp @@ -0,0 +1,34 @@ +#include "capture_runtime.h" +#include "Extension/UI/Overlay/overlay.h" +#include + +namespace dingosdk::capture { +namespace { +std::span built_in(const wchar_t* name) { + HMODULE module{}; + if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&built_in), &module)) + return {}; + const auto resource = FindResourceW(module, name, MAKEINTRESOURCEW(10) /* RT_RCDATA */); + const auto handle = resource ? LoadResource(module, resource) : nullptr; + const auto* data = handle ? static_cast(LockResource(handle)) : nullptr; + return data ? std::span(data, SizeofResource(module, resource)) : std::span{}; +} +bool grab_active() noexcept { return recorder().active(); } +bool grab_wants() noexcept { return recorder().wants_frame(); } +void grab_take(const overlay::GrabbedFrame& frame) noexcept { + recorder().take_frame({frame.data, frame.width, frame.height, frame.pitch, frame.bgra, frame.depth, frame.depth_pitch}); +} +constexpr overlay::FrameGrabHost frame_grab_host{&grab_active, &grab_wants, &grab_take}; +} // namespace + +Recorder& recorder() noexcept { + static auto* value = new Recorder; // process lifetime: its saving thread may outlive statics + return *value; +} +const overlay::FrameGrabHost* frame_grab() noexcept { return &frame_grab_host; } +PanelFonts menu_fonts() noexcept { + static const PanelFonts fonts{built_in(L"FONT_HEADING"), built_in(L"FONT_BODY"), built_in(L"FONT_BRUSH")}; + return fonts; +} +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_runtime.h b/Extension/Rendering/Capture/capture_runtime.h new file mode 100644 index 00000000..81970ab1 --- /dev/null +++ b/Extension/Rendering/Capture/capture_runtime.h @@ -0,0 +1,13 @@ +#pragma once +#include "capture_recorder.h" + +namespace dingosdk::overlay { struct FrameGrabHost; } + +// The process's one screen recorder (reskate.capture), fed by the overlay's frame grab, and the menu's +// fonts its cards are drawn in (built into ReSkate.dll as FONT_HEADING, FONT_BODY and FONT_BRUSH). +namespace dingosdk::capture { +Recorder& recorder() noexcept; +// What the overlay asks before drawing each frame (overlay::set_frame_grab); process lifetime. +const overlay::FrameGrabHost* frame_grab() noexcept; +PanelFonts menu_fonts() noexcept; +} // namespace dingosdk::capture diff --git a/Extension/Rendering/Capture/capture_vp8.cpp b/Extension/Rendering/Capture/capture_vp8.cpp new file mode 100644 index 00000000..dab9db78 --- /dev/null +++ b/Extension/Rendering/Capture/capture_vp8.cpp @@ -0,0 +1,458 @@ +#include "capture_vp8.h" +#include "capture_vp8_tables.h" +#include +#include +#include +#include +#include + +// Prediction and reconstruction follow libvpx's decoder exactly (vp8/decoder/decodeframe.c, +// vp8/common/idctllm.c, reconintra.c): the encoder predicts from what the decoder will have, so +// any difference would build up across the frame. +namespace dingosdk::capture::vp8 { +namespace { +// --- the boolean entropy coder (RFC 6386, section 7.3) +struct BoolEncoder { + std::vector out; + std::uint32_t range = 255, bottom = 0; + int bit_count = 24; + void carry() { + for (auto i = out.size(); i-- > 0;) { + if (out[i] == 255) { out[i] = 0; continue; } + ++out[i]; + break; + } + } + void put(bool bit, int probability) { + const std::uint32_t split = 1 + (((range - 1) * static_cast(probability)) >> 8); + if (bit) bottom += split, range -= split; + else range = split; + while (range < 128) { + range <<= 1; + if (bottom & (1u << 31)) carry(); + bottom <<= 1; + if (!--bit_count) { + out.push_back(static_cast(bottom >> 24)); + bottom &= (1u << 24) - 1; + bit_count = 8; + } + } + } + void literal(std::uint32_t value, int bits) { + for (int b = bits - 1; b >= 0; --b) put((value >> b) & 1, 128); + } + std::vector finish() { + for (int i = 0; i < 32; ++i) put(false, 128); // as libvpx's vp8_stop_encode + return std::move(out); + } +}; + +enum Mode { dc_pred, v_pred, h_pred, tm_pred }; +constexpr std::array zigzag{0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15}; +constexpr std::array bands{0, 1, 2, 3, 6, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 0}; +constexpr std::uint8_t pcat3[] = {173, 148, 140}, pcat4[] = {176, 155, 140, 135}, pcat5[] = {180, 157, 141, 134, 130}, + pcat6[] = {254, 254, 243, 230, 196, 177, 153, 140, 133, 130, 129}; +constexpr std::array kf_ymode_prob{145, 156, 163, 128}; +constexpr std::array kf_uv_mode_prob{142, 114, 183}; +enum BlockType { y_after_y2 = 0, y2 = 1, chroma = 2 }; + +const std::uint8_t* probabilities(int type, int band, int context) { + return default_coef_probs + ((type * 8 + band) * 3 + context) * 11; +} + +// One block's tokens, its quantized coefficients in raster order; the "has coefficients" flag +// the neighbours' contexts take. +bool put_tokens(BoolEncoder& e, const std::array& raster, int type, int first, int context) { + int last = -1; + for (int i = first; i < 16; ++i) + if (raster[static_cast(zigzag[static_cast(i)])]) last = i; + const auto* p = probabilities(type, bands[static_cast(first)], context); + if (last < 0) { + e.put(false, p[0]); + return false; + } + e.put(true, p[0]); + for (int i = first; i <= last; ++i) { + const int v = raster[static_cast(zigzag[static_cast(i)])], a = std::abs(v); + const auto next = static_cast(i + 1); + if (!a) { + e.put(false, p[1]); + p = probabilities(type, bands[next], 0); + continue; + } + e.put(true, p[1]); + int next_context = 2; + if (a == 1) { + e.put(false, p[2]); + next_context = 1; + } else { + e.put(true, p[2]); + if (a <= 4) { + e.put(false, p[3]); + if (a == 2) e.put(false, p[4]); + else e.put(true, p[4]), e.put(a == 4, p[5]); + } else { + e.put(true, p[3]); + if (a <= 10) { + e.put(false, p[6]); + if (a <= 6) e.put(false, p[7]), e.put(a == 6, 159); + else { + e.put(true, p[7]); + const int extra = a - 7; + e.put((extra >> 1) & 1, 165); + e.put(extra & 1, 145); + } + } else { + e.put(true, p[6]); + const int category = a <= 18 ? 0 : a <= 34 ? 1 : a <= 66 ? 2 : 3; + static constexpr int base[] = {11, 19, 35, 67}, bits[] = {3, 4, 5, 11}; + static constexpr const std::uint8_t* table[] = {pcat3, pcat4, pcat5, pcat6}; + e.put(category >> 1, p[8]); + e.put(category & 1, p[9 + (category >> 1)]); + const int extra = a - base[category]; + for (int b = bits[category] - 1, k = 0; b >= 0; --b, ++k) e.put((extra >> b) & 1, table[category][k]); + } + } + } + e.put(v < 0, 128); + if (next == 16) break; + p = probabilities(type, bands[next], next_context); + e.put(i != last, p[0]); // more to come, or the end of the block + } + return true; +} + +// --- transforms (libvpx: vp8/encoder/dct.c, vp8/common/idctllm.c) +void forward_dct(const short* in, short* out) { // in: 4x4 raster + short temp[16]; + for (int i = 0; i < 4; ++i) { + const short* ip = in + i * 4; + short* op = temp + i * 4; + const int a1 = (ip[0] + ip[3]) * 8, b1 = (ip[1] + ip[2]) * 8, c1 = (ip[1] - ip[2]) * 8, d1 = (ip[0] - ip[3]) * 8; + op[0] = static_cast(a1 + b1); + op[2] = static_cast(a1 - b1); + op[1] = static_cast((c1 * 2217 + d1 * 5352 + 14500) >> 12); + op[3] = static_cast((d1 * 2217 - c1 * 5352 + 7500) >> 12); + } + for (int i = 0; i < 4; ++i) { + const short* ip = temp + i; + short* op = out + i; + const int a1 = ip[0] + ip[12], b1 = ip[4] + ip[8], c1 = ip[4] - ip[8], d1 = ip[0] - ip[12]; + op[0] = static_cast((a1 + b1 + 7) >> 4); + op[8] = static_cast((a1 - b1 + 7) >> 4); + op[4] = static_cast(((c1 * 2217 + d1 * 5352 + 12000) >> 16) + (d1 != 0)); + op[12] = static_cast((d1 * 2217 - c1 * 5352 + 51000) >> 16); + } +} +void forward_walsh(const short* in, short* out) { + short temp[16]; + for (int i = 0; i < 4; ++i) { + const short* ip = in + i * 4; + short* op = temp + i * 4; + const int a1 = (ip[0] + ip[2]) * 4, d1 = (ip[1] + ip[3]) * 4, c1 = (ip[1] - ip[3]) * 4, b1 = (ip[0] - ip[2]) * 4; + op[0] = static_cast(a1 + d1 + (a1 != 0)); + op[1] = static_cast(b1 + c1); + op[2] = static_cast(b1 - c1); + op[3] = static_cast(a1 - d1); + } + for (int i = 0; i < 4; ++i) { + const short* ip = temp + i; + short* op = out + i; + const int a1 = ip[0] + ip[8], d1 = ip[4] + ip[12], c1 = ip[4] - ip[12], b1 = ip[0] - ip[8]; + int a2 = a1 + d1, b2 = b1 + c1, c2 = b1 - c1, d2 = a1 - d1; + a2 += a2 < 0, b2 += b2 < 0, c2 += c2 < 0, d2 += d2 < 0; + op[0] = static_cast((a2 + 3) >> 3); + op[4] = static_cast((b2 + 3) >> 3); + op[8] = static_cast((c2 + 3) >> 3); + op[12] = static_cast((d2 + 3) >> 3); + } +} +void inverse_walsh(const int* in, int* out) { + int temp[16]; + for (int i = 0; i < 4; ++i) { + const int* ip = in + i; + int* op = temp + i; + const int a1 = ip[0] + ip[12], b1 = ip[4] + ip[8], c1 = ip[4] - ip[8], d1 = ip[0] - ip[12]; + op[0] = static_cast(a1 + b1); + op[4] = static_cast(c1 + d1); + op[8] = static_cast(a1 - b1); + op[12] = static_cast(d1 - c1); + } + for (int i = 0; i < 4; ++i) { + const int* ip = temp + i * 4; + const int a1 = ip[0] + ip[3], b1 = ip[1] + ip[2], c1 = ip[1] - ip[2], d1 = ip[0] - ip[3]; + out[i * 4 + 0] = static_cast((a1 + b1 + 3) >> 3); + out[i * 4 + 1] = static_cast((c1 + d1 + 3) >> 3); + out[i * 4 + 2] = static_cast((a1 - b1 + 3) >> 3); + out[i * 4 + 3] = static_cast((d1 - c1 + 3) >> 3); + } +} +// Adds the inverse DCT of dequantized `in` (raster) to the prediction in place. +void inverse_dct_add(const int* in, std::uint8_t* dst, std::size_t stride) { + constexpr int cospi8sqrt2minus1 = 20091, sinpi8sqrt2 = 35468; + short temp[16]; + for (int i = 0; i < 4; ++i) { + const short ip0 = static_cast(in[i]), ip4 = static_cast(in[4 + i]), ip8 = static_cast(in[8 + i]), + ip12 = static_cast(in[12 + i]); + const int a1 = ip0 + ip8, b1 = ip0 - ip8; + int t1 = (ip4 * sinpi8sqrt2) >> 16, t2 = ip12 + ((ip12 * cospi8sqrt2minus1) >> 16); + const int c1 = t1 - t2; + t1 = ip4 + ((ip4 * cospi8sqrt2minus1) >> 16); + t2 = (ip12 * sinpi8sqrt2) >> 16; + const int d1 = t1 + t2; + temp[i] = static_cast(a1 + d1); + temp[12 + i] = static_cast(a1 - d1); + temp[4 + i] = static_cast(b1 + c1); + temp[8 + i] = static_cast(b1 - c1); + } + for (int r = 0; r < 4; ++r) { + const short* ip = temp + r * 4; + const int a1 = ip[0] + ip[2], b1 = ip[0] - ip[2]; + int t1 = (ip[1] * sinpi8sqrt2) >> 16, t2 = ip[3] + ((ip[3] * cospi8sqrt2minus1) >> 16); + const int c1 = t1 - t2; + t1 = ip[1] + ((ip[1] * cospi8sqrt2minus1) >> 16); + t2 = (ip[3] * sinpi8sqrt2) >> 16; + const int d1 = t1 + t2; + const int row[4] = {static_cast((a1 + d1 + 4) >> 3), static_cast((b1 + c1 + 4) >> 3), + static_cast((b1 - c1 + 4) >> 3), static_cast((a1 - d1 + 4) >> 3)}; + for (int c = 0; c < 4; ++c) { + auto& d = dst[static_cast(r) * stride + static_cast(c)]; + d = static_cast(std::clamp(row[c] + d, 0, 255)); + } + } +} +void dc_only_add(int dc, std::uint8_t* dst, std::size_t stride) { + const int a1 = (dc + 4) >> 3; + for (std::size_t r = 0; r < 4; ++r) + for (std::size_t c = 0; c < 4; ++c) dst[r * stride + c] = static_cast(std::clamp(a1 + dst[r * stride + c], 0, 255)); +} + +int quantize(int coefficient, int step, bool dc) { + // Rounding with a slight dead zone for AC coefficients: fewer stray small ones to code. + const int a = std::abs(coefficient), bias = dc ? step / 2 : step * 3 / 8; + const int q = std::min((a + bias) / step, 2048); + return coefficient < 0 ? -q : q; +} + +// A padded plane and its reconstruction (what the decoder will see), with VP8's frame borders: +// 127 above the frame (top-left corner included), 129 left of it. +struct Plane { + std::size_t width{}, height{}; + std::vector source, recon; + Plane(const std::vector& from, std::size_t w, std::size_t h, std::size_t padded_w, std::size_t padded_h) + : width(padded_w), height(padded_h), source(padded_w * padded_h), recon(padded_w * padded_h) { + for (std::size_t y = 0; y < padded_h; ++y) + for (std::size_t x = 0; x < padded_w; ++x) source[y * padded_w + x] = from[std::min(y, h - 1) * w + std::min(x, w - 1)]; + } + int above(std::size_t x, std::size_t y) const { return y ? recon[(y - 1) * width + x] : 127; } + int left(std::size_t x, std::size_t y) const { return x ? recon[y * width + x - 1] : 129; } + int corner(std::size_t x, std::size_t y) const { return !y ? 127 : !x ? 129 : recon[(y - 1) * width + x - 1]; } + // The `size` x `size` prediction at (x, y). + void predict(std::size_t x0, std::size_t y0, std::size_t size, Mode mode, std::uint8_t* out) const { + for (std::size_t r = 0; r < size; ++r) + for (std::size_t c = 0; c < size; ++c) { + int v = 128; + switch (mode) { + case v_pred: v = above(x0 + c, y0); break; + case h_pred: v = left(x0, y0 + r); break; + case tm_pred: v = std::clamp(left(x0, y0 + r) + above(x0 + c, y0) - corner(x0, y0), 0, 255); break; + case dc_pred: { + int sum = 0, count = 0; + if (y0) for (std::size_t i = 0; i < size; ++i) sum += above(x0 + i, y0), ++count; + if (x0) for (std::size_t i = 0; i < size; ++i) sum += left(x0, y0 + i), ++count; + v = count ? (sum + count / 2) / count : 128; + break; + } + } + out[r * size + c] = static_cast(v); + } + } + // The mode whose prediction is closest to the source. + Mode best_mode(std::size_t x0, std::size_t y0, std::size_t size) const { + std::vector guess(size * size); + Mode best = dc_pred; + std::uint64_t best_error = std::numeric_limits::max(); + for (const Mode mode : {dc_pred, v_pred, h_pred, tm_pred}) { + predict(x0, y0, size, mode, guess.data()); + std::uint64_t error = 0; + for (std::size_t r = 0; r < size; ++r) + for (std::size_t c = 0; c < size; ++c) { + const int d = source[(y0 + r) * width + x0 + c] - guess[r * size + c]; + error += static_cast(d * d); + } + if (error < best_error) best_error = error, best = mode; + } + return best; + } + // Writes the prediction into the reconstruction and returns the 4x4 residual blocks (raster + // order of blocks, each raster order) of the source against it. + std::vector> residual(std::size_t x0, std::size_t y0, std::size_t size, Mode mode) { + std::vector guess(size * size); + predict(x0, y0, size, mode, guess.data()); + const std::size_t per_row = size / 4; + std::vector> blocks(per_row * per_row); + for (std::size_t r = 0; r < size; ++r) + for (std::size_t c = 0; c < size; ++c) { + recon[(y0 + r) * width + x0 + c] = guess[r * size + c]; + blocks[(r / 4) * per_row + c / 4][(r % 4) * 4 + c % 4] = + static_cast(source[(y0 + r) * width + x0 + c] - guess[r * size + c]); + } + return blocks; + } + std::uint8_t* at(std::size_t x, std::size_t y) { return recon.data() + y * width + x; } +}; + +void put_y_mode(BoolEncoder& e, Mode mode) { + e.put(true, kf_ymode_prob[0]); // not B_PRED + const bool second = mode == h_pred || mode == tm_pred; + e.put(second, kf_ymode_prob[1]); + if (second) e.put(mode == tm_pred, kf_ymode_prob[3]); + else e.put(mode == v_pred, kf_ymode_prob[2]); +} +void put_uv_mode(BoolEncoder& e, Mode mode) { + e.put(mode != dc_pred, kf_uv_mode_prob[0]); + if (mode == dc_pred) return; + e.put(mode != v_pred, kf_uv_mode_prob[1]); + if (mode != v_pred) e.put(mode == tm_pred, kf_uv_mode_prob[2]); +} + +// Token contexts: whether the block above / to the left had coefficients. Per macroblock column +// (above) or for the current row (left): Y 0..3, U 4..5, V 6..7, Y2 8. +using Contexts = std::array; +} // namespace + +std::vector encode_keyframe(const Planes& planes, int quality) { + const auto w = planes.width, h = planes.height; + if (!w || !h || w > 16383 || h > 16383) throw std::invalid_argument("VP8 frame size out of range."); + const std::size_t cw = (w + 1) / 2, ch = (h + 1) / 2; + if (planes.y.size() < std::size_t{w} * h || planes.u.size() < cw * ch || planes.v.size() < cw * ch) + throw std::invalid_argument("VP8 planes too small."); + const std::size_t mb_cols = (w + 15) / 16, mb_rows = (h + 15) / 16; + Plane Y(planes.y, w, h, mb_cols * 16, mb_rows * 16), U(planes.u, cw, ch, mb_cols * 8, mb_rows * 8), + V(planes.v, cw, ch, mb_cols * 8, mb_rows * 8); + const int qi = std::clamp(quality, 0, 127); + const int y1_dc = dc_qlookup[qi], y1_ac = ac_qlookup[qi], y2_dc = dc_qlookup[qi] * 2, + y2_ac = std::max((ac_qlookup[qi] * 101581) >> 16, 8), uv_dc = std::min(dc_qlookup[qi], 132), + uv_ac = ac_qlookup[qi]; + + BoolEncoder header, tokens; + header.put(false, 128); // colour space + header.put(false, 128); // clamping required + header.put(false, 128); // no segmentation + header.put(false, 128); // normal loop filter... + header.literal(0, 6); // ...at level 0: off + header.literal(0, 3); // sharpness + header.put(false, 128); // no loop filter deltas + header.literal(0, 2); // one token partition + header.literal(static_cast(qi), 7); + for (int i = 0; i < 5; ++i) header.put(false, 128); // no quantizer deltas + header.put(false, 128); // refresh_entropy_probs: keep the defaults + for (const auto probability : coef_update_probs) header.put(false, probability); // no probability updates + header.put(false, 128); // mb_no_coeff_skip off: every macroblock has tokens + + std::vector above(mb_cols); + for (std::size_t my = 0; my < mb_rows; ++my) { + Contexts left{}; + for (std::size_t mx = 0; mx < mb_cols; ++mx) { + auto& top = above[mx]; + const std::size_t x = mx * 16, y = my * 16, cx = mx * 8, cy = my * 8; + const Mode y_mode = Y.best_mode(x, y, 16); + // Chroma: one mode for U and V, chosen on U and V together. + Mode uv_mode = dc_pred; + { + std::uint64_t best = std::numeric_limits::max(); + std::array gu{}, gv{}; + for (const Mode mode : {dc_pred, v_pred, h_pred, tm_pred}) { + U.predict(cx, cy, 8, mode, gu.data()); + V.predict(cx, cy, 8, mode, gv.data()); + std::uint64_t error = 0; + for (std::size_t r = 0; r < 8; ++r) + for (std::size_t c = 0; c < 8; ++c) { + const int du = U.source[(cy + r) * U.width + cx + c] - gu[r * 8 + c]; + const int dv = V.source[(cy + r) * V.width + cx + c] - gv[r * 8 + c]; + error += static_cast(du * du + dv * dv); + } + if (error < best) best = error, uv_mode = mode; + } + } + put_y_mode(header, y_mode); + put_uv_mode(header, uv_mode); + + // Luma: 16 blocks, their DCs gathered into the second-order (Y2) block. + const auto y_res = Y.residual(x, y, 16, y_mode); + std::array, 16> y_coeff{}; + std::array dcs{}, y2_coeff{}; + for (std::size_t b = 0; b < 16; ++b) { + forward_dct(y_res[b].data(), y_coeff[b].data()); + dcs[b] = y_coeff[b][0]; + } + forward_walsh(dcs.data(), y2_coeff.data()); + std::array y2_q{}; + for (std::size_t i = 0; i < 16; ++i) y2_q[i] = quantize(y2_coeff[i], i ? y2_ac : y2_dc, true); + left[8] = top[8] = put_tokens(tokens, y2_q, y2, 0, top[8] + left[8]); + // The DCs the decoder will have, from the quantized Y2. + std::array y2_dq{}, block_dc{}; + bool y2_has_ac = false; + for (std::size_t i = 1; i < 16; ++i) y2_has_ac |= y2_q[static_cast(zigzag[i])] != 0; + for (std::size_t i = 0; i < 16; ++i) y2_dq[i] = y2_q[i] * (i ? y2_ac : y2_dc); + if (y2_has_ac) inverse_walsh(y2_dq.data(), block_dc.data()); + else block_dc.fill(static_cast((y2_dq[0] + 3) >> 3)); + for (std::size_t b = 0; b < 16; ++b) { + std::array q{}; + bool has_ac = false; + for (std::size_t i = 1; i < 16; ++i) has_ac |= (q[i] = quantize(y_coeff[b][i], y1_ac, false)) != 0; + const std::size_t col = b % 4, row = b / 4; + left[row] = top[col] = put_tokens(tokens, q, y_after_y2, 1, top[col] + left[row]); + auto* dst = Y.at(x + col * 4, y + row * 4); + if (has_ac) { + std::array dq{}; + dq[0] = block_dc[b]; + for (std::size_t i = 1; i < 16; ++i) dq[i] = q[i] * y1_ac; + inverse_dct_add(dq.data(), dst, Y.width); + } else { + dc_only_add(block_dc[b], dst, Y.width); + } + } + (void)y1_dc; // luma DCs go through Y2; the first-order DC step is unused without B_PRED + + // Chroma: four 4x4 blocks each for U and V. + for (int plane = 0; plane < 2; ++plane) { + auto& P = plane ? V : U; + const auto res = P.residual(cx, cy, 8, uv_mode); + for (std::size_t b = 0; b < 4; ++b) { + std::array coeff{}; + forward_dct(res[b].data(), coeff.data()); + std::array q{}; + bool beyond_first = false; + for (std::size_t i = 0; i < 16; ++i) q[i] = quantize(coeff[i], i ? uv_ac : uv_dc, i == 0); + for (std::size_t i = 1; i < 16; ++i) beyond_first |= q[static_cast(zigzag[i])] != 0; + const std::size_t col = b % 2, row = b / 2, slot = 4 + static_cast(plane) * 2; + left[slot + row] = top[slot + col] = put_tokens(tokens, q, chroma, 0, top[slot + col] + left[slot + row]); + auto* dst = P.at(cx + col * 4, cy + row * 4); + if (beyond_first) { + std::array dq{}; + for (std::size_t i = 0; i < 16; ++i) dq[i] = q[i] * (i ? uv_ac : uv_dc); + inverse_dct_add(dq.data(), dst, P.width); + } else { + dc_only_add(q[0] * uv_dc, dst, P.width); + } + } + } + } + } + + const auto first = header.finish(); + const auto second = tokens.finish(); + if (first.size() >= (1u << 19)) throw std::runtime_error("VP8 header partition too large."); + std::vector out; + out.reserve(10 + first.size() + second.size()); + const std::uint32_t tag = 0 /* key frame */ | (0 << 1) /* version */ | (1 << 4) /* shown */ | + (static_cast(first.size()) << 5); + out.insert(out.end(), {static_cast(tag), static_cast(tag >> 8), static_cast(tag >> 16), + 0x9d, 0x01, 0x2a, static_cast(w), static_cast(w >> 8), + static_cast(h), static_cast(h >> 8)}); + out.insert(out.end(), first.begin(), first.end()); + out.insert(out.end(), second.begin(), second.end()); + return out; +} +} // namespace dingosdk::capture::vp8 diff --git a/Extension/Rendering/Capture/capture_vp8.h b/Extension/Rendering/Capture/capture_vp8.h new file mode 100644 index 00000000..9cd1c5f7 --- /dev/null +++ b/Extension/Rendering/Capture/capture_vp8.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +// A small VP8 encoder (RFC 6386) for the captures' WebM files: every frame a key frame, 16x16 +// intra prediction only, one token partition, no loop filter. Simple rather than small, which +// suits a few seconds of a turning board. +namespace dingosdk::capture::vp8 { +// 8-bit 4:2:0: `y` is width x height, `u` and `v` are half that, rounded up. +struct Planes { + std::uint32_t width{}, height{}; + std::vector y, u, v; +}; +// One key frame. `quality` is VP8's quantizer index, 0 (best) .. 127. +std::vector encode_keyframe(const Planes& planes, int quality); +} // namespace dingosdk::capture::vp8 diff --git a/Extension/Rendering/Capture/capture_vp8_tables.h b/Extension/Rendering/Capture/capture_vp8_tables.h new file mode 100644 index 00000000..92d9e6c0 --- /dev/null +++ b/Extension/Rendering/Capture/capture_vp8_tables.h @@ -0,0 +1,226 @@ +#pragma once +#include + +// VP8 constants (RFC 6386) for the captures' WebM writer, taken from libvpx: +// vp8/common/coefupdateprobs.h, vp8/common/default_coef_probs.h, vp8/common/quant_common.c. +// Copyright (c) 2010 The WebM project authors. All Rights Reserved. BSD-style license: +// https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/LICENSE +namespace dingosdk::capture::vp8 { +// [block type 4][band 8][context 3][node 11], flattened. +inline constexpr std::uint8_t coef_update_probs[1056] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 176, 246, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 223, 241, 252, 255, 255, 255, 255, 255, 255, 255, 255, + 249, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 244, 252, 255, 255, 255, 255, 255, 255, 255, 255, + 234, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 246, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 239, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 248, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 251, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 251, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 253, 255, 254, 255, 255, 255, 255, 255, 255, + 250, 255, 254, 255, 254, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 217, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 225, 252, 241, 253, 255, 255, 254, 255, 255, 255, 255, + 234, 250, 241, 250, 253, 255, 253, 254, 255, 255, 255, + 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 223, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 238, 253, 254, 254, 255, 255, 255, 255, 255, 255, 255, + 255, 248, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 249, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 247, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 252, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 250, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 186, 251, 250, 255, 255, 255, 255, 255, 255, 255, 255, + 234, 251, 244, 254, 255, 255, 255, 255, 255, 255, 255, + 251, 251, 243, 253, 254, 255, 254, 255, 255, 255, 255, + 255, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 236, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 251, 253, 253, 254, 254, 255, 255, 255, 255, 255, 255, + 255, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 248, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 250, 254, 252, 254, 255, 255, 255, 255, 255, 255, 255, + 248, 254, 249, 253, 255, 255, 255, 255, 255, 255, 255, + 255, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 246, 253, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 252, 254, 251, 254, 254, 255, 255, 255, 255, 255, 255, + 255, 254, 252, 255, 255, 255, 255, 255, 255, 255, 255, + 248, 254, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 255, 254, 254, 255, 255, 255, 255, 255, 255, 255, + 255, 251, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 245, 251, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 253, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 251, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 252, 253, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 252, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 249, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 254, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 253, 255, 255, 255, 255, 255, 255, 255, 255, + 250, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +}; +inline constexpr std::uint8_t default_coef_probs[1056] = { + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 253, 136, 254, 255, 228, 219, 128, 128, 128, 128, 128, + 189, 129, 242, 255, 227, 213, 255, 219, 128, 128, 128, + 106, 126, 227, 252, 214, 209, 255, 255, 128, 128, 128, + 1, 98, 248, 255, 236, 226, 255, 255, 128, 128, 128, + 181, 133, 238, 254, 221, 234, 255, 154, 128, 128, 128, + 78, 134, 202, 247, 198, 180, 255, 219, 128, 128, 128, + 1, 185, 249, 255, 243, 255, 128, 128, 128, 128, 128, + 184, 150, 247, 255, 236, 224, 128, 128, 128, 128, 128, + 77, 110, 216, 255, 236, 230, 128, 128, 128, 128, 128, + 1, 101, 251, 255, 241, 255, 128, 128, 128, 128, 128, + 170, 139, 241, 252, 236, 209, 255, 255, 128, 128, 128, + 37, 116, 196, 243, 228, 255, 255, 255, 128, 128, 128, + 1, 204, 254, 255, 245, 255, 128, 128, 128, 128, 128, + 207, 160, 250, 255, 238, 128, 128, 128, 128, 128, 128, + 102, 103, 231, 255, 211, 171, 128, 128, 128, 128, 128, + 1, 152, 252, 255, 240, 255, 128, 128, 128, 128, 128, + 177, 135, 243, 255, 234, 225, 128, 128, 128, 128, 128, + 80, 129, 211, 255, 194, 224, 128, 128, 128, 128, 128, + 1, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 246, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 255, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 198, 35, 237, 223, 193, 187, 162, 160, 145, 155, 62, + 131, 45, 198, 221, 172, 176, 220, 157, 252, 221, 1, + 68, 47, 146, 208, 149, 167, 221, 162, 255, 223, 128, + 1, 149, 241, 255, 221, 224, 255, 255, 128, 128, 128, + 184, 141, 234, 253, 222, 220, 255, 199, 128, 128, 128, + 81, 99, 181, 242, 176, 190, 249, 202, 255, 255, 128, + 1, 129, 232, 253, 214, 197, 242, 196, 255, 255, 128, + 99, 121, 210, 250, 201, 198, 255, 202, 128, 128, 128, + 23, 91, 163, 242, 170, 187, 247, 210, 255, 255, 128, + 1, 200, 246, 255, 234, 255, 128, 128, 128, 128, 128, + 109, 178, 241, 255, 231, 245, 255, 255, 128, 128, 128, + 44, 130, 201, 253, 205, 192, 255, 255, 128, 128, 128, + 1, 132, 239, 251, 219, 209, 255, 165, 128, 128, 128, + 94, 136, 225, 251, 218, 190, 255, 255, 128, 128, 128, + 22, 100, 174, 245, 186, 161, 255, 199, 128, 128, 128, + 1, 182, 249, 255, 232, 235, 128, 128, 128, 128, 128, + 124, 143, 241, 255, 227, 234, 128, 128, 128, 128, 128, + 35, 77, 181, 251, 193, 211, 255, 205, 128, 128, 128, + 1, 157, 247, 255, 236, 231, 255, 255, 128, 128, 128, + 121, 141, 235, 255, 225, 227, 255, 255, 128, 128, 128, + 45, 99, 188, 251, 195, 217, 255, 224, 128, 128, 128, + 1, 1, 251, 255, 213, 255, 128, 128, 128, 128, 128, + 203, 1, 248, 255, 255, 128, 128, 128, 128, 128, 128, + 137, 1, 177, 255, 224, 255, 128, 128, 128, 128, 128, + 253, 9, 248, 251, 207, 208, 255, 192, 128, 128, 128, + 175, 13, 224, 243, 193, 185, 249, 198, 255, 255, 128, + 73, 17, 171, 221, 161, 179, 236, 167, 255, 234, 128, + 1, 95, 247, 253, 212, 183, 255, 255, 128, 128, 128, + 239, 90, 244, 250, 211, 209, 255, 255, 128, 128, 128, + 155, 77, 195, 248, 188, 195, 255, 255, 128, 128, 128, + 1, 24, 239, 251, 218, 219, 255, 205, 128, 128, 128, + 201, 51, 219, 255, 196, 186, 128, 128, 128, 128, 128, + 69, 46, 190, 239, 201, 218, 255, 228, 128, 128, 128, + 1, 191, 251, 255, 255, 128, 128, 128, 128, 128, 128, + 223, 165, 249, 255, 213, 255, 128, 128, 128, 128, 128, + 141, 124, 248, 255, 255, 128, 128, 128, 128, 128, 128, + 1, 16, 248, 255, 255, 128, 128, 128, 128, 128, 128, + 190, 36, 230, 255, 236, 255, 128, 128, 128, 128, 128, + 149, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 1, 226, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 247, 192, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 240, 128, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 1, 134, 252, 255, 255, 128, 128, 128, 128, 128, 128, + 213, 62, 250, 255, 255, 128, 128, 128, 128, 128, 128, + 55, 93, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 202, 24, 213, 235, 186, 191, 220, 160, 240, 175, 255, + 126, 38, 182, 232, 169, 184, 228, 174, 255, 187, 128, + 61, 46, 138, 219, 151, 178, 240, 170, 255, 216, 128, + 1, 112, 230, 250, 199, 191, 247, 159, 255, 255, 128, + 166, 109, 228, 252, 211, 215, 255, 174, 128, 128, 128, + 39, 77, 162, 232, 172, 180, 245, 178, 255, 255, 128, + 1, 52, 220, 246, 198, 199, 249, 220, 255, 255, 128, + 124, 74, 191, 243, 183, 193, 250, 221, 255, 255, 128, + 24, 71, 130, 219, 154, 170, 243, 182, 255, 255, 128, + 1, 182, 225, 249, 219, 240, 255, 224, 128, 128, 128, + 149, 150, 226, 252, 216, 205, 255, 171, 128, 128, 128, + 28, 108, 170, 242, 183, 194, 254, 223, 255, 255, 128, + 1, 81, 230, 252, 204, 203, 255, 192, 128, 128, 128, + 123, 102, 209, 247, 188, 196, 255, 233, 128, 128, 128, + 20, 95, 153, 243, 164, 173, 255, 203, 128, 128, 128, + 1, 222, 248, 255, 216, 213, 128, 128, 128, 128, 128, + 168, 175, 246, 252, 235, 205, 255, 255, 128, 128, 128, + 47, 116, 215, 255, 211, 212, 255, 255, 128, 128, 128, + 1, 121, 236, 253, 212, 214, 255, 255, 128, 128, 128, + 141, 84, 213, 252, 201, 202, 255, 219, 128, 128, 128, + 42, 80, 160, 240, 162, 185, 255, 205, 128, 128, 128, + 1, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 244, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, + 238, 1, 255, 128, 128, 128, 128, 128, 128, 128, 128, +}; +inline constexpr std::uint16_t dc_qlookup[128] = { + 4, 5, 6, 7, 8, 9, 10, 10, 11, 12, 13, 14, 15, 16, 17, 17, + 18, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 25, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 91, 93, 95, 96, 98, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, + 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 143, 145, 148, 151, 154, 157, +}; +inline constexpr std::uint16_t ac_qlookup[128] = { + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, + 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, + 110, 112, 114, 116, 119, 122, 125, 128, 131, 134, 137, 140, 143, 146, 149, 152, + 155, 158, 161, 164, 167, 170, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, + 213, 217, 221, 225, 229, 234, 239, 245, 249, 254, 259, 264, 269, 274, 279, 284, +}; +} // namespace dingosdk::capture::vp8 diff --git a/Extension/Rendering/Capture/capture_webm.cpp b/Extension/Rendering/Capture/capture_webm.cpp new file mode 100644 index 00000000..8b1b9835 --- /dev/null +++ b/Extension/Rendering/Capture/capture_webm.cpp @@ -0,0 +1,183 @@ +#include "capture_encode.h" +#include "capture_vp8.h" +#include +#include +#include +#include + +// WebM (Matroska) with VP8 video, and transparency the way Chrome, Firefox and ffmpeg read it: the +// alpha as a second VP8 picture (its luma) in each frame's BlockAdditional, the track marked with +// AlphaMode 1. +namespace dingosdk::capture { +namespace { +// EBML element ids and sizes. +void put_id(std::vector& out, std::uint32_t id) { + int bytes = id > 0xffffff ? 4 : id > 0xffff ? 3 : id > 0xff ? 2 : 1; + while (bytes--) out.push_back(static_cast(id >> (bytes * 8))); +} +// Always 8 bytes: sizes are known, and fixed-width ones keep this simple. +void put_size(std::vector& out, std::uint64_t size) { + out.push_back(0x01); + for (int i = 6; i >= 0; --i) out.push_back(static_cast(size >> (i * 8))); +} +void element(std::vector& out, std::uint32_t id, const std::vector& body) { + put_id(out, id); + put_size(out, body.size()); + out.insert(out.end(), body.begin(), body.end()); +} +void uint_element(std::vector& out, std::uint32_t id, std::uint64_t value) { + std::vector body; + int bytes = 1; + while (bytes < 8 && (value >> (bytes * 8))) ++bytes; + for (int i = bytes - 1; i >= 0; --i) body.push_back(static_cast(value >> (i * 8))); + element(out, id, body); +} +void string_element(std::vector& out, std::uint32_t id, std::string_view text) { + element(out, id, std::vector(text.begin(), text.end())); +} +void float_element(std::vector& out, std::uint32_t id, double value) { + std::uint64_t bits; + std::memcpy(&bits, &value, 8); + std::vector body; + for (int i = 7; i >= 0; --i) body.push_back(static_cast(bits >> (i * 8))); + element(out, id, body); +} + +// BT.601 limited range, the colour VP8 decoders assume. +vp8::Planes to_planes(const Image& image, bool alpha) { + vp8::Planes planes; + planes.width = image.width; + planes.height = image.height; + const std::size_t cw = (image.width + 1) / 2, ch = (image.height + 1) / 2; + planes.y.resize(std::size_t{image.width} * image.height); + planes.u.assign(cw * ch, 128); + planes.v.assign(cw * ch, 128); + for (std::uint32_t y = 0; y < image.height; ++y) + for (std::uint32_t x = 0; x < image.width; ++x) { + const auto* p = image.at(x, y); + planes.y[std::size_t{y} * image.width + x] = + alpha ? p[3] : static_cast(std::lround(16 + 0.2568 * p[0] + 0.5041 * p[1] + 0.0979 * p[2])); + } + if (alpha) return planes; + for (std::size_t cy = 0; cy < ch; ++cy) + for (std::size_t cx = 0; cx < cw; ++cx) { + double r = 0, g = 0, b = 0; + int n = 0; + for (std::size_t dy = 0; dy < 2; ++dy) + for (std::size_t dx = 0; dx < 2; ++dx) { + const auto x = cx * 2 + dx, y = cy * 2 + dy; + if (x >= image.width || y >= image.height) continue; + const auto* p = image.at(static_cast(x), static_cast(y)); + r += p[0], g += p[1], b += p[2], ++n; + } + r /= n, g /= n, b /= n; + planes.u[cy * cw + cx] = static_cast(std::clamp(std::lround(128 - 0.1482 * r - 0.2910 * g + 0.4392 * b), 16L, 240L)); + planes.v[cy * cw + cx] = static_cast(std::clamp(std::lround(128 + 0.4392 * r - 0.3678 * g - 0.0714 * b), 16L, 240L)); + } + return planes; +} + +// Fully transparent pixels take the colour of the nearest opaque ones along their row, so VP8's +// blocks do not smear an arbitrary hidden colour into the board's edge. +Image bleed_colour(const Image& image) { + Image out = image; + for (std::uint32_t y = 0; y < out.height; ++y) { + std::int64_t last = -1; + for (std::uint32_t x = 0; x < out.width; ++x) { + if (out.at(x, y)[3]) { + if (last < 0) for (std::uint32_t k = 0; k < x; ++k) std::copy_n(out.at(x, y), 3, out.at(k, y)); + last = x; + } else if (last >= 0) { + std::copy_n(out.at(static_cast(last), y), 3, out.at(x, y)); + } + } + } + return out; +} +} // namespace + +std::vector encode_webm(const Animation& animation, const WebmOptions& options) { + if (animation.frames.empty() || animation.frames.front().empty()) throw std::invalid_argument("No frames to save."); + const auto width = animation.frames.front().width, height = animation.frames.front().height; + for (const auto& frame : animation.frames) + if (frame.width != width || frame.height != height) throw std::invalid_argument("The frames are not all the same size."); + bool transparent = false; + for (const auto& frame : animation.frames) + for (std::size_t i = 3; i < frame.rgba.size() && !transparent; i += 4) transparent = frame.rgba[i] != 255; + const auto fps = std::clamp(animation.fps, 1, 120); + + std::vector out; + { + std::vector ebml; + uint_element(ebml, 0x4286, 1); // EBMLVersion + uint_element(ebml, 0x42f7, 1); // EBMLReadVersion + uint_element(ebml, 0x42f2, 4); // EBMLMaxIDLength + uint_element(ebml, 0x42f3, 8); // EBMLMaxSizeLength + string_element(ebml, 0x4282, "webm"); + uint_element(ebml, 0x4287, 4); // DocTypeVersion (BlockAdditions, AlphaMode) + uint_element(ebml, 0x4285, 2); // DocTypeReadVersion + element(out, 0x1a45dfa3, ebml); + } + std::vector segment; + { + std::vector info; + uint_element(info, 0x2ad7b1, 1000000); // TimestampScale: milliseconds + float_element(info, 0x4489, 1000.0 * static_cast(animation.frames.size()) / fps); // Duration + string_element(info, 0x4d80, "ReSkate Fit Check"); + string_element(info, 0x5741, "ReSkate Fit Check"); + element(segment, 0x1549a966, info); + } + { + std::vector video, track, tracks; + uint_element(video, 0xb0, width); + uint_element(video, 0xba, height); + if (transparent) uint_element(video, 0x53c0, 1); // AlphaMode + uint_element(track, 0xd7, 1); // TrackNumber + uint_element(track, 0x73c5, 1); // TrackUID + uint_element(track, 0x83, 1); // TrackType: video + uint_element(track, 0x9c, 0); // FlagLacing + uint_element(track, 0x23e383, 1000000000ull / fps); // DefaultDuration (ns) + string_element(track, 0x86, "V_VP8"); + element(track, 0xe0, video); + element(tracks, 0xae, track); + element(segment, 0x1654ae6b, tracks); + } + // Clusters of at most 20 seconds, so the 16-bit block timestamps stay in range. + std::vector cluster; + std::uint64_t cluster_start = 0; + const auto close_cluster = [&] { + if (!cluster.empty()) element(segment, 0x1f43b675, cluster); + cluster.clear(); + }; + for (std::size_t i = 0; i < animation.frames.size(); ++i) { + const std::uint64_t time = i * 1000 / fps; + if (cluster.empty() || time - cluster_start >= 20000) { + close_cluster(); + cluster_start = time; + uint_element(cluster, 0xe7, cluster_start); // Timestamp + } + const auto& frame = animation.frames[i]; + const auto picture = vp8::encode_keyframe(to_planes(transparent ? bleed_colour(frame) : frame, false), options.quality); + const auto relative = static_cast(time - cluster_start); + std::vector block{0x81, static_cast(static_cast(relative) >> 8), + static_cast(relative), 0x00}; + block.insert(block.end(), picture.begin(), picture.end()); + if (!transparent) { + // A SimpleBlock, flagged as a key frame. + block[3] = 0x80; + element(cluster, 0xa3, block); + continue; + } + std::vector group, additions, more; + element(group, 0xa1, block); // Block + uint_element(more, 0xee, 1); // BlockAddID + element(more, 0xa5, vp8::encode_keyframe(to_planes(frame, true), options.alpha_quality)); // BlockAdditional + element(additions, 0xa6, more); + element(group, 0x75a1, additions); + element(cluster, 0xa0, group); + } + close_cluster(); + element(out, 0x18538067, segment); + return out; +} +} // namespace dingosdk::capture diff --git a/Extension/UI/Overlay/depth_probe.cpp b/Extension/UI/Overlay/depth_probe.cpp new file mode 100644 index 00000000..003cb714 --- /dev/null +++ b/Extension/UI/Overlay/depth_probe.cpp @@ -0,0 +1,520 @@ +#include "overlay_internal.h" +#include "Engine/Core/Log/logging.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Research aid for screen captures (`capture depth`; Fit Check's `fitcheck depth`): where the game keeps its depth while it draws a +// frame, for cutting the board or skater out exactly. Each time a depth buffer the size of the +// screen (or near it) stops being written to over the next two frames (a transition barrier out of +// DEPTH_WRITE), a copy of it is put on that same command list, just before the barrier; a few frames +// later the copies are read back and written to Captures\depth as 16-bit PGM pictures (near +// and far stretched to black and white), with a log of what each was. +// +// ID3D12GraphicsCommandList::ResourceBarrier (slot 26) is hooked once, when the overlay starts +// (through its own command list: every list of the runtime shares it). Unarmed, the hook only +// passes the call on. +namespace dingosdk::overlay { +namespace { +using ResourceBarrierFn = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList*, UINT, const D3D12_RESOURCE_BARRIER*); +std::atomic original_barrier{}; + +struct Snapshot { + Microsoft::WRL::ComPtr readback; + D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout{}; + D3D12_RESOURCE_DESC desc{}; + D3D12_RESOURCE_STATES after{}; + std::uintptr_t resource{}; + unsigned frame{}, order{}; +}; +struct Probe { + std::mutex mutex; + std::atomic armed{}; // presents left in the window (frames recorded then are probed) + std::atomic settle{}; // presents to wait after the window before reading back + std::atomic requested{}; + std::filesystem::path folder; + std::uint32_t screen_width{}, screen_height{}; + unsigned frame{}, order{}; + std::vector snapshots; + // Readback buffers made when the probe starts, on the overlay's device, each big enough for a + // screen-sized depth plane: making them in the hook, on the game's list's device, failed. + std::vector> spare; + UINT64 spare_size{}; + unsigned barriers{}, depth_barriers{}, skipped_small{}, skipped_type{}; + std::vector notes; // each depth barrier, and why one was not copied (the first few) + std::vector seen; // each depth resource's description, once +}; +Probe& probe() { static auto* value = new Probe; return *value; } + +bool depth_capable(const D3D12_RESOURCE_DESC& d) { + return d.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE2D && (d.Flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL) && d.SampleDesc.Count == 1; +} + +// The depth tap: while a capture records (reskate.capture), the game's main depth buffer (the screen's size, the +// first one each frame that goes from being written to being read, found with `capture depth`, +// 2026-10-21) is copied each frame into one of a few readback buffers, with a number after it so +// that the copy the GPU has made can be told from one still waiting. +struct TapSlot { + Microsoft::WRL::ComPtr readback, marker; + std::uint32_t* marker_cpu{}; + std::uint32_t seq{}; + D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout{}; + UINT64 total{}; +}; +struct Tap { + std::mutex mutex; + std::atomic on{}; + std::vector slots; + std::size_t next{}; + std::uint32_t seq{}; + std::uint64_t epoch{}, copied_epoch{~std::uint64_t{0}}; + std::uint32_t width{}, height{}; + UINT64 slot_size{}; + unsigned idle{}; // presents since a grab last wanted it + TapSlot* mapped{}; // the copy handed out by depth_tap_latest + bool said{}; +}; +Tap& tap() { static auto* value = new Tap; return *value; } + +constexpr std::size_t tap_slots = 4; + +void tap_barrier(ID3D12GraphicsCommandList* list, const D3D12_RESOURCE_BARRIER& b, ResourceBarrierFn original) { + auto& t = tap(); + if (b.Type != D3D12_RESOURCE_BARRIER_TYPE_TRANSITION || !b.Transition.pResource) return; + if (!(b.Transition.StateBefore & D3D12_RESOURCE_STATE_DEPTH_WRITE)) return; + constexpr auto read = D3D12_RESOURCE_STATE_DEPTH_READ | D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE | D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE; + if (!(b.Transition.StateAfter & read)) return; + if (b.Flags & D3D12_RESOURCE_BARRIER_FLAG_END_ONLY) return; + if (b.Transition.Subresource != D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES && b.Transition.Subresource != 0) return; + std::lock_guard lock(t.mutex); + if (t.slots.empty() || t.copied_epoch == t.epoch) return; + const auto desc = b.Transition.pResource->GetDesc(); + if (!depth_capable(desc) || desc.Width != t.width || desc.Height != t.height || desc.DepthOrArraySize != 1) return; + if (desc.Format != DXGI_FORMAT_R32G8X24_TYPELESS && desc.Format != DXGI_FORMAT_D32_FLOAT_S8X24_UINT && + desc.Format != DXGI_FORMAT_R32_TYPELESS && desc.Format != DXGI_FORMAT_D32_FLOAT) + return; + if (list->GetType() != D3D12_COMMAND_LIST_TYPE_DIRECT) return; + auto& slot = t.slots[t.next]; + t.next = (t.next + 1) % t.slots.size(); + slot.layout = {}; + slot.layout.Footprint.Format = DXGI_FORMAT_R32_TYPELESS; + slot.layout.Footprint.Width = static_cast(desc.Width); + slot.layout.Footprint.Height = desc.Height; + slot.layout.Footprint.Depth = 1; + slot.layout.Footprint.RowPitch = (static_cast(desc.Width) * 4 + 255) / 256 * 256; + slot.total = std::uint64_t{slot.layout.Footprint.RowPitch} * desc.Height; + if (slot.total + 4 > t.slot_size) return; + D3D12_RESOURCE_BARRIER ours[2]{}; + ours[0].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + ours[0].Transition.pResource = b.Transition.pResource; + ours[0].Transition.Subresource = b.Transition.Subresource; + ours[0].Transition.StateBefore = b.Transition.StateBefore; + ours[0].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + ours[1] = ours[0]; + std::swap(ours[1].Transition.StateBefore, ours[1].Transition.StateAfter); + original(list, 1, &ours[0]); + D3D12_TEXTURE_COPY_LOCATION to{}, from{}; + to.pResource = slot.readback.Get(); + to.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + to.PlacedFootprint = slot.layout; + from.pResource = b.Transition.pResource; + from.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + from.SubresourceIndex = 0; + list->CopyTextureRegion(&to, 0, 0, 0, &from, nullptr); + original(list, 1, &ours[1]); + // The number, after the copy on the same list: when it is in the readback buffer, so is the copy. + slot.seq = ++t.seq; + *slot.marker_cpu = slot.seq; + list->CopyBufferRegion(slot.readback.Get(), slot.total, slot.marker.Get(), 0, 4); + t.copied_epoch = t.epoch; +} + +void STDMETHODCALLTYPE hooked_barrier(ID3D12GraphicsCommandList* list, UINT count, const D3D12_RESOURCE_BARRIER* barriers) { + const auto original = original_barrier.load(); + if (tap().on.load(std::memory_order_relaxed) && list && barriers) { + try { + for (UINT i = 0; i < count; ++i) tap_barrier(list, barriers[i], original); + } catch (...) {} + } + auto& p = probe(); + if (p.armed.load(std::memory_order_relaxed) > 0 && list && barriers) { + try { + std::lock_guard lock(p.mutex); + p.barriers += count; + for (UINT i = 0; i < count; ++i) { + const auto& b = barriers[i]; + if (b.Type != D3D12_RESOURCE_BARRIER_TYPE_TRANSITION || !b.Transition.pResource) continue; + if (!(b.Transition.StateBefore & D3D12_RESOURCE_STATE_DEPTH_WRITE)) continue; + const auto desc = b.Transition.pResource->GetDesc(); + if (!depth_capable(desc)) continue; + ++p.depth_barriers; + const auto key = reinterpret_cast(b.Transition.pResource); + const auto line = std::format("{:#x} {}x{} format {} mips {} array {} flags {:#x}", key, desc.Width, desc.Height, + static_cast(desc.Format), desc.MipLevels, desc.DepthOrArraySize, + static_cast(desc.Flags)); + if (std::find(p.seen.begin(), p.seen.end(), line) == p.seen.end() && p.seen.size() < 64) p.seen.push_back(line); + // Only near the screen's size, on a direct list, whole or its first subresource. + const auto note = [&](const char* what) { + if (p.notes.size() < 80) + p.notes.push_back(std::format("frame {} {:#x} sub {:#x} {:#x}->{:#x} flags {:#x}: {}", p.frame, key, b.Transition.Subresource, + static_cast(b.Transition.StateBefore), static_cast(b.Transition.StateAfter), + static_cast(b.Flags), what)); + }; + if (desc.Width * 3 < std::uint64_t{p.screen_width} || desc.Height * 3 < p.screen_height) { ++p.skipped_small; continue; } + if (list->GetType() != D3D12_COMMAND_LIST_TYPE_DIRECT) { ++p.skipped_type; continue; } + // A split barrier's start leaves the buffer as it is until then (copied before it); its + // end finds it already on its way. + if (b.Flags & D3D12_RESOURCE_BARRIER_FLAG_END_ONLY) { note("end of a split barrier, not copied"); continue; } + if (b.Transition.Subresource != D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES && b.Transition.Subresource != 0) { note("another plane, not copied"); continue; } + if (desc.DepthOrArraySize > 1) { note("an array (shadow cascades), not copied"); continue; } + if (std::any_of(p.snapshots.begin(), p.snapshots.end(), [&](const Snapshot& taken) { return taken.resource == key; })) { + note("copied already"); + continue; + } + if (p.spare.empty()) { note("no readback buffer left"); continue; } + Microsoft::WRL::ComPtr device; + if (FAILED(list->GetDevice(IID_PPV_ARGS(&device)))) { note("no device"); continue; } + Snapshot snap; + UINT rows{}; + UINT64 row_bytes{}, total{}; + device->GetCopyableFootprints(&desc, 0, 1, 0, &snap.layout, &rows, &row_bytes, &total); + // The runtime turns down the game's depth buffers (flag 0x400, newer than it knows) and + // says so with a total of all ones: the depth plane is laid out by hand then, four bytes + // a pixel, rows 256-byte aligned. + const bool four_byte_depth = desc.Format == DXGI_FORMAT_R32G8X24_TYPELESS || desc.Format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT || + desc.Format == DXGI_FORMAT_R32_TYPELESS || desc.Format == DXGI_FORMAT_D32_FLOAT || + desc.Format == DXGI_FORMAT_R24G8_TYPELESS || desc.Format == DXGI_FORMAT_D24_UNORM_S8_UINT; + if ((total == 0 || total == ~UINT64{0}) && four_byte_depth) { + snap.layout = {}; + snap.layout.Footprint.Format = desc.Format == DXGI_FORMAT_R24G8_TYPELESS || desc.Format == DXGI_FORMAT_D24_UNORM_S8_UINT + ? DXGI_FORMAT_R24G8_TYPELESS + : DXGI_FORMAT_R32_TYPELESS; + snap.layout.Footprint.Width = static_cast(desc.Width); + snap.layout.Footprint.Height = desc.Height; + snap.layout.Footprint.Depth = 1; + snap.layout.Footprint.RowPitch = (static_cast(desc.Width) * 4 + 255) / 256 * 256; + total = std::uint64_t{snap.layout.Footprint.RowPitch} * desc.Height; + } + if (!total || total == ~UINT64{0}) { note("no copyable footprint"); continue; } + if (total > p.spare_size) { + note(std::format("bigger than the readback buffers ({} bytes)", total).c_str()); + continue; + } + snap.readback = std::move(p.spare.back()); + p.spare.pop_back(); + // DEPTH_WRITE -> COPY_SOURCE, the copy, and back, before the game's own barrier. + D3D12_RESOURCE_BARRIER ours[2]{}; + ours[0].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + ours[0].Transition.pResource = b.Transition.pResource; + ours[0].Transition.Subresource = b.Transition.Subresource; + ours[0].Transition.StateBefore = b.Transition.StateBefore; + ours[0].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + ours[1] = ours[0]; + std::swap(ours[1].Transition.StateBefore, ours[1].Transition.StateAfter); + original(list, 1, &ours[0]); + D3D12_TEXTURE_COPY_LOCATION to{}, from{}; + to.pResource = snap.readback.Get(); + to.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + to.PlacedFootprint = snap.layout; + from.pResource = b.Transition.pResource; + from.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + from.SubresourceIndex = 0; + list->CopyTextureRegion(&to, 0, 0, 0, &from, nullptr); + original(list, 1, &ours[1]); + snap.desc = desc; + snap.after = b.Transition.StateAfter; + snap.resource = key; + snap.frame = p.frame; + snap.order = p.order++; + p.snapshots.push_back(std::move(snap)); + note("copied"); + } + } catch (...) {} + } + original(list, count, barriers); +} + +// A depth value as 0..1 from a copied row: floats as they are, 24-bit and 16-bit as fractions. +double depth_at(const std::uint8_t* row, std::uint32_t x, DXGI_FORMAT format, std::uint32_t bytes) { + switch (format) { + case DXGI_FORMAT_R32_TYPELESS: + case DXGI_FORMAT_D32_FLOAT: + case DXGI_FORMAT_R32_FLOAT: { + float v; + std::memcpy(&v, row + std::size_t{x} * 4, 4); + return v; + } + case DXGI_FORMAT_R24G8_TYPELESS: + case DXGI_FORMAT_D24_UNORM_S8_UINT: + case DXGI_FORMAT_R24_UNORM_X8_TYPELESS: { + std::uint32_t v; + std::memcpy(&v, row + std::size_t{x} * 4, 4); + return (v & 0xffffff) / 16777215.0; + } + default: + if (bytes == 2) { + std::uint16_t v; + std::memcpy(&v, row + std::size_t{x} * 2, 2); + return v / 65535.0; + } + float v; + std::memcpy(&v, row + std::size_t{x} * 4, 4); + return v; + } +} + +void write_snapshots(detail::State& s) { + auto& p = probe(); + std::vector snapshots; + std::vector seen, notes; + unsigned barriers, depth_barriers, too_small, other_list; + { + std::lock_guard lock(p.mutex); + snapshots.swap(p.snapshots); + seen.swap(p.seen); + notes.swap(p.notes); + barriers = p.barriers, depth_barriers = p.depth_barriers, too_small = p.skipped_small, other_list = p.skipped_type; + } + std::error_code error; + std::filesystem::create_directories(p.folder, error); + std::ofstream log(p.folder / L"depth-probe.txt", std::ios::trunc); + log << std::format("Screen {}x{}. Barriers seen: {}; out of DEPTH_WRITE on a depth texture: {} ({} too small, {} not on a direct list).\n", + p.screen_width, p.screen_height, barriers, depth_barriers, too_small, other_list); + log << "Depth textures seen:\n"; + for (const auto& line : seen) log << " " << line << "\n"; + log << "Depth barriers:\n"; + for (const auto& line : notes) log << " " << line << "\n"; + // The copies are on the game's lists, submitted by now; this makes sure they are done. + const UINT64 signal = ++s.next_fence; + const bool done = SUCCEEDED(s.queue->Signal(s.fence.Get(), signal)) && detail::completed(signal, 5000); + if (!done) log << "The GPU did not finish in time; the pictures may be empty.\n"; + for (std::size_t i = 0; i < snapshots.size(); ++i) { + const auto& snap = snapshots[i]; + const auto w = static_cast(snap.desc.Width), h = snap.desc.Height; + const auto format = snap.layout.Footprint.Format; + const auto bytes = w ? static_cast(snap.layout.Footprint.RowPitch / std::max(w, 1)) : 4; + void* mapped{}; + const D3D12_RANGE range{0, static_cast(snap.layout.Footprint.RowPitch) * h}; + if (FAILED(snap.readback->Map(0, &range, &mapped)) || !mapped) { + log << std::format("#{}: could not be read back.\n", i); + continue; + } + const auto* base = static_cast(mapped) + snap.layout.Offset; + std::vector values(std::size_t{w} * h); + double low = 1e30, high = -1e30; + std::size_t zeros = 0, ones = 0; + for (std::uint32_t y = 0; y < h; ++y) + for (std::uint32_t x = 0; x < w; ++x) { + const auto v = depth_at(base + std::size_t{y} * snap.layout.Footprint.RowPitch, x, format, bytes); + values[std::size_t{y} * w + x] = v; + if (v <= 0.0) ++zeros; + else if (v >= 1.0) ++ones; + else low = std::min(low, v), high = std::max(high, v); + } + const D3D12_RANGE nothing{}; + snap.readback->Unmap(0, ¬hing); + if (high < low) low = 0, high = 1; + const auto name = std::format("depth-{:02}-frame{}-{}x{}.pgm", i, snap.frame, w, h); + log << std::format("#{} {}: resource {:#x}, footprint format {}, then state {:#x}, order {}; values {:.6f}..{:.6f}, {} at 0, {} at 1.\n", + i, name, snap.resource, static_cast(format), static_cast(snap.after), snap.order, low, high, zeros, + ones); + std::ofstream picture(p.folder / name, std::ios::binary | std::ios::trunc); + picture << "P5\n" << w << " " << h << "\n65535\n"; + std::vector row(std::size_t{w} * 2); + for (std::uint32_t y = 0; y < h; ++y) { + for (std::uint32_t x = 0; x < w; ++x) { + const auto v = std::clamp((values[std::size_t{y} * w + x] - low) / std::max(high - low, 1e-12), 0.0, 1.0); + const auto q = static_cast(std::lround(v * 65535)); + row[std::size_t{x} * 2] = static_cast(q >> 8), row[std::size_t{x} * 2 + 1] = static_cast(q); + } + picture.write(reinterpret_cast(row.data()), static_cast(row.size())); + } + } + log << std::format("{} picture(s).\n", snapshots.size()); + { + std::lock_guard lock(p.mutex); + p.spare.clear(); + } + logging::printf(logging::Level::info, logging::Channel::graphics, "Fit Check depth probe: %zu depth picture(s) and depth-probe.txt written.", + snapshots.size()); +} +} // namespace + +void request_depth_probe(const wchar_t* folder) noexcept { + try { + auto& p = probe(); + std::lock_guard lock(p.mutex); + p.folder = folder; + p.requested.store(true); + } catch (...) {} +} + +namespace detail { +void install_depth_probe(ID3D12GraphicsCommandList* list) noexcept { + static std::atomic installed{}; + if (!list || installed.exchange(true)) return; + auto* const table = *reinterpret_cast(list); + void* original{}; + if (hook_prepare(table[26], reinterpret_cast(&hooked_barrier), &original) != HookOk) return; + original_barrier.store(reinterpret_cast(original)); + if (hook_enable(table[26]) != HookOk) + logging::printf(logging::Level::warning, logging::Channel::graphics, "Fit Check depth probe: the barrier hook could not start."); +} + +bool depth_probe_pending() { + auto& p = probe(); + if (p.requested.load() || p.armed.load() > 0 || p.settle.load() > 0) return true; + auto& t = tap(); + std::lock_guard lock(t.mutex); + return !t.slots.empty(); // presents until its buffers are let go +} + +void depth_tap_present(State& s, std::uint32_t width, std::uint32_t height, bool wanted) { + auto& t = tap(); + if (!original_barrier.load()) return; + std::lock_guard lock(t.mutex); + ++t.epoch; + if (!wanted) { + t.on.store(false); + // Let go a few frames after the last grab: the GPU may still be copying into them. + if (!t.slots.empty() && ++t.idle > 8) t.slots.clear(); + return; + } + t.idle = 0; + if (t.slots.empty() || t.width != width || t.height != height) { + t.on.store(false); + t.slots.clear(); + t.width = width, t.height = height; + t.slot_size = ((std::uint64_t{width} * 4 + 255) / 256 * 256) * height + 256; + D3D12_HEAP_PROPERTIES heap{}; + heap.Type = D3D12_HEAP_TYPE_READBACK; + D3D12_RESOURCE_DESC bytes{}; + bytes.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bytes.Width = t.slot_size; + bytes.Height = 1; + bytes.DepthOrArraySize = 1; + bytes.MipLevels = 1; + bytes.SampleDesc.Count = 1; + bytes.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + D3D12_HEAP_PROPERTIES upload{}; + upload.Type = D3D12_HEAP_TYPE_UPLOAD; + D3D12_RESOURCE_DESC marker_desc = bytes; + marker_desc.Width = 256; + for (std::size_t i = 0; i < tap_slots; ++i) { + TapSlot slot; + void* cpu{}; + if (FAILED(s.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &bytes, D3D12_RESOURCE_STATE_COPY_DEST, nullptr, + IID_PPV_ARGS(&slot.readback))) || + FAILED(s.device->CreateCommittedResource(&upload, D3D12_HEAP_FLAG_NONE, &marker_desc, D3D12_RESOURCE_STATE_GENERIC_READ, nullptr, + IID_PPV_ARGS(&slot.marker))) || + FAILED(slot.marker->Map(0, nullptr, &cpu)) || !cpu) { + if (!t.said) + logging::printf(logging::Level::warning, logging::Channel::graphics, + "Capture depth: no buffers for the depth copies; cutting out by colour."); + t.said = true; + t.slots.clear(); + return; + } + slot.marker_cpu = static_cast(cpu); + *slot.marker_cpu = 0; + t.slots.push_back(std::move(slot)); + } + t.next = 0; + } + t.on.store(true); +} + +const std::uint8_t* depth_tap_latest(std::uint32_t& width, std::uint32_t& height, std::uint32_t& pitch) { + auto& t = tap(); + std::lock_guard lock(t.mutex); + t.mapped = nullptr; + TapSlot* best{}; + for (auto& slot : t.slots) { + if (!slot.seq || !slot.readback) continue; + void* cpu{}; + const D3D12_RANGE tail{static_cast(slot.total), static_cast(slot.total + 4)}; + if (FAILED(slot.readback->Map(0, &tail, &cpu)) || !cpu) continue; + std::uint32_t written{}; + std::memcpy(&written, static_cast(cpu) + slot.total, 4); + const D3D12_RANGE nothing{}; + slot.readback->Unmap(0, ¬hing); + if (written == slot.seq && (!best || slot.seq > best->seq)) best = &slot; + } + if (!best) return nullptr; + void* cpu{}; + const D3D12_RANGE all{0, static_cast(best->total)}; + if (FAILED(best->readback->Map(0, &all, &cpu)) || !cpu) return nullptr; + t.mapped = best; + width = best->layout.Footprint.Width, height = best->layout.Footprint.Height, pitch = best->layout.Footprint.RowPitch; + return static_cast(cpu); +} + +void depth_tap_done() { + auto& t = tap(); + std::lock_guard lock(t.mutex); + if (!t.mapped) return; + const D3D12_RANGE nothing{}; + t.mapped->readback->Unmap(0, ¬hing); + t.mapped = nullptr; +} + +void depth_probe_present(State& s, std::uint32_t width, std::uint32_t height) { + auto& p = probe(); + if (!original_barrier.load()) { + if (p.requested.exchange(false)) + logging::printf(logging::Level::warning, logging::Channel::graphics, "Fit Check depth probe: no barrier hook, nothing to probe."); + return; + } + if (p.requested.exchange(false)) { + std::lock_guard lock(p.mutex); + p.screen_width = width, p.screen_height = height; + p.snapshots.clear(), p.seen.clear(), p.notes.clear(); + p.barriers = p.depth_barriers = p.skipped_small = p.skipped_type = 0; + p.frame = 0, p.order = 0; + // Readback buffers: up to eight, each for a full-screen plane of four bytes a pixel. + p.spare.clear(); + p.spare_size = ((std::uint64_t{width} * 4 + 255) / 256 * 256) * height + 65536; + D3D12_HEAP_PROPERTIES heap{}; + heap.Type = D3D12_HEAP_TYPE_READBACK; + D3D12_RESOURCE_DESC bytes{}; + bytes.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bytes.Width = p.spare_size; + bytes.Height = 1; + bytes.DepthOrArraySize = 1; + bytes.MipLevels = 1; + bytes.SampleDesc.Count = 1; + bytes.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + HRESULT made = S_OK; + for (int i = 0; i < 8; ++i) { + Microsoft::WRL::ComPtr buffer; + made = s.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &bytes, D3D12_RESOURCE_STATE_COPY_DEST, nullptr, IID_PPV_ARGS(&buffer)); + if (FAILED(made)) break; + p.spare.push_back(std::move(buffer)); + } + if (p.spare.empty()) { + logging::printf(logging::Level::warning, logging::Channel::graphics, "Fit Check depth probe: no readback buffer could be made (0x%08lx).", + static_cast(made)); + return; + } + p.armed.store(3); // this frame's recording onwards, two whole frames + return; + } + if (p.armed.load() > 0) { + std::lock_guard lock(p.mutex); + ++p.frame; + if (p.armed.fetch_sub(1) == 1) p.settle.store(4); + return; + } + if (p.settle.load() > 0 && p.settle.fetch_sub(1) == 1) { + try { write_snapshots(s); } catch (...) {} + } +} +} // namespace detail +} // namespace dingosdk::overlay diff --git a/Extension/UI/Overlay/frame_grab.cpp b/Extension/UI/Overlay/frame_grab.cpp new file mode 100644 index 00000000..196b28d3 --- /dev/null +++ b/Extension/UI/Overlay/frame_grab.cpp @@ -0,0 +1,185 @@ +#include "overlay_internal.h" +#include "Engine/Core/Log/logging.h" +#include +#include +#include +#include +#include + +// Reads a presented frame back from the GPU for a screen capture (overlay.h, FrameGrabHost; +// reskate.capture), from Fit Check (pull request 326). The copy is made on the game's own queue right +// before the overlay draws, and waited for: a capture asks for a few hundred frames at most, and a +// frame's hitch then does not matter. +namespace dingosdk::overlay { +namespace { +std::atomic frame_grab_host{}; +} +void set_frame_grab(const FrameGrabHost* host) noexcept { frame_grab_host.store(host); } +} // namespace dingosdk::overlay + +namespace dingosdk::overlay::detail { +namespace { +// Render thread only (under the render mutex), like the rest of the overlay's graphics state. +struct Grab { + ComPtr allocator; + ComPtr commands; + ComPtr readback; + UINT64 readback_size{}; + std::vector converted; // 8-bit copy of a frame in a wider format + DXGI_FORMAT warned{DXGI_FORMAT_UNKNOWN}; +}; +Grab& grab() { static auto* value = new Grab; return *value; } // process lifetime + +float half_to_float(std::uint16_t h) noexcept { + const std::uint32_t sign = (h >> 15) & 1, exponent = (h >> 10) & 31, mantissa = h & 0x3ff; + float value; + if (!exponent) value = std::ldexp(static_cast(mantissa), -24); + else if (exponent == 31) value = mantissa ? 0.0f : 65504.0f; + else value = std::ldexp(static_cast(mantissa | 0x400), static_cast(exponent) - 25); + return sign ? -value : value; +} +std::uint8_t linear_to_srgb8(float v) noexcept { + v = std::clamp(v, 0.0f, 1.0f); + const float s = v <= 0.0031308f ? v * 12.92f : 1.055f * std::pow(v, 1.0f / 2.4f) - 0.055f; + return static_cast(std::lround(s * 255.0f)); +} + +const FrameGrabHost* host() { return frame_grab_host.load(); } +} // namespace + +bool frame_grab_pending() { + const auto* h = host(); + if (!h || !h->active) return false; + try { return h->active(); } catch (...) { return false; } +} + +void release_frame_grab() { + auto& g = grab(); + g.commands.Reset(); + g.allocator.Reset(); + g.readback.Reset(); + g.readback_size = 0; + g.converted = {}; +} + +void grab_presented_frame(State& s, ID3D12Resource* buffer) { + const auto* h = host(); + if (!h || !h->wants_frame || !h->take_frame || !buffer || !s.device || !s.queue || !s.fence) return; + bool wanted = false; + try { wanted = h->wants_frame(); } catch (...) {} + if (!wanted) return; + auto& g = grab(); + const auto description = buffer->GetDesc(); + const auto format = description.Format; + const bool eight_bit = format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB || + format == DXGI_FORMAT_B8G8R8A8_UNORM || format == DXGI_FORMAT_B8G8R8A8_UNORM_SRGB; + const bool ten_bit = format == DXGI_FORMAT_R10G10B10A2_UNORM; + const bool half = format == DXGI_FORMAT_R16G16B16A16_FLOAT; + if (!eight_bit && !ten_bit && !half) { + if (g.warned != format) + logging::printf(logging::Level::warning, logging::Channel::graphics, + "Capture: the backbuffer format %u cannot be read back.", static_cast(format)); + g.warned = format; + h->take_frame({}); + return; + } + if ((ten_bit || half) && g.warned != format) { + // An HDR swapchain: the picture is converted, but its colours will not match what SDR shows. + logging::printf(logging::Level::warning, logging::Channel::graphics, + "Capture: the game is presenting in HDR (format %u); captured colours are converted to SDR and may look off.", + static_cast(format)); + g.warned = format; + } + D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout{}; + UINT rows{}; + UINT64 row_bytes{}, total{}; + s.device->GetCopyableFootprints(&description, 0, 1, 0, &layout, &rows, &row_bytes, &total); + const auto fail = [&](const char* what) { + logging::printf(logging::Level::warning, logging::Channel::graphics, "Capture: frame read-back failed at %s.", what); + try { h->take_frame({}); } catch (...) {} + }; + if (!total || rows != description.Height) return fail("its layout"); + if (!g.allocator && + FAILED(s.device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(&g.allocator)))) + return fail("CreateCommandAllocator"); + if (!g.commands) { + if (FAILED(s.device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, g.allocator.Get(), nullptr, IID_PPV_ARGS(&g.commands)))) + return fail("CreateCommandList"); + g.commands->Close(); + } + if (g.readback_size < total) { + g.readback.Reset(); + D3D12_HEAP_PROPERTIES heap{}; + heap.Type = D3D12_HEAP_TYPE_READBACK; + D3D12_RESOURCE_DESC bytes{}; + bytes.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bytes.Width = total; + bytes.Height = 1; + bytes.DepthOrArraySize = 1; + bytes.MipLevels = 1; + bytes.SampleDesc.Count = 1; + bytes.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + if (FAILED(s.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &bytes, D3D12_RESOURCE_STATE_COPY_DEST, + nullptr, IID_PPV_ARGS(&g.readback)))) + return fail("CreateCommittedResource"); + g.readback_size = total; + } + if (FAILED(g.allocator->Reset()) || FAILED(g.commands->Reset(g.allocator.Get(), nullptr))) return fail("Reset"); + D3D12_RESOURCE_BARRIER barrier{}; + barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + barrier.Transition.pResource = buffer; + barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_PRESENT; + barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + g.commands->ResourceBarrier(1, &barrier); + D3D12_TEXTURE_COPY_LOCATION to{}, from{}; + to.pResource = g.readback.Get(); + to.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + to.PlacedFootprint = layout; + from.pResource = buffer; + from.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + from.SubresourceIndex = 0; + g.commands->CopyTextureRegion(&to, 0, 0, 0, &from, nullptr); + std::swap(barrier.Transition.StateBefore, barrier.Transition.StateAfter); + g.commands->ResourceBarrier(1, &barrier); + if (FAILED(g.commands->Close())) return fail("Close"); + ID3D12CommandList* lists[] = {g.commands.Get()}; + s.queue->ExecuteCommandLists(1, lists); + const UINT64 signal = ++s.next_fence; + if (FAILED(s.queue->Signal(s.fence.Get(), signal)) || !completed(signal, render_fence_timeout_ms)) return fail("the GPU wait"); + void* mapped{}; + const D3D12_RANGE range{0, static_cast(total)}; + if (FAILED(g.readback->Map(0, &range, &mapped)) || !mapped) return fail("Map"); + const auto* pixels = static_cast(mapped) + layout.Offset; + const auto width = static_cast(description.Width), height = description.Height; + GrabbedFrame frame{pixels, width, height, layout.Footprint.RowPitch, + format == DXGI_FORMAT_B8G8R8A8_UNORM || format == DXGI_FORMAT_B8G8R8A8_UNORM_SRGB}; + if (!eight_bit) { + g.converted.resize(std::size_t{width} * height * 4); + for (std::uint32_t y = 0; y < height; ++y) + for (std::uint32_t x = 0; x < width; ++x) { + auto* out = g.converted.data() + (std::size_t{y} * width + x) * 4; + const auto* in = pixels + std::size_t{y} * layout.Footprint.RowPitch; + if (ten_bit) { + std::uint32_t packed; + std::memcpy(&packed, in + std::size_t{x} * 4, 4); + for (int c = 0; c < 3; ++c) out[c] = static_cast(((packed >> (10 * c)) & 0x3ff) * 255 / 1023); + } else { + std::uint16_t channel[4]; + std::memcpy(channel, in + std::size_t{x} * 8, 8); + for (int c = 0; c < 3; ++c) out[c] = linear_to_srgb8(half_to_float(channel[c])); + } + out[3] = 255; + } + frame = {g.converted.data(), width, height, width * 4, false}; + } + // The depth of this frame (a capture cuts what it shows out by it), when there is one. + std::uint32_t depth_width{}, depth_height{}; + frame.depth = depth_tap_latest(depth_width, depth_height, frame.depth_pitch); + if (frame.depth && (depth_width != width || depth_height != height)) frame.depth = nullptr; + try { h->take_frame(frame); } catch (...) {} + depth_tap_done(); + const D3D12_RANGE nothing{}; + g.readback->Unmap(0, ¬hing); +} +} // namespace dingosdk::overlay::detail diff --git a/Extension/UI/Overlay/overlay.h b/Extension/UI/Overlay/overlay.h index 6bf40e9e..7a146fca 100644 --- a/Extension/UI/Overlay/overlay.h +++ b/Extension/UI/Overlay/overlay.h @@ -494,6 +494,30 @@ struct ItemBrowserHost { }; // `host` must stay valid for the life of the process. void set_item_browser(const ItemBrowserHost *host) noexcept; +// A presented frame as the game drew it, before ReSkate's own UI goes on top: 8-bit RGBA rows +// (BGRA when `bgra`), `pitch` bytes apart. Valid only during the call it is handed to. +struct GrabbedFrame { + const std::uint8_t *data{}; + std::uint32_t width{}, height{}, pitch{}; + bool bgra{}; + // The game's depth for the same frame, when it was found (depth_probe.cpp): 32-bit floats, + // reversed (1 near, 0 at the far plane), `depth_pitch` bytes a row, the size of the frame. + const std::uint8_t *depth{}; + std::uint32_t depth_pitch{}; +}; +// Screen captures (Extension/Rendering/Capture, reskate.capture): asked on every presented frame, on +// the render thread, whether they want its pixels, and handed them when they do. While `active` says +// so the overlay keeps working even with nothing of its own on screen. +struct FrameGrabHost { + bool (*active)() noexcept {}; + bool (*wants_frame)() noexcept {}; + void (*take_frame)(const GrabbedFrame &frame) noexcept {}; +}; +// `host` must stay valid for the life of the process. +void set_frame_grab(const FrameGrabHost *host) noexcept; +// Research aid for captures (`capture depth`): the game's depth buffers over the next two frames are +// copied and written to `folder` as pictures, with depth-probe.txt (depth_probe.cpp). +void request_depth_probe(const wchar_t *folder) noexcept; using ParkSurfaceQueue = bool (*)(const EditorSurfaceRequest &); void set_park_surface_queue(ParkSurfaceQueue) noexcept; using ParkPreviewQueue = bool (*)(const EditorPreviewRequest &); diff --git a/Extension/UI/Overlay/overlay_internal.h b/Extension/UI/Overlay/overlay_internal.h index c93038ef..07c6c29d 100644 --- a/Extension/UI/Overlay/overlay_internal.h +++ b/Extension/UI/Overlay/overlay_internal.h @@ -282,6 +282,23 @@ LRESULT CALLBACK window_proc(HWND window, UINT message, WPARAM wp, LPARAM lp); // Swapchain selection, device objects and the per-frame draw (overlay_render.cpp). bool completed(UINT64 value, DWORD timeout_ms); void destroy_graphics(); +// Screen captures (frame_grab.cpp): a frame is being asked for, so the overlay keeps working; the +// presented frame's pixels handed over before the overlay draws on it; its GPU objects let go. +bool frame_grab_pending(); +void grab_presented_frame(State& s, ID3D12Resource* buffer); +void release_frame_grab(); +// The depth probe (depth_probe.cpp, `capture depth`): its hook, installed with the overlay's command +// list; whether it needs this present; and its turn at it. +void install_depth_probe(ID3D12GraphicsCommandList* list) noexcept; +bool depth_probe_pending(); +void depth_probe_present(State& s, std::uint32_t width, std::uint32_t height); +// The depth tap (depth_probe.cpp): while a frame grab runs, the game's main depth buffer is copied +// every frame. depth_tap_present runs at each present (`wanted`: a grab is running); after the +// frame's own work is known finished, depth_tap_latest maps the newest copy the GPU has made (or +// gives null) until depth_tap_done. +void depth_tap_present(State& s, std::uint32_t width, std::uint32_t height, bool wanted); +const std::uint8_t* depth_tap_latest(std::uint32_t& width, std::uint32_t& height, std::uint32_t& pitch); +void depth_tap_done(); ComPtr object_identity(IUnknown* object); Binding* find_binding(IUnknown* identity); void guarded_render(IDXGISwapChain* chain, UINT flags) noexcept; diff --git a/Extension/UI/Overlay/overlay_render.cpp b/Extension/UI/Overlay/overlay_render.cpp index 9f469a01..49df09f5 100644 --- a/Extension/UI/Overlay/overlay_render.cpp +++ b/Extension/UI/Overlay/overlay_render.cpp @@ -87,6 +87,7 @@ void destroy_graphics() { menu_map_pixels_lost(); s.mod_icon_texture.Reset(); s.mod_icon_upload.Reset(); mod_icon_lost(); + release_frame_grab(); for (auto& texture : s.plugin_textures) texture.Reset(); for (auto& upload : s.plugin_uploads) upload.Reset(); plugin_images_lost(); @@ -367,6 +368,7 @@ bool setup_graphics() { s.commands->SetName(L"ReSkate overlay draw"); dingosdk::overlay::watch_draws(s.commands.Get()); dingosdk::overlay::watch_addresses(s.commands.Get()); + install_depth_probe(s.commands.Get()); s.fence->SetName(L"ReSkate overlay fence"); s.fence_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); if (!s.fence_event) return false; @@ -771,6 +773,7 @@ void render(IDXGISwapChain* presented, UINT flags) { const bool item_browser_frame = item_browser_pending(); const bool perf_frame = perf_hud_pending() || trainer_hud_pending(); const bool hub_frame = hub_page_pending(); + const bool grab_frame = frame_grab_pending() || depth_probe_pending(); // A plugin's screen look is drawn under every frame it asks for (reskate.screen.look). const bool look_frame = screen_look::pending(); if (trainer_open_requested()) s.visible.store(true); @@ -791,7 +794,8 @@ void render(IDXGISwapChain* presented, UINT flags) { } // Hidden, the overlay still draws while a notice or chat line is on screen. if (s.loaded_notice_posted && !notices_pending() && !chat_frame && !game_text_frame && !skate_hud_frame && - !nametag_frame && !plugins_frame && !item_browser_frame && !perf_frame && !hub_frame && !map_download_frame && !look_frame) return; + !nametag_frame && !plugins_frame && !item_browser_frame && !perf_frame && !hub_frame && !map_download_frame && !look_frame && + !grab_frame) return; } else if (!s.ui_was_interactive) { s.ui_was_interactive = true; s.last_model = {}; // Reopening immediately reads fresh state. @@ -813,6 +817,16 @@ void render(IDXGISwapChain* presented, UINT flags) { restore_input(true); return; } + // A screen capture takes the game's frame before anything of ReSkate's is drawn on it. + { + const auto size = frame.buffer->GetDesc(); + const bool grabbing = frame_grab_pending(); + depth_tap_present(s, static_cast(size.Width), size.Height, grabbing); + if (grab_frame) { + if (grabbing) grab_presented_frame(s, frame.buffer.Get()); + depth_probe_present(s, static_cast(size.Width), size.Height); + } + } ImGuiContext* previous = ImGui::GetCurrentContext(); ImGui::SetCurrentContext(s.context); { diff --git a/Extension/UI/Overlay/scripted_pad.cpp b/Extension/UI/Overlay/scripted_pad.cpp index 2ee54409..cc7a0dba 100644 --- a/Extension/UI/Overlay/scripted_pad.cpp +++ b/Extension/UI/Overlay/scripted_pad.cpp @@ -1,6 +1,8 @@ #include "scripted_pad.h" +#include #include #include +#include #include namespace dingosdk::overlay { @@ -13,6 +15,10 @@ struct State { std::chrono::steady_clock::time_point last_tick, done_at; bool active{}; DWORD packet{}; + // A stick held (hold_stick). + int stick{}; + SHORT value{}; + std::chrono::steady_clock::time_point held_until{}; }; State &state() { static auto *value = new State; return *value; } constexpr std::chrono::milliseconds linger{2000}, no_ticks{250}, tick_length{17}; @@ -41,12 +47,27 @@ void pad_tick() noexcept { std::lock_guard lock(s.mutex); if (s.active) advance(s, std::chrono::steady_clock::now()); } +void hold_stick(int stick, float value, unsigned milliseconds) noexcept { + auto &s = state(); + std::lock_guard lock(s.mutex); + s.stick = stick < 0 || stick > 3 ? 2 : stick; + s.value = static_cast(std::lround(std::clamp(value, -1.0f, 1.0f) * 32767.0f)); + s.held_until = value == 0 ? std::chrono::steady_clock::time_point{} + : std::chrono::steady_clock::now() + std::chrono::milliseconds(milliseconds); +} bool scripted_pad(DWORD user, XINPUT_STATE &output) noexcept { auto &s = state(); if (user != 0) return false; std::lock_guard lock(s.mutex); - if (!s.active) return false; const auto now = std::chrono::steady_clock::now(); + if (!s.active) { + if (now >= s.held_until) return false; + output.Gamepad = XINPUT_GAMEPAD{}; + SHORT* axes[] = {&output.Gamepad.sThumbLX, &output.Gamepad.sThumbLY, &output.Gamepad.sThumbRX, &output.Gamepad.sThumbRY}; + *axes[s.stick] = s.value; + output.dwPacketNumber = ++s.packet; + return true; + } // With no game ticks, time alone moves the script on. while (s.current + 1 < s.steps.size() && now - s.last_tick > no_ticks) advance(s, s.last_tick + tick_length); const bool done = s.current + 1 == s.steps.size(); diff --git a/Extension/UI/Overlay/scripted_pad.h b/Extension/UI/Overlay/scripted_pad.h index c275b1db..f03761f1 100644 --- a/Extension/UI/Overlay/scripted_pad.h +++ b/Extension/UI/Overlay/scripted_pad.h @@ -19,4 +19,8 @@ void play_pad(std::vector steps); void pad_tick() noexcept; // True when it wrote the script's pad to `state`. bool scripted_pad(DWORD user, XINPUT_STATE &state) noexcept; +// One stick of the stand-in pad held at `value` (-1..1) for `milliseconds` more, the pad otherwise +// neutral, for turning the game menu's preview (reskate.room.turn); 0 lets go. `stick`: 0 left x, +// 1 left y, 2 right x, 3 right y. A script playing (play_pad) comes first. Any thread. +void hold_stick(int stick, float value, unsigned milliseconds) noexcept; } // namespace dingosdk::overlay diff --git a/Extension/World/character_room.cpp b/Extension/World/character_room.cpp index 6daac750..c37129fc 100644 --- a/Extension/World/character_room.cpp +++ b/Extension/World/character_room.cpp @@ -410,4 +410,6 @@ void tick(std::uintptr_t base, std::uintptr_t skater, bool ready, bool prepare) if (s.way != Way::none) fail(s, "an error in the walk"); } } +std::string menu_buttons(std::uintptr_t base) noexcept { return menu_page(base); } +bool is_skaters_page(const std::string& buttons) noexcept { return skaters_page(buttons); } } // namespace dingosdk::character_room diff --git a/Extension/World/character_room.h b/Extension/World/character_room.h index 18f76838..e1e57608 100644 --- a/Extension/World/character_room.h +++ b/Extension/World/character_room.h @@ -21,4 +21,8 @@ void leave() noexcept; // `prepare`: the room is going to be wanted, so the game's navigation event is looked for now (once a map, // in the background), and the walk does not wait for it. void tick(std::uintptr_t base, std::uintptr_t skater, bool ready, bool prepare) noexcept; +// The game menu's open page, as the navigations of its buttons in one line ("CASPresetPurchase ..."), +// empty when none shows; and whether that is the Skaters page. Client thread. +[[nodiscard]] std::string menu_buttons(std::uintptr_t base) noexcept; +[[nodiscard]] bool is_skaters_page(const std::string& buttons) noexcept; } // namespace dingosdk::character_room diff --git a/Runtime/runtime_init.cpp b/Runtime/runtime_init.cpp index a4bb764c..b254016f 100644 --- a/Runtime/runtime_init.cpp +++ b/Runtime/runtime_init.cpp @@ -1,5 +1,6 @@ #include "runtime_internal.h" #include "Extension/Customization/item_browser.h" +#include "Extension/Rendering/Capture/capture_runtime.h" #include "bootstrap.h" #include "Engine/Core/Hooks/hooks.h" #include "Engine/Core/Log/logging.h" @@ -250,6 +251,7 @@ extern "C" __declspec(dllexport) BOOL WINAPI DingoSDKDebugInitialize() { dingosdk::overlay::set_hub_callbacks(native_callbacks); dingosdk::overlay::set_ui_sound(dingosdk::multiplayer::queue_ui_sound); dingosdk::overlay::set_item_browser(dingosdk::item_browser::overlay_host()); + dingosdk::overlay::set_frame_grab(dingosdk::capture::frame_grab()); dingosdk::overlay::set_game_text_feed(dingosdk::multiplayer::skate_debug_text); dingosdk::overlay::set_skate_hud_feed(dingosdk::multiplayer::skate_hud); dingosdk::overlay::set_nametag_feed(dingosdk::multiplayer::custom_nametags); diff --git a/cmake/Core.cmake b/cmake/Core.cmake index 5d875fb4..9e928e3b 100644 --- a/cmake/Core.cmake +++ b/cmake/Core.cmake @@ -146,6 +146,7 @@ add_library(dingosdk_plugins STATIC Extension/Plugins/plugin_api_world.cpp Extension/Plugins/plugin_api_services.cpp Extension/Plugins/plugin_api_throwdowns.cpp + Extension/Plugins/plugin_api_capture.cpp # The throwdown kit's store and the plugins' modes, which the plugin host and the game's # native UI share (kit::flag is the game's: Extension/Throwdowns/native_flag_session.cpp). Extension/Throwdowns/throwdown_kit.cpp diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index afe8d9f5..3412928f 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -32,7 +32,8 @@ endif() enable_language(C) add_library(dingosdk_miniz STATIC External/miniz/miniz.c) target_include_directories(dingosdk_miniz SYSTEM PUBLIC External/miniz) -target_compile_definitions(dingosdk_miniz PUBLIC MINIZ_NO_ARCHIVE_WRITING_APIS MINIZ_NO_DEFLATE_APIS MINIZ_NO_ZLIB_APIS MINIZ_NO_STDIO MINIZ_NO_TIME) +# Deflate is on for the captures' PNG and APNG files (Extension/Rendering/Capture). +target_compile_definitions(dingosdk_miniz PUBLIC MINIZ_NO_ARCHIVE_WRITING_APIS MINIZ_NO_ZLIB_APIS MINIZ_NO_STDIO MINIZ_NO_TIME) set_target_properties(dingosdk_miniz PROPERTIES FOLDER "Dependencies") add_library(dingosdk_lz4 STATIC External/lz4/lz4.c) target_include_directories(dingosdk_lz4 SYSTEM PUBLIC External/lz4) diff --git a/cmake/Features.cmake b/cmake/Features.cmake index 8b8e81a1..6ffd7d45 100644 --- a/cmake/Features.cmake +++ b/cmake/Features.cmake @@ -35,3 +35,19 @@ target_link_libraries(dingosdk_local_profile PUBLIC dingosdk_fast_travel_unlock include(cmake/ProfileData.cmake) +# Screen captures (Extension/Rendering/Capture, reskate.capture): the recording, matting, the card and the +# GIF / APNG / PNG writers, and WebM through a small VP8 encoder of its own (from Fit Check, pull request +# 326). No Windows or game code, so its tests run anywhere. +add_library(dingosdk_capture STATIC + Extension/Rendering/Capture/capture_image.cpp + Extension/Rendering/Capture/capture_matte.cpp + Extension/Rendering/Capture/capture_panel.cpp + Extension/Rendering/Capture/capture_png.cpp + Extension/Rendering/Capture/capture_gif.cpp + Extension/Rendering/Capture/capture_vp8.cpp + Extension/Rendering/Capture/capture_webm.cpp + Extension/Rendering/Capture/capture_recorder.cpp) +target_include_directories(dingosdk_capture PUBLIC "${PROJECT_SOURCE_DIR}") +# stb_truetype, as shipped with Dear ImGui. +target_include_directories(dingosdk_capture SYSTEM PRIVATE "${PROJECT_SOURCE_DIR}/External/imgui") +target_link_libraries(dingosdk_capture PUBLIC dingosdk_miniz) diff --git a/cmake/Runtime.cmake b/cmake/Runtime.cmake index c4a73140..74809b4e 100644 --- a/cmake/Runtime.cmake +++ b/cmake/Runtime.cmake @@ -208,6 +208,9 @@ add_library(dingosdk_runtime SHARED Extension/Customization/item_origins.cpp Extension/Customization/local_customization_runtime.cpp Extension/Customization/item_browser.cpp + Extension/Customization/worn_items.cpp + Extension/Rendering/Capture/capture_runtime.cpp + Extension/Rendering/Capture/capture_commands.cpp Extension/Progression/local_entitlement_trigger_runtime.cpp Extension/Progression/local_neighborhood_runtime.cpp Extension/Customization/local_player_card_runtime.cpp @@ -399,3 +402,4 @@ if(EXISTS "${dingosdk_startup_art}") "STARTUP_SPLASH RCDATA \"${dingosdk_startup_art_native}\"\n") target_sources(dingosdk_runtime PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/generated/startup_art.rc") endif() +target_link_libraries(dingosdk_runtime PRIVATE dingosdk_capture) diff --git a/cmake/UI.cmake b/cmake/UI.cmake index dfb6996c..36ab9f5a 100644 --- a/cmake/UI.cmake +++ b/cmake/UI.cmake @@ -9,6 +9,8 @@ add_library(dingosdk_overlay STATIC Extension/UI/Overlay/overlay_console.cpp Extension/UI/Overlay/overlay_input.cpp Extension/UI/Overlay/item_browser_overlay.cpp + Extension/UI/Overlay/frame_grab.cpp + Extension/UI/Overlay/depth_probe.cpp Extension/UI/Overlay/overlay_notices.cpp Extension/UI/Overlay/chat_overlay.cpp Extension/UI/Overlay/map_download_card.cpp diff --git a/contrib/plugins/fit-check/README.md b/contrib/plugins/fit-check/README.md new file mode 100644 index 00000000..89df8c15 --- /dev/null +++ b/contrib/plugins/fit-check/README.md @@ -0,0 +1,67 @@ +# Fit Check (plugin) + +Pull request 326's Fit Check as a sandboxed Luau plugin. Press V on the Board screen to spin the +board, or on the Skaters screen (with no item grid open) to spin the skater. It turns once by itself, +hands off, and is saved as a GIF, APNG or WebM, cut out from the backdrop, with an info card beside +it: the deck, grip, trucks and wheels, or the outfit, by name. V again stops it. + +How it works: + +- It holds a stand-in pad's stick over with `reskate.room.turn`, as a player turns the board on the + stick, so it works on mouse and keyboard too. +- `reskate.capture` captures every frame the game draws, before ReSkate's overlay, and cuts each + out by the game's depth (or by colour when there is no depth). +- Once the turn has come round, the frames nearest evenly spaced points of that one turn are saved. +- While it turns, the game's screen-space contact shadows, bloom, lens flare and dirt, film grain, + chromatic aberration and motion blur are switched off with `reskate.settings.set`. For a skater, + screen-space reflections and ambient occlusion are switched off too. All of them are put back + afterwards. + +The files are saved in the plugin's data folder, +`Mods/.reskate_plugin_data/fit-check/fit-checks/`, as `-