DieMitchell/ReSkate
1
0 Fork 0
mirror of https://github.com/Dingo-Shenanigans/ReSkate.git synced 2026-10-28 19:36:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #357 from hypnomalice/fix/capture-shoe-depth

Browse Source 
Capture: preserve shoes in depth cut-outs against curved floors
This commit is contained in:
zeex64 authored and GitHub committed 2026-10-24 21:52:06 -05:00
2 parents d47a7c8492 0d91f6adf9
commit dd2218dbc7
2 files changed
+162 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Extension/Rendering/Capture/Test/capture_tests.cpp
+93
View File
@@ -418,7 +418,100 @@ void test_recording(const std::filesystem::path& fonts, Format format, bool skat
expect(recorder.status().state == State::idle && recorder.kept().empty(), "discarded, nothing is held");
}
// The cut by the game's depth: a skater standing on a floor that curves away at the sides. Shoes only
// 3% nearer than the floor under them are kept (they went missing, 2026-10-24); the curved floor is
// not taken for the skater.
void test_depth_cut() {
constexpr std::uint32_t w = 200, h = 150;
std::vector<float> depth(std::size_t{w} * h);
std::vector<std::uint8_t> pixels(std::size_t{w} * h * 4, 90);
const auto floor_at = [&](std::uint32_t x) {
const float t = (2.0f * x + 1) / w - 1;
return 100.0f - 30.0f * t * t; // nearer at the sides, as a curved backdrop is
};
for (std::uint32_t y = 0; y < h; ++y)
for (std::uint32_t x = 0; x < w; ++x) {
float distance = floor_at(x);
if (x >= 80 && x < 120 && y >= 20 && y < 130) distance *= 0.6f; // the skater
else if (x >= 85 && x < 115 && y >= 130 && y < 140) distance *= 0.97f; // their shoes
depth[std::size_t{y} * w + x] = 1.0f / distance;
}
const PixelView frame{pixels.data(), w, h, w * 4, true, reinterpret_cast<const std::uint8_t*>(depth.data()), w * 4};
Image picture(w, h);
expect(depth_mask(frame, {0, 0, w, h}, picture), "the depth shows something in front");
std::size_t shoes = 0, floor = 0;
for (std::uint32_t y = 130; y < 140; ++y)
for (std::uint32_t x = 87; x < 113; ++x) shoes += picture.at(x, y)[3] == 255;
for (std::uint32_t y = 0; y < h; ++y)
for (std::uint32_t x = 0; x < w; ++x)
if ((x < 70 || x >= 130) && picture.at(x, y)[3]) ++floor;
expect(shoes == 26 * 10, "shoes just in front of the floor are kept");
expect(floor == 0, "a backdrop curving away is not taken for the subject");
}
// The cut by the game's depth: a skater standing on a floor that curves away at the sides. Shoes only
// 0.5% nearer than the floor under them are kept (they went missing, 2026-10-24); the curved floor is
// not taken for the skater.
void test_depth_cut_contact() {
constexpr std::uint32_t w = 200, h = 150;
std::vector<float> depth(std::size_t{w} * h);
std::vector<std::uint8_t> pixels(std::size_t{w} * h * 4, 90);
const auto floor_at = [&](std::uint32_t x) {
const float t = (2.0f * x + 1) / w - 1;
return 100.0f - 30.0f * t * t; // nearer at the sides, as a curved backdrop is
};
for (std::uint32_t y = 0; y < h; ++y)
for (std::uint32_t x = 0; x < w; ++x) {
float distance = floor_at(x);
if (x >= 80 && x < 120 && y >= 20 && y < 130) distance *= 0.6f; // the skater
else if (x >= 85 && x < 115 && y >= 130 && y < 134) distance *= 0.995f; // their shoes
depth[std::size_t{y} * w + x] = 1.0f / distance;
}
const PixelView frame{pixels.data(), w, h, w * 4, true, reinterpret_cast<const std::uint8_t*>(depth.data()), w * 4};
Image picture(w, h);
expect(depth_mask(frame, {0, 0, w, h}, picture), "the depth shows something in front");
std::size_t shoes = 0, floor = 0;
for (std::uint32_t y = 130; y < 134; ++y)
for (std::uint32_t x = 87; x < 113; ++x) shoes += picture.at(x, y)[3] == 255;
for (std::uint32_t y = 0; y < h; ++y)
for (std::uint32_t x = 0; x < w; ++x)
if ((x < 70 || x >= 130) && picture.at(x, y)[3]) ++floor;
expect(shoes == 26 * 4, "near-contact soles extend the depth silhouette without colour contrast");
expect(floor == 0, "a backdrop curving away is not taken for the subject");
}
// A wide shallow foreground and far depth outliers must not bias the floor fit,
// including the floor directly under the subject where sole recovery can grow.
void test_depth_cut_wide_and_far() {
constexpr std::uint32_t w=800,h=200;
for (bool far : {false,true}) {
std::vector<float> depth(std::size_t{w}*h);
std::vector<std::uint8_t> pixels(std::size_t{w}*h*4,90);
for (std::uint32_t y=0;y<h;++y) for (std::uint32_t x=0;x<w;++x) {
const double t=(2.0*x+1)/w-1;
double d=100-25*t*t;
if (x>=300 && x<500 && y>=20 && y<130) d*=.6;
if (x>=200 && x<600 && y>=130 && y<140) d*=.97;
if (far && x>=650 && x<660) d=9000000;
depth[std::size_t{y}*w+x]=static_cast<float>(1/d);
}
Image picture(w,h);
const PixelView frame{pixels.data(),w,h,w*4,true,reinterpret_cast<const std::uint8_t*>(depth.data()),w*4};
expect(depth_mask(frame,{0,0,w,h},picture),"wide subject has a depth cut-out");
for (std::uint32_t y=130;y<140;++y) for (std::uint32_t x=205;x<595;++x)
expect(picture.at(x,y)[3]==255,"wide 3%-nearer shoes survive a curved floor fit");
for (std::uint32_t y=140;y<h;++y) for (std::uint32_t x=0;x<w;++x)
expect(picture.at(x,y)[3]==0,"floor directly beneath subject stays transparent");
for (std::uint32_t y=0;y<h;++y) for (std::uint32_t x=650;x<660;++x)
expect(picture.at(x,y)[3]==0,"far depth outliers never become foreground");
}
}
int main(int argc, char** argv) {
test_depth_cut_wide_and_far();
test_depth_cut_contact();
test_depth_cut();
test_matte();
test_backdrop_key(false);
test_backdrop_key(true);
Extension/Rendering/Capture/capture_recorder.cpp
+69 -12
View File
@@ -31,32 +31,86 @@ Image opaque_image(const PixelView& view) {
}
} // namespace
// What turns, by the game's depth: nearer than the backdrop behind it. The backdrop's distance for
// each row is that of the area's outer columns (the backdrop, or the floor where the row meets it:
// the camera is level, so the floor is about as far all along a row), and what is 8% nearer
// than that is the board or skater. The game's UI is never in the depth, and neither are a colour's
// likeness to the backdrop or the floor's reflection. Into `picture`'s alpha; false when the depth
// shows nothing there.
// What turns, by the game's depth: nearer than the backdrop behind it. The game's UI is never in the
// depth, and neither are a colour's likeness to the backdrop or the floor's reflection.
//
// The backdrop's distance is fitted along each row: everything clearly nearer than the row's outer
// columns (8%) is set aside with a wide border round it, and a curve (quadratic across the row) is
// fitted to the distance of the rest, the backdrop or the floor, curved as a studio backdrop is.
// Against that fit, anything 1.5% nearer is in front. A shoe on the floor is only centimetres
// nearer than the floor behind it: the rough per-row guess with an 8% margin cut shoes away where
// they met the floor (2026-10-24). Into `picture`'s alpha; false when the depth shows nothing there.
bool depth_mask(const PixelView& frame, Rect area, Image& picture) {
if (!frame.depth || picture.width != area.width || picture.height != area.height) return false;
const auto w = area.width, h = area.height;
const auto margin = std::max<std::uint32_t>(2, w * 6 / 100);
std::vector<float> edge;
const auto border = std::max<std::uint32_t>(2, w * 6 / 100);
std::vector<float> distance(w), edge;
std::vector<std::uint8_t> aside(w), near_floor(std::size_t{w} * h);
std::vector<int> intervals(std::size_t{w} + 1);
float behind = 0;
std::array<double, 3> fit{};
bool have_fit = false;
std::size_t found = 0;
for (std::uint32_t y = 0; y < h; ++y) {
edge.clear();
for (std::uint32_t x = 0; x < w; x = x + 1 == margin ? w - margin : x + 1) {
for (std::uint32_t x = 0; x < w; ++x) {
const float d = frame.depth_at(area.x + x, area.y + y);
if (d > 1e-7f) edge.push_back(1.0f / d);
distance[x] = d > 1e-7f ? 1.0f / d : 0.0f; // 0: nothing there (the sky, the far plane)
if ((x < margin || x >= w - margin) && distance[x] > 0) edge.push_back(distance[x]);
}
if (!edge.empty()) {
std::nth_element(edge.begin(), edge.begin() + static_cast<std::ptrdiff_t>(edge.size() / 2), edge.end());
behind = edge[edge.size() / 2];
}
// Expand foreground intervals in linear time rather than filling a
// wide border separately for every foreground pixel on the render thread.
std::fill(intervals.begin(), intervals.end(), 0);
for (std::uint32_t x = 0; x < w; ++x)
if (distance[x] > 0 && behind > 0 && distance[x] < behind * 0.92f) {
++intervals[x > border ? x - border : 0u];
--intervals[std::min(w, x + border + 1)];
}
int covered = 0;
for (std::uint32_t x = 0; x < w; ++x) {
const float d = frame.depth_at(area.x + x, area.y + y);
const bool in_front = d > 1e-7f && (behind <= 0 || 1.0f / d < behind * 0.92f);
covered += intervals[x];
aside[x] = covered != 0;
}
// Anchor the initial curve to the outer backdrop, so wide shallow shoes
// cannot pull it towards themselves. Refine over the row only with
// samples agreeing with that anchor. Far props/holes never enter sums.
const auto solve = [&](bool edges_only, const std::array<double, 3>& reference) {
double s0=0,s1=0,s2=0,s3=0,s4=0,r0=0,r1=0,r2=0;
for (std::uint32_t x=0; x<w; ++x) {
const double value=distance[x], t=(2.0*x+1)/w-1, t2=t*t;
if (aside[x] || value<=0 || behind<=0 || value<behind*.92 || value>behind*1.5) continue;
if (edges_only && x>=margin && x<w-margin) continue;
const double predicted=reference[0]+reference[1]*t+reference[2]*t2;
if (!edges_only && (predicted<=0 || std::abs(value-predicted)>predicted*.015)) continue;
s0+=1; s1+=t; s2+=t2; s3+=t2*t; s4+=t2*t2; r0+=value; r1+=value*t; r2+=value*t2;
}
const double det=s0*(s2*s4-s3*s3)-s1*(s1*s4-s3*s2)+s2*(s1*s3-s2*s2);
if (s0<6 || std::abs(det)<=1e-9) return false;
const std::array<double,3> result{
(r0*(s2*s4-s3*s3)-s1*(r1*s4-s3*r2)+s2*(r1*s3-s2*r2))/det,
(s0*(r1*s4-r2*s3)-r0*(s1*s4-s3*s2)+s2*(s1*r2-r1*s2))/det,
(s0*(s2*r2-s3*r1)-s1*(s1*r2-r1*s2)+r0*(s1*s3-s2*s2))/det};
if (!std::all_of(result.begin(),result.end(),[](double v){return std::isfinite(v);})) return false;
fit=result;
have_fit=true;
return true;
};
if (behind>0) {
fit={behind,0,0}; have_fit=true;
if (solve(true,fit)) { const auto anchor=fit; (void)solve(false,anchor); }
}
for (std::uint32_t x = 0; x < w; ++x) {
const double t = (2.0 * x + 1) / w - 1;
const double backdrop = fit[0] + fit[1] * t + fit[2] * t * t;
const bool in_front = distance[x] > 0 && (!have_fit || backdrop <= 0 || distance[x] < backdrop * 0.985);
// Weak depth evidence is only used by the bounded sole extension
// below, never to seed a new foreground region on the floor.
near_floor[std::size_t{y} * w + x] = distance[x] > 0 && have_fit && backdrop > 0 && distance[x] < backdrop * 0.997;
picture.at(x, y)[3] = in_front ? 255 : 0;
found += in_front;
}
@@ -83,13 +137,16 @@ bool depth_mask(const PixelView& frame, Rect area, Image& picture) {
auto* p = picture.at(x, y);
int off = 0;
for (int c = 0; c < 3; ++c) off = std::max(off, std::abs(p[c] - floor_colour[y][static_cast<std::size_t>(c)]));
if (off <= 24) break;
// Captured soles can be just 0.5% nearer and close to the floor's
// colour. Continue an established column with that depth evidence.
if (off <= 24 && !near_floor[std::size_t{y} * w + x]) break;
p[3] = 255;
}
}
return found > 64;
}
// Keeps only what turned, and says where it is. What moves over the turn (opaque and unlike the
// first frame) is the subject; what is solid in any frame, changes at all, and is joined to that is
// the rest of it (the nose and tail sit on the axis the board turns about and change little);