From ea8e8dca8ef60a4dff2a14e419bff34e80488bdb Mon Sep 17 00:00:00 2001 From: thunderheronight Date: Tue, 13 Oct 2026 11:21:16 +0200 Subject: [PATCH] Resume: restart an interrupted training instead of failing The pipeline only reuses completed stages and refuses an incomplete training directory. Remove it, so images and camera positions are reused and only the training runs again. Co-Authored-By: Claude Opus 5.5 --- scripts/splat.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/splat.sh b/scripts/splat.sh index aad76a1..15158c5 100755 --- a/scripts/splat.sh +++ b/scripts/splat.sh @@ -59,6 +59,16 @@ log=$RUNS_DIR/$slug.log mkdir -p "$RUNS_DIR" if [ "${FRESH:-0}" = 1 ]; then rm -rf "$out" "$log"; fi +# The pipeline resumes finished stages only: an interrupted training run blocks --resume. +# Then start the training over; images and camera positions are reused. +manifest=$out/spirula/run.json +if [[ " $* " == *" --resume "* ]] && [ -f "$manifest" ] && ! grep -q '"status": *"complete"' "$manifest"; then + ckpt=$(ls -d "$out"/spirula/training/step-*.ckpt 2>/dev/null | tail -1) + echo "Training was interrupted${ckpt:+ (last checkpoint: ${ckpt##*/step-})}; the pipeline cannot continue it." | tee -a "$log" + echo "Restarting the training, images and camera positions are reused." | tee -a "$log" + rm -rf "$out/spirula" +fi + # Image pairs from the duration: , clamped to the pipeline limits 100..500 pairs() { local secs