From d635d837250a1fff42fd79b7a4e11e3a654a079b Mon Sep 17 00:00:00 2001 From: eltanschauung <158000645+eltanschauung@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:16:42 -0400 Subject: [PATCH 1/4] Fix dynamic lighting storage corruption on Windows --- port/linux/src/halo_linker_common.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/port/linux/src/halo_linker_common.c b/port/linux/src/halo_linker_common.c index 4362a601..1fd6b332 100644 --- a/port/linux/src/halo_linker_common.c +++ b/port/linux/src/halo_linker_common.c @@ -125,7 +125,9 @@ HALO_COMMON(hs_global_data, 4); /* struct data_array *hs_global_data */ HALO_COMMON(hs_thread_data, 4); /* struct data_array *hs_thread_data */ HALO_COMMON(hud_msg_def, 4); /* struct hud_messaging_parameters_definition *hud_msg_def */ HALO_COMMON(interrupt_result, 24); /* HRESULT interrupt_result */ -HALO_COMMON(light_cluster_partition, 12); /* struct cluster_partition light_cluster_partition */ +/* object_lights.c owns light_cluster_partition. On COFF a weak fallback +can override its tentative COMMON definition; let the owner allocate the +native struct cluster_partition, including its reference-cache fields. */ HALO_COMMON(light_data, 4); /* struct data_array *light_data */ HALO_COMMON(local_player_index_for_draw_string_and_hack_in_icons, 4); /* short local_player_index_for_draw_string_and_hack_in_icons */ HALO_COMMON(looping_sound_data, 4); /* struct data_array *looping_sound_data */ From 8817c374704d5a98174626cbc722f123e4565e33 Mon Sep 17 00:00:00 2001 From: eltanschauung <158000645+eltanschauung@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:17:23 -0400 Subject: [PATCH 2/4] Test native light storage with ordinary and LTO links --- tools/test_light_storage.py | 153 ++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 tools/test_light_storage.py diff --git a/tools/test_light_storage.py b/tools/test_light_storage.py new file mode 100644 index 00000000..cc12325b --- /dev/null +++ b/tools/test_light_storage.py @@ -0,0 +1,153 @@ +"""Link the real light-partition definitions and initialize their native caches. + +No game assets, SDK, C runtime or profile needed. Windows uses COFF/i686; +Linux uses ELF/i386. Exercise both ordinary and full-LTO links. The Windows +negative control reinstates the retail-sized weak fallback and must corrupt +the light-data pointer, proving this checks storage selected by the linker. +""" + +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def function(source, name): + start = re.search(r"(?:static )?void " + re.escape(name) + r"\([^;{]*\)\s*\{", source).start() + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def owner_definition(source, declaration, export): + # Use the owning source's tentative definition, not another typed fallback. + assert declaration in source + return export + declaration + "\n" + + +def run(cc): + windows = sys.platform == "win32" + common = (ROOT / "port/linux/src/halo_linker_common.c").read_text() + lights = (ROOT / "source/objects/object_lights.c").read_text() + partitions = (ROOT / "source/structures/cluster_partitions.c").read_text() + macro = common[common.index("#define HALO_COMMON"):common.index("HALO_COMMON(ai_globals")] + fallbacks = "\n".join(re.findall( + r"^HALO_COMMON\((?:light_cluster_partition|light_data),[^\n]+", common, re.M)) + header = (ROOT / "source/structures/cluster_partitions.h").as_posix() + prefix = f''' +/* Suppress header dependencies unrelated to the partition's layout. */ +#define __REAL_MATH_H +#define __DATA_H +typedef union real_point3d real_point3d; +struct location; +struct data_array {{ unsigned long signature; }}; +#include "{header}" +typedef char require_32_bit_abi[sizeof(void *) == 4 && sizeof(long) == 4 ? 1 : -1]; +''' + # Keep symbol layout observable under whole-program scalar replacement. + export = '__declspec(dllexport) ' if windows else '' + owner = prefix + owner_definition(lights, 'struct cluster_partition light_cluster_partition;', export) + owner += owner_definition(lights, 'struct data_array *light_data;', export) + owner += r''' +#define FALSE 0 +#define MAXIMUM_CLUSTERS_PER_STRUCTURE 8 +#define HALO_PORT_MAXIMUM_CLUSTER_REFERENCES 16 +#define MAXIMUM_PORT_CLUSTER_PARTITIONS 8 +static struct cluster_partition *cluster_partitions_port[MAXIMUM_PORT_CLUSTER_PARTITIONS]; +static short cluster_partitions_port_count; +static unsigned long arena[1024], allocated; +static void *allocate(unsigned long bytes) { + unsigned long words = (bytes + 3) / 4; + void *result = arena + allocated; + allocated += words; + return result; +} +#define game_state_malloc(name, label, bytes) allocate(bytes) +#define reference_list_new(name, count) ((struct data_array *)allocate(sizeof(struct data_array))) +#define debug_malloc(bytes, clear, file, line) allocate(bytes) +#define sprintf(buffer, ...) ((void)0) +#define error(...) ((void)0) +static void csmemset(void *destination, int value, unsigned long bytes) { + unsigned char *out = destination; + while (bytes--) *out++ = (unsigned char)value; +} +''' + forget = function(partitions, 'cluster_partition_port_forget') + owner += forget + owner += function(partitions, 'cluster_partition_new') + owner += ''' +void initialize_partition(void) { + cluster_partition_new(&light_cluster_partition, "light"); + cluster_partition_port_forget(&light_cluster_partition); +} +''' + caller = prefix + r''' +extern struct cluster_partition light_cluster_partition; +extern struct data_array *light_data; +void initialize_partition(void); +static struct data_array lights = { 0x64407440UL }; +__attribute__((noinline,optnone)) static int check(void) { + light_data = &lights; + initialize_partition(); + if (light_data != &lights || light_data->signature != 0x64407440UL) return 71; + if (!light_cluster_partition.cluster_first_data_references || + !light_cluster_partition.data_reference_data || + !light_cluster_partition.cluster_reference_data) return 72; + if (!light_cluster_partition.port_previous_references || + !light_cluster_partition.port_cluster_data_references) return 73; + if (light_cluster_partition.port_modification_count != 2) return 74; + for (int i = 0; i < 16; i++) { + if (light_cluster_partition.port_previous_references[i] != -1 || + light_cluster_partition.port_cluster_data_references[i] != -1) return 75; + } + return 0; +} +''' + caller += 'int mainCRTStartup(void) { return check(); }\n' if windows else r''' +void _start(void) { + int result = check(); + __asm__ volatile("int $0x80" : : "a"(1), "b"(result) : "memory"); + __builtin_unreachable(); +} +''' + with tempfile.TemporaryDirectory(prefix='halo-light-storage-') as directory: + work = Path(directory) + (work / 'owner.c').write_text(owner) + (work / 'caller.c').write_text(caller) + for lto in (False, True): + for legacy in ((False, True) if windows and lto else (False,)): + fallback = fallbacks + if legacy: + fallback = re.sub(r'^HALO_COMMON\(light_cluster_partition,[^\n]+\n?', '', fallback, flags=re.M) + fallback = 'HALO_COMMON(light_cluster_partition, 12);\n' + fallback + (work / 'fallback.c').write_text(macro + fallback + '\n') + executable = work / ('probe.exe' if windows else 'probe') + abi = ['--target=i686-pc-windows-msvc'] if windows else ['-m32'] + link = ['-Wl,/entry:mainCRTStartup', '-Wl,/subsystem:console'] if windows else [ + '-Wl,-e,_start', '-Wl,--export-dynamic', '-no-pie', '-static'] + command = [cc, *abi, '-std=gnu99', '-O2', '-fcommon', '-ffreestanding', + '-fno-builtin', '-nostdlib', '-fuse-ld=lld', '-I' + str(ROOT / 'source'), *link] + if lto: + command.append('-flto=full') + command += [str(work / name) for name in ('owner.c', 'caller.c', 'fallback.c')] + command += ['-o', str(executable)] + subprocess.run(command, check=True, timeout=60) + result = subprocess.run([str(executable)], timeout=10) + expected = 71 if legacy else 0 + assert result.returncode == expected, (command, result.returncode, expected) + print(f'PASS: {"COFF" if windows else "ELF"} {"full LTO" if lto else "ordinary"}: ' + f'{"legacy pointer corruption detected" if legacy else "light data and native caches preserved"}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cc', default='clang') + args = parser.parse_args() + run(args.cc) From 592be6270a5a86f23e46f046e3eb72b60f32a8ab Mon Sep 17 00:00:00 2001 From: eltanschauung <158000645+eltanschauung@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:17:37 -0400 Subject: [PATCH 3/4] Run light storage regressions in Windows and Linux CI --- .github/workflows/build.yml | 741 ++++++++++++++++++------------------ 1 file changed, 373 insertions(+), 368 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b54a4239..84305f0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,368 +1,373 @@ -name: Build - -# Debug and release builds of the native ports (Linux, Windows, Android) for -# every pushed commit, uploaded as artifacts. tools/ci_build.py does each -# build; see it for the options. -# -# Kept cheap: -# - one job per platform builds both configurations, so its set-up (tools, -# downloads, caches) is paid once; -# - compiles go through ccache, whose cache carries over from run to run, so -# a commit only recompiles what it changed; downloads (SDL, musl, Gradle) -# are cached too; -# - a newer push to the same branch cancels a run still going; -# - pushes that only change documentation do not build; -# - Windows, whose minutes cost twice Linux's, builds only Windows; -# - artifacts are kept for three days. -# -# A successful build of main is also published as a GitHub release, the -# newest marked latest, so that -# https://github.com//releases/latest/download/ always -# gives the latest build; the last five are kept. -on: - push: - paths-ignore: - - "**.md" - workflow_dispatch: - -concurrency: - group: build-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -env: - CI_COMPILER_LAUNCHER: ccache - CCACHE_DIR: ${{ github.workspace }}/.ccache - CCACHE_BASEDIR: ${{ github.workspace }} - CCACHE_MAXSIZE: 1G - -jobs: - linux: - # Arch Linux has what the build needs as packages: clang 22 (for the - # profile-guided optimisation in pgo/) and 32-bit SDL 3. - runs-on: ubuntu-24.04 - container: archlinux:latest - timeout-minutes: 40 - steps: - - name: Install the toolchain - run: | - printf '\n[multilib]\nInclude = /etc/pacman.d/mirrorlist\n' >> /etc/pacman.conf - pacman -Syu --noconfirm --needed git zstd python python-pytest ninja ccache \ - clang lld gcc lib32-glibc lib32-gcc-libs lib32-sdl3 - - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: .ccache - key: ccache-linux-${{ github.sha }} - restore-keys: ccache-linux- - - - name: Debug - run: python tools/ci_build.py linux debug - - - name: Release - run: python tools/ci_build.py linux release - - - name: Test - run: python -m pytest -q tools/test_linux_port.py - - - name: Compile cache - run: ccache --show-stats - - - uses: actions/upload-artifact@v4 - with: - name: halo-linux-debug - path: dist/halo-linux-debug - retention-days: 3 - - - uses: actions/upload-artifact@v4 - with: - name: halo-linux-release - path: dist/halo-linux-release - retention-days: 3 - - android: - # The runner image has the Android SDK and NDK, a JDK and CMake. - runs-on: ubuntu-24.04 - timeout-minutes: 40 - steps: - - name: Install the toolchain - run: | - mkdir -p "$RUNNER_TEMP/tools" - curl -fsSL -o ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip - unzip -q ninja.zip -d "$RUNNER_TEMP/tools" && rm ninja.zip - curl -fsSL https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz \ - | tar -xJ -C "$RUNNER_TEMP/tools" --strip-components=1 ccache-4.10.2-linux-x86_64/ccache - echo "$RUNNER_TEMP/tools" >> "$GITHUB_PATH" - - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: .ccache - key: ccache-android-${{ github.sha }} - restore-keys: ccache-android- - - # musl and SDL, which configure.py fetches - - uses: actions/cache@v4 - with: - path: build/android/third_party - key: android-third-party-${{ hashFiles('tools/android_build.py') }} - - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: gradle-${{ hashFiles('port/android/**/*.gradle*', 'port/android/gradle/wrapper/gradle-wrapper.properties') }} - - # the app's signing key, from the ANDROID_KEYSTORE_BASE64 secret (a - # keystore, base64) and ANDROID_KEYSTORE_PASSWORD (its password, and - # the key's, alias halo): every build signed with it installs over the - # one before, as the app's self-updater (Updater.java) needs. Without - # them each build is signed with the runner's own debug key. - - name: Signing key - env: - KEYSTORE: ${{ secrets.ANDROID_KEYSTORE_BASE64 }} - run: | - if [ -n "$KEYSTORE" ]; then - printf '%s' "$KEYSTORE" | base64 -d > "$RUNNER_TEMP/halo.keystore" - echo "HALO_ANDROID_KEYSTORE=$RUNNER_TEMP/halo.keystore" >> "$GITHUB_ENV" - else - echo "::warning::no ANDROID_KEYSTORE_BASE64 secret: the Android builds cannot update over each other" - fi - - - name: Debug - env: - HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} - run: python3 tools/ci_build.py android debug - - - name: Release - env: - HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} - run: python3 tools/ci_build.py android release - - - name: Compile cache - run: ccache --show-stats - - - uses: actions/upload-artifact@v4 - with: - name: halo-android-debug - path: dist/halo-android-debug - retention-days: 3 - - - uses: actions/upload-artifact@v4 - with: - name: halo-android-release - path: dist/halo-android-release - retention-days: 3 - - windows: - # The runner image has LLVM, Python and Visual Studio's x86 libraries and - # Windows SDK. - runs-on: windows-2025 - timeout-minutes: 40 - defaults: - run: - shell: bash - steps: - - name: Install the toolchain - shell: pwsh - run: | - Invoke-WebRequest -OutFile ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip - Expand-Archive ninja.zip -DestinationPath "$env:RUNNER_TEMP\tools" - Invoke-WebRequest -OutFile ccache.zip https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-windows-x86_64.zip - Expand-Archive ccache.zip -DestinationPath $env:RUNNER_TEMP - Copy-Item "$env:RUNNER_TEMP\ccache-4.10.2-windows-x86_64\ccache.exe" "$env:RUNNER_TEMP\tools" - Remove-Item ninja.zip, ccache.zip - "$env:RUNNER_TEMP\tools" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH - - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: .ccache - key: ccache-windows-${{ github.sha }} - restore-keys: ccache-windows- - - # SDL's development package, which configure.py fetches - - uses: actions/cache@v4 - with: - path: build/windows/third_party - key: windows-third-party-${{ hashFiles('tools/windows_build.py') }} - - - name: Debug - run: python tools/ci_build.py windows debug - - - name: Release - run: python tools/ci_build.py windows release - - - name: Compile cache - run: ccache --show-stats - - - uses: actions/upload-artifact@v4 - with: - name: halo-windows-debug - path: dist/halo-windows-debug - retention-days: 3 - - - uses: actions/upload-artifact@v4 - with: - name: halo-windows-release - path: dist/halo-windows-release - retention-days: 3 - - release: - # The six builds as a release, tagged build- and marked latest - # (every file keeps its name from release to release, for the README's - # latest/download links); then all but the newest five build-* releases - # go, tags and all. Only for main, and only when every platform built. - needs: [linux, android, windows] - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: ubuntu-24.04 - timeout-minutes: 15 - permissions: - contents: write - outputs: - tag: ${{ steps.publish.outputs.tag }} - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - steps: - - uses: actions/download-artifact@v4 - with: - pattern: halo-* - path: artifacts - - - name: Package - run: | - mkdir release - for build in artifacts/*/; do - name="$(basename "$build")" - (cd "$build" && zip -qr "$GITHUB_WORKSPACE/release/$name.zip" .) - done - ls -l release - - - name: Publish - id: publish - env: - MESSAGE: ${{ github.event.head_commit.message }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - tag="build-$GITHUB_RUN_NUMBER" - subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1)" - printf '%s\n\nCommit: %s\nBuild: %s\n\nThe release builds are for playing; the debug builds stop at the first failed assertion and log it.\n' \ - "$subject" "$COMMIT_URL" "$RUN_URL" > notes.md - gh release create "$tag" release/*.zip --target "$GITHUB_SHA" --latest \ - --title "Build $GITHUB_RUN_NUMBER (${GITHUB_SHA:0:7})" --notes-file notes.md - echo "tag=$tag" >> "$GITHUB_OUTPUT" - - - name: Keep the last five - run: | - gh release list --limit 100 --json tagName,createdAt \ - --jq '[.[] | select(.tagName | startswith("build-"))] | sort_by(.createdAt) | reverse | .[5:] | .[].tagName' | - while read -r tag; do - gh release delete "$tag" --cleanup-tag --yes - done - - notify: - # A Discord embed for the run, to the DISCORD_WEBHOOK secret (nothing - # without it): green when every platform built, red when one did not, - # with each platform's result, this run's release (main only), and links - # to the latest release's builds. - needs: [linux, android, windows, release] - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - name: Post to Discord - env: - WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - LINUX: ${{ needs.linux.result }} - ANDROID: ${{ needs.android.result }} - WINDOWS: ${{ needs.windows.result }} - RELEASE_TAG: ${{ needs.release.outputs.tag }} - SERVER: ${{ github.server_url }} - REPOSITORY: ${{ github.repository }} - BRANCH: ${{ github.ref_name }} - SHA: ${{ github.sha }} - ACTOR: ${{ github.actor }} - MESSAGE: ${{ github.event.head_commit.message }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} - run: | - if [ -z "$WEBHOOK" ]; then - echo "no DISCORD_WEBHOOK secret: nothing to post" - exit 0 - fi - status() { - case "$1" in - success) echo "✅ Built" ;; - failure) echo "❌ Failed" ;; - cancelled) echo "⏹️ Cancelled" ;; - *) echo "➖ Skipped" ;; - esac - } - if [ "$LINUX" = success ] && [ "$ANDROID" = success ] && [ "$WINDOWS" = success ]; then - title="Build succeeded"; color=5763719 - else - title="Build failed"; color=15548997 - fi - latest="$SERVER/$REPOSITORY/releases/latest" - downloads="[Linux]($latest/download/halo-linux-release.zip) · [Windows]($latest/download/halo-windows-release.zip) · [Android]($latest/download/halo-android-release.zip) ([all builds]($latest))" - if [ -n "$RELEASE_TAG" ]; then - release="[$RELEASE_TAG]($SERVER/$REPOSITORY/releases/tag/$RELEASE_TAG)" - else - release="None (only a successful build of main makes one)" - fi - subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1 | cut -c1-200)" - jq -n \ - --arg title "$title" --argjson color "$color" --arg url "$RUN_URL" \ - --arg description "[\`${SHA:0:7}\`]($COMMIT_URL) $subject — $ACTOR ($BRANCH)" \ - --arg release "$release" --arg downloads "$downloads" \ - --arg linux "$(status "$LINUX")" --arg windows "$(status "$WINDOWS")" --arg android "$(status "$ANDROID")" \ - --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{embeds: [{ - title: $title, url: $url, color: $color, description: $description, timestamp: $timestamp, - fields: [ - {name: "Linux", value: $linux, inline: true}, - {name: "Windows", value: $windows, inline: true}, - {name: "Android", value: $android, inline: true}, - {name: "Release", value: $release}, - {name: "Latest release builds", value: $downloads} - ] - }]}' > payload.json - # Discord rate limits webhooks (HTTP 429, with how many seconds to - # wait in retry_after): wait and try again, a few times. A post - # still limited, or a network or Discord error, is a warning rather - # than a failed run; any other answer (a deleted webhook) fails. (At - # most 5 tries of 15 seconds and 4 waits of 30: inside the job's 5 - # minutes.) - for attempt in 1 2 3 4 5; do - code=$(curl -sS --max-time 15 -o response.json -w '%{http_code}' \ - -H "Content-Type: application/json" -d @payload.json "$WEBHOOK") || code=000 - case "$code" in - 2??) - exit 0 - ;; - 429) - wait=$(jq -r '.retry_after // 5' response.json 2>/dev/null || echo 5) - wait=$(awk -v w="$wait" 'BEGIN { w += 0; if (w < 1) w = 1; if (w > 30) w = 30; printf "%d", w + 0.999 }') - echo "Discord is rate limiting the webhook; trying again in ${wait}s (attempt $attempt of 5)" - ;; - 000|5??) - wait=5 - echo "Discord answered HTTP $code; trying again in ${wait}s (attempt $attempt of 5)" - ;; - *) - echo "::error::Discord answered HTTP $code: $(cat response.json 2>/dev/null)" - exit 1 - ;; - esac - [ "$attempt" -lt 5 ] && sleep "$wait" - done - echo "::warning::The Discord post did not go through after 5 attempts (HTTP $code)" +name: Build + +# Debug and release builds of the native ports (Linux, Windows, Android) for +# every pushed commit, uploaded as artifacts. tools/ci_build.py does each +# build; see it for the options. +# +# Kept cheap: +# - one job per platform builds both configurations, so its set-up (tools, +# downloads, caches) is paid once; +# - compiles go through ccache, whose cache carries over from run to run, so +# a commit only recompiles what it changed; downloads (SDL, musl, Gradle) +# are cached too; +# - a newer push to the same branch cancels a run still going; +# - pushes that only change documentation do not build; +# - Windows, whose minutes cost twice Linux's, builds only Windows; +# - artifacts are kept for three days. +# +# A successful build of main is also published as a GitHub release, the +# newest marked latest, so that +# https://github.com//releases/latest/download/ always +# gives the latest build; the last five are kept. +on: + push: + paths-ignore: + - "**.md" + workflow_dispatch: + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CI_COMPILER_LAUNCHER: ccache + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_MAXSIZE: 1G + +jobs: + linux: + # Arch Linux has what the build needs as packages: clang 22 (for the + # profile-guided optimisation in pgo/) and 32-bit SDL 3. + runs-on: ubuntu-24.04 + container: archlinux:latest + timeout-minutes: 40 + steps: + - name: Install the toolchain + run: | + printf '\n[multilib]\nInclude = /etc/pacman.d/mirrorlist\n' >> /etc/pacman.conf + pacman -Syu --noconfirm --needed git zstd python python-pytest ninja ccache \ + clang lld gcc lib32-glibc lib32-gcc-libs lib32-sdl3 + + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-linux-${{ github.sha }} + restore-keys: ccache-linux- + + - name: Debug + run: python tools/ci_build.py linux debug + + - name: Release + run: python tools/ci_build.py linux release + + - name: Test + run: | + python -m pytest -q tools/test_linux_port.py + python tools/test_light_storage.py + + - name: Compile cache + run: ccache --show-stats + + - uses: actions/upload-artifact@v4 + with: + name: halo-linux-debug + path: dist/halo-linux-debug + retention-days: 3 + + - uses: actions/upload-artifact@v4 + with: + name: halo-linux-release + path: dist/halo-linux-release + retention-days: 3 + + android: + # The runner image has the Android SDK and NDK, a JDK and CMake. + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - name: Install the toolchain + run: | + mkdir -p "$RUNNER_TEMP/tools" + curl -fsSL -o ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip + unzip -q ninja.zip -d "$RUNNER_TEMP/tools" && rm ninja.zip + curl -fsSL https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz \ + | tar -xJ -C "$RUNNER_TEMP/tools" --strip-components=1 ccache-4.10.2-linux-x86_64/ccache + echo "$RUNNER_TEMP/tools" >> "$GITHUB_PATH" + + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-android-${{ github.sha }} + restore-keys: ccache-android- + + # musl and SDL, which configure.py fetches + - uses: actions/cache@v4 + with: + path: build/android/third_party + key: android-third-party-${{ hashFiles('tools/android_build.py') }} + + - uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ hashFiles('port/android/**/*.gradle*', 'port/android/gradle/wrapper/gradle-wrapper.properties') }} + + # the app's signing key, from the ANDROID_KEYSTORE_BASE64 secret (a + # keystore, base64) and ANDROID_KEYSTORE_PASSWORD (its password, and + # the key's, alias halo): every build signed with it installs over the + # one before, as the app's self-updater (Updater.java) needs. Without + # them each build is signed with the runner's own debug key. + - name: Signing key + env: + KEYSTORE: ${{ secrets.ANDROID_KEYSTORE_BASE64 }} + run: | + if [ -n "$KEYSTORE" ]; then + printf '%s' "$KEYSTORE" | base64 -d > "$RUNNER_TEMP/halo.keystore" + echo "HALO_ANDROID_KEYSTORE=$RUNNER_TEMP/halo.keystore" >> "$GITHUB_ENV" + else + echo "::warning::no ANDROID_KEYSTORE_BASE64 secret: the Android builds cannot update over each other" + fi + + - name: Debug + env: + HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} + run: python3 tools/ci_build.py android debug + + - name: Release + env: + HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} + run: python3 tools/ci_build.py android release + + - name: Compile cache + run: ccache --show-stats + + - uses: actions/upload-artifact@v4 + with: + name: halo-android-debug + path: dist/halo-android-debug + retention-days: 3 + + - uses: actions/upload-artifact@v4 + with: + name: halo-android-release + path: dist/halo-android-release + retention-days: 3 + + windows: + # The runner image has LLVM, Python and Visual Studio's x86 libraries and + # Windows SDK. + runs-on: windows-2025 + timeout-minutes: 40 + defaults: + run: + shell: bash + steps: + - name: Install the toolchain + shell: pwsh + run: | + Invoke-WebRequest -OutFile ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip + Expand-Archive ninja.zip -DestinationPath "$env:RUNNER_TEMP\tools" + Invoke-WebRequest -OutFile ccache.zip https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-windows-x86_64.zip + Expand-Archive ccache.zip -DestinationPath $env:RUNNER_TEMP + Copy-Item "$env:RUNNER_TEMP\ccache-4.10.2-windows-x86_64\ccache.exe" "$env:RUNNER_TEMP\tools" + Remove-Item ninja.zip, ccache.zip + "$env:RUNNER_TEMP\tools" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH + + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-windows-${{ github.sha }} + restore-keys: ccache-windows- + + # SDL's development package, which configure.py fetches + - uses: actions/cache@v4 + with: + path: build/windows/third_party + key: windows-third-party-${{ hashFiles('tools/windows_build.py') }} + + - name: Debug + run: python tools/ci_build.py windows debug + + - name: Release + run: python tools/ci_build.py windows release + + - name: Test light storage + run: python tools/test_light_storage.py + + - name: Compile cache + run: ccache --show-stats + + - uses: actions/upload-artifact@v4 + with: + name: halo-windows-debug + path: dist/halo-windows-debug + retention-days: 3 + + - uses: actions/upload-artifact@v4 + with: + name: halo-windows-release + path: dist/halo-windows-release + retention-days: 3 + + release: + # The six builds as a release, tagged build- and marked latest + # (every file keeps its name from release to release, for the README's + # latest/download links); then all but the newest five build-* releases + # go, tags and all. Only for main, and only when every platform built. + needs: [linux, android, windows] + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + outputs: + tag: ${{ steps.publish.outputs.tag }} + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - uses: actions/download-artifact@v4 + with: + pattern: halo-* + path: artifacts + + - name: Package + run: | + mkdir release + for build in artifacts/*/; do + name="$(basename "$build")" + (cd "$build" && zip -qr "$GITHUB_WORKSPACE/release/$name.zip" .) + done + ls -l release + + - name: Publish + id: publish + env: + MESSAGE: ${{ github.event.head_commit.message }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + tag="build-$GITHUB_RUN_NUMBER" + subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1)" + printf '%s\n\nCommit: %s\nBuild: %s\n\nThe release builds are for playing; the debug builds stop at the first failed assertion and log it.\n' \ + "$subject" "$COMMIT_URL" "$RUN_URL" > notes.md + gh release create "$tag" release/*.zip --target "$GITHUB_SHA" --latest \ + --title "Build $GITHUB_RUN_NUMBER (${GITHUB_SHA:0:7})" --notes-file notes.md + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Keep the last five + run: | + gh release list --limit 100 --json tagName,createdAt \ + --jq '[.[] | select(.tagName | startswith("build-"))] | sort_by(.createdAt) | reverse | .[5:] | .[].tagName' | + while read -r tag; do + gh release delete "$tag" --cleanup-tag --yes + done + + notify: + # A Discord embed for the run, to the DISCORD_WEBHOOK secret (nothing + # without it): green when every platform built, red when one did not, + # with each platform's result, this run's release (main only), and links + # to the latest release's builds. + needs: [linux, android, windows, release] + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Post to Discord + env: + WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + LINUX: ${{ needs.linux.result }} + ANDROID: ${{ needs.android.result }} + WINDOWS: ${{ needs.windows.result }} + RELEASE_TAG: ${{ needs.release.outputs.tag }} + SERVER: ${{ github.server_url }} + REPOSITORY: ${{ github.repository }} + BRANCH: ${{ github.ref_name }} + SHA: ${{ github.sha }} + ACTOR: ${{ github.actor }} + MESSAGE: ${{ github.event.head_commit.message }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + run: | + if [ -z "$WEBHOOK" ]; then + echo "no DISCORD_WEBHOOK secret: nothing to post" + exit 0 + fi + status() { + case "$1" in + success) echo "✅ Built" ;; + failure) echo "❌ Failed" ;; + cancelled) echo "⏹️ Cancelled" ;; + *) echo "➖ Skipped" ;; + esac + } + if [ "$LINUX" = success ] && [ "$ANDROID" = success ] && [ "$WINDOWS" = success ]; then + title="Build succeeded"; color=5763719 + else + title="Build failed"; color=15548997 + fi + latest="$SERVER/$REPOSITORY/releases/latest" + downloads="[Linux]($latest/download/halo-linux-release.zip) · [Windows]($latest/download/halo-windows-release.zip) · [Android]($latest/download/halo-android-release.zip) ([all builds]($latest))" + if [ -n "$RELEASE_TAG" ]; then + release="[$RELEASE_TAG]($SERVER/$REPOSITORY/releases/tag/$RELEASE_TAG)" + else + release="None (only a successful build of main makes one)" + fi + subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1 | cut -c1-200)" + jq -n \ + --arg title "$title" --argjson color "$color" --arg url "$RUN_URL" \ + --arg description "[\`${SHA:0:7}\`]($COMMIT_URL) $subject — $ACTOR ($BRANCH)" \ + --arg release "$release" --arg downloads "$downloads" \ + --arg linux "$(status "$LINUX")" --arg windows "$(status "$WINDOWS")" --arg android "$(status "$ANDROID")" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{embeds: [{ + title: $title, url: $url, color: $color, description: $description, timestamp: $timestamp, + fields: [ + {name: "Linux", value: $linux, inline: true}, + {name: "Windows", value: $windows, inline: true}, + {name: "Android", value: $android, inline: true}, + {name: "Release", value: $release}, + {name: "Latest release builds", value: $downloads} + ] + }]}' > payload.json + # Discord rate limits webhooks (HTTP 429, with how many seconds to + # wait in retry_after): wait and try again, a few times. A post + # still limited, or a network or Discord error, is a warning rather + # than a failed run; any other answer (a deleted webhook) fails. (At + # most 5 tries of 15 seconds and 4 waits of 30: inside the job's 5 + # minutes.) + for attempt in 1 2 3 4 5; do + code=$(curl -sS --max-time 15 -o response.json -w '%{http_code}' \ + -H "Content-Type: application/json" -d @payload.json "$WEBHOOK") || code=000 + case "$code" in + 2??) + exit 0 + ;; + 429) + wait=$(jq -r '.retry_after // 5' response.json 2>/dev/null || echo 5) + wait=$(awk -v w="$wait" 'BEGIN { w += 0; if (w < 1) w = 1; if (w > 30) w = 30; printf "%d", w + 0.999 }') + echo "Discord is rate limiting the webhook; trying again in ${wait}s (attempt $attempt of 5)" + ;; + 000|5??) + wait=5 + echo "Discord answered HTTP $code; trying again in ${wait}s (attempt $attempt of 5)" + ;; + *) + echo "::error::Discord answered HTTP $code: $(cat response.json 2>/dev/null)" + exit 1 + ;; + esac + [ "$attempt" -lt 5 ] && sleep "$wait" + done + echo "::warning::The Discord post did not go through after 5 attempts (HTTP $code)" From 53c268b8a65e0560e2bf5d05dc9acb8c6ee94b12 Mon Sep 17 00:00:00 2001 From: eltanschauung <158000645+eltanschauung@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:18:49 -0400 Subject: [PATCH 4/4] Preserve existing workflow line endings --- .github/workflows/build.yml | 746 ++++++++++++++++++------------------ 1 file changed, 373 insertions(+), 373 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 84305f0f..b4c407eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,373 +1,373 @@ -name: Build - -# Debug and release builds of the native ports (Linux, Windows, Android) for -# every pushed commit, uploaded as artifacts. tools/ci_build.py does each -# build; see it for the options. -# -# Kept cheap: -# - one job per platform builds both configurations, so its set-up (tools, -# downloads, caches) is paid once; -# - compiles go through ccache, whose cache carries over from run to run, so -# a commit only recompiles what it changed; downloads (SDL, musl, Gradle) -# are cached too; -# - a newer push to the same branch cancels a run still going; -# - pushes that only change documentation do not build; -# - Windows, whose minutes cost twice Linux's, builds only Windows; -# - artifacts are kept for three days. -# -# A successful build of main is also published as a GitHub release, the -# newest marked latest, so that -# https://github.com//releases/latest/download/ always -# gives the latest build; the last five are kept. -on: - push: - paths-ignore: - - "**.md" - workflow_dispatch: - -concurrency: - group: build-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -env: - CI_COMPILER_LAUNCHER: ccache - CCACHE_DIR: ${{ github.workspace }}/.ccache - CCACHE_BASEDIR: ${{ github.workspace }} - CCACHE_MAXSIZE: 1G - -jobs: - linux: - # Arch Linux has what the build needs as packages: clang 22 (for the - # profile-guided optimisation in pgo/) and 32-bit SDL 3. - runs-on: ubuntu-24.04 - container: archlinux:latest - timeout-minutes: 40 - steps: - - name: Install the toolchain - run: | - printf '\n[multilib]\nInclude = /etc/pacman.d/mirrorlist\n' >> /etc/pacman.conf - pacman -Syu --noconfirm --needed git zstd python python-pytest ninja ccache \ - clang lld gcc lib32-glibc lib32-gcc-libs lib32-sdl3 - - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: .ccache - key: ccache-linux-${{ github.sha }} - restore-keys: ccache-linux- - - - name: Debug - run: python tools/ci_build.py linux debug - - - name: Release - run: python tools/ci_build.py linux release - - - name: Test - run: | - python -m pytest -q tools/test_linux_port.py - python tools/test_light_storage.py - - - name: Compile cache - run: ccache --show-stats - - - uses: actions/upload-artifact@v4 - with: - name: halo-linux-debug - path: dist/halo-linux-debug - retention-days: 3 - - - uses: actions/upload-artifact@v4 - with: - name: halo-linux-release - path: dist/halo-linux-release - retention-days: 3 - - android: - # The runner image has the Android SDK and NDK, a JDK and CMake. - runs-on: ubuntu-24.04 - timeout-minutes: 40 - steps: - - name: Install the toolchain - run: | - mkdir -p "$RUNNER_TEMP/tools" - curl -fsSL -o ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip - unzip -q ninja.zip -d "$RUNNER_TEMP/tools" && rm ninja.zip - curl -fsSL https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz \ - | tar -xJ -C "$RUNNER_TEMP/tools" --strip-components=1 ccache-4.10.2-linux-x86_64/ccache - echo "$RUNNER_TEMP/tools" >> "$GITHUB_PATH" - - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: .ccache - key: ccache-android-${{ github.sha }} - restore-keys: ccache-android- - - # musl and SDL, which configure.py fetches - - uses: actions/cache@v4 - with: - path: build/android/third_party - key: android-third-party-${{ hashFiles('tools/android_build.py') }} - - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: gradle-${{ hashFiles('port/android/**/*.gradle*', 'port/android/gradle/wrapper/gradle-wrapper.properties') }} - - # the app's signing key, from the ANDROID_KEYSTORE_BASE64 secret (a - # keystore, base64) and ANDROID_KEYSTORE_PASSWORD (its password, and - # the key's, alias halo): every build signed with it installs over the - # one before, as the app's self-updater (Updater.java) needs. Without - # them each build is signed with the runner's own debug key. - - name: Signing key - env: - KEYSTORE: ${{ secrets.ANDROID_KEYSTORE_BASE64 }} - run: | - if [ -n "$KEYSTORE" ]; then - printf '%s' "$KEYSTORE" | base64 -d > "$RUNNER_TEMP/halo.keystore" - echo "HALO_ANDROID_KEYSTORE=$RUNNER_TEMP/halo.keystore" >> "$GITHUB_ENV" - else - echo "::warning::no ANDROID_KEYSTORE_BASE64 secret: the Android builds cannot update over each other" - fi - - - name: Debug - env: - HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} - run: python3 tools/ci_build.py android debug - - - name: Release - env: - HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} - run: python3 tools/ci_build.py android release - - - name: Compile cache - run: ccache --show-stats - - - uses: actions/upload-artifact@v4 - with: - name: halo-android-debug - path: dist/halo-android-debug - retention-days: 3 - - - uses: actions/upload-artifact@v4 - with: - name: halo-android-release - path: dist/halo-android-release - retention-days: 3 - - windows: - # The runner image has LLVM, Python and Visual Studio's x86 libraries and - # Windows SDK. - runs-on: windows-2025 - timeout-minutes: 40 - defaults: - run: - shell: bash - steps: - - name: Install the toolchain - shell: pwsh - run: | - Invoke-WebRequest -OutFile ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip - Expand-Archive ninja.zip -DestinationPath "$env:RUNNER_TEMP\tools" - Invoke-WebRequest -OutFile ccache.zip https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-windows-x86_64.zip - Expand-Archive ccache.zip -DestinationPath $env:RUNNER_TEMP - Copy-Item "$env:RUNNER_TEMP\ccache-4.10.2-windows-x86_64\ccache.exe" "$env:RUNNER_TEMP\tools" - Remove-Item ninja.zip, ccache.zip - "$env:RUNNER_TEMP\tools" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH - - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: .ccache - key: ccache-windows-${{ github.sha }} - restore-keys: ccache-windows- - - # SDL's development package, which configure.py fetches - - uses: actions/cache@v4 - with: - path: build/windows/third_party - key: windows-third-party-${{ hashFiles('tools/windows_build.py') }} - - - name: Debug - run: python tools/ci_build.py windows debug - - - name: Release - run: python tools/ci_build.py windows release - - - name: Test light storage - run: python tools/test_light_storage.py - - - name: Compile cache - run: ccache --show-stats - - - uses: actions/upload-artifact@v4 - with: - name: halo-windows-debug - path: dist/halo-windows-debug - retention-days: 3 - - - uses: actions/upload-artifact@v4 - with: - name: halo-windows-release - path: dist/halo-windows-release - retention-days: 3 - - release: - # The six builds as a release, tagged build- and marked latest - # (every file keeps its name from release to release, for the README's - # latest/download links); then all but the newest five build-* releases - # go, tags and all. Only for main, and only when every platform built. - needs: [linux, android, windows] - if: ${{ github.ref == 'refs/heads/main' }} - runs-on: ubuntu-24.04 - timeout-minutes: 15 - permissions: - contents: write - outputs: - tag: ${{ steps.publish.outputs.tag }} - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - steps: - - uses: actions/download-artifact@v4 - with: - pattern: halo-* - path: artifacts - - - name: Package - run: | - mkdir release - for build in artifacts/*/; do - name="$(basename "$build")" - (cd "$build" && zip -qr "$GITHUB_WORKSPACE/release/$name.zip" .) - done - ls -l release - - - name: Publish - id: publish - env: - MESSAGE: ${{ github.event.head_commit.message }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - tag="build-$GITHUB_RUN_NUMBER" - subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1)" - printf '%s\n\nCommit: %s\nBuild: %s\n\nThe release builds are for playing; the debug builds stop at the first failed assertion and log it.\n' \ - "$subject" "$COMMIT_URL" "$RUN_URL" > notes.md - gh release create "$tag" release/*.zip --target "$GITHUB_SHA" --latest \ - --title "Build $GITHUB_RUN_NUMBER (${GITHUB_SHA:0:7})" --notes-file notes.md - echo "tag=$tag" >> "$GITHUB_OUTPUT" - - - name: Keep the last five - run: | - gh release list --limit 100 --json tagName,createdAt \ - --jq '[.[] | select(.tagName | startswith("build-"))] | sort_by(.createdAt) | reverse | .[5:] | .[].tagName' | - while read -r tag; do - gh release delete "$tag" --cleanup-tag --yes - done - - notify: - # A Discord embed for the run, to the DISCORD_WEBHOOK secret (nothing - # without it): green when every platform built, red when one did not, - # with each platform's result, this run's release (main only), and links - # to the latest release's builds. - needs: [linux, android, windows, release] - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - name: Post to Discord - env: - WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - LINUX: ${{ needs.linux.result }} - ANDROID: ${{ needs.android.result }} - WINDOWS: ${{ needs.windows.result }} - RELEASE_TAG: ${{ needs.release.outputs.tag }} - SERVER: ${{ github.server_url }} - REPOSITORY: ${{ github.repository }} - BRANCH: ${{ github.ref_name }} - SHA: ${{ github.sha }} - ACTOR: ${{ github.actor }} - MESSAGE: ${{ github.event.head_commit.message }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} - run: | - if [ -z "$WEBHOOK" ]; then - echo "no DISCORD_WEBHOOK secret: nothing to post" - exit 0 - fi - status() { - case "$1" in - success) echo "✅ Built" ;; - failure) echo "❌ Failed" ;; - cancelled) echo "⏹️ Cancelled" ;; - *) echo "➖ Skipped" ;; - esac - } - if [ "$LINUX" = success ] && [ "$ANDROID" = success ] && [ "$WINDOWS" = success ]; then - title="Build succeeded"; color=5763719 - else - title="Build failed"; color=15548997 - fi - latest="$SERVER/$REPOSITORY/releases/latest" - downloads="[Linux]($latest/download/halo-linux-release.zip) · [Windows]($latest/download/halo-windows-release.zip) · [Android]($latest/download/halo-android-release.zip) ([all builds]($latest))" - if [ -n "$RELEASE_TAG" ]; then - release="[$RELEASE_TAG]($SERVER/$REPOSITORY/releases/tag/$RELEASE_TAG)" - else - release="None (only a successful build of main makes one)" - fi - subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1 | cut -c1-200)" - jq -n \ - --arg title "$title" --argjson color "$color" --arg url "$RUN_URL" \ - --arg description "[\`${SHA:0:7}\`]($COMMIT_URL) $subject — $ACTOR ($BRANCH)" \ - --arg release "$release" --arg downloads "$downloads" \ - --arg linux "$(status "$LINUX")" --arg windows "$(status "$WINDOWS")" --arg android "$(status "$ANDROID")" \ - --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{embeds: [{ - title: $title, url: $url, color: $color, description: $description, timestamp: $timestamp, - fields: [ - {name: "Linux", value: $linux, inline: true}, - {name: "Windows", value: $windows, inline: true}, - {name: "Android", value: $android, inline: true}, - {name: "Release", value: $release}, - {name: "Latest release builds", value: $downloads} - ] - }]}' > payload.json - # Discord rate limits webhooks (HTTP 429, with how many seconds to - # wait in retry_after): wait and try again, a few times. A post - # still limited, or a network or Discord error, is a warning rather - # than a failed run; any other answer (a deleted webhook) fails. (At - # most 5 tries of 15 seconds and 4 waits of 30: inside the job's 5 - # minutes.) - for attempt in 1 2 3 4 5; do - code=$(curl -sS --max-time 15 -o response.json -w '%{http_code}' \ - -H "Content-Type: application/json" -d @payload.json "$WEBHOOK") || code=000 - case "$code" in - 2??) - exit 0 - ;; - 429) - wait=$(jq -r '.retry_after // 5' response.json 2>/dev/null || echo 5) - wait=$(awk -v w="$wait" 'BEGIN { w += 0; if (w < 1) w = 1; if (w > 30) w = 30; printf "%d", w + 0.999 }') - echo "Discord is rate limiting the webhook; trying again in ${wait}s (attempt $attempt of 5)" - ;; - 000|5??) - wait=5 - echo "Discord answered HTTP $code; trying again in ${wait}s (attempt $attempt of 5)" - ;; - *) - echo "::error::Discord answered HTTP $code: $(cat response.json 2>/dev/null)" - exit 1 - ;; - esac - [ "$attempt" -lt 5 ] && sleep "$wait" - done - echo "::warning::The Discord post did not go through after 5 attempts (HTTP $code)" +name: Build + +# Debug and release builds of the native ports (Linux, Windows, Android) for +# every pushed commit, uploaded as artifacts. tools/ci_build.py does each +# build; see it for the options. +# +# Kept cheap: +# - one job per platform builds both configurations, so its set-up (tools, +# downloads, caches) is paid once; +# - compiles go through ccache, whose cache carries over from run to run, so +# a commit only recompiles what it changed; downloads (SDL, musl, Gradle) +# are cached too; +# - a newer push to the same branch cancels a run still going; +# - pushes that only change documentation do not build; +# - Windows, whose minutes cost twice Linux's, builds only Windows; +# - artifacts are kept for three days. +# +# A successful build of main is also published as a GitHub release, the +# newest marked latest, so that +# https://github.com//releases/latest/download/ always +# gives the latest build; the last five are kept. +on: + push: + paths-ignore: + - "**.md" + workflow_dispatch: + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CI_COMPILER_LAUNCHER: ccache + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_MAXSIZE: 1G + +jobs: + linux: + # Arch Linux has what the build needs as packages: clang 22 (for the + # profile-guided optimisation in pgo/) and 32-bit SDL 3. + runs-on: ubuntu-24.04 + container: archlinux:latest + timeout-minutes: 40 + steps: + - name: Install the toolchain + run: | + printf '\n[multilib]\nInclude = /etc/pacman.d/mirrorlist\n' >> /etc/pacman.conf + pacman -Syu --noconfirm --needed git zstd python python-pytest ninja ccache \ + clang lld gcc lib32-glibc lib32-gcc-libs lib32-sdl3 + + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-linux-${{ github.sha }} + restore-keys: ccache-linux- + + - name: Debug + run: python tools/ci_build.py linux debug + + - name: Release + run: python tools/ci_build.py linux release + + - name: Test + run: | + python -m pytest -q tools/test_linux_port.py + python tools/test_light_storage.py + + - name: Compile cache + run: ccache --show-stats + + - uses: actions/upload-artifact@v4 + with: + name: halo-linux-debug + path: dist/halo-linux-debug + retention-days: 3 + + - uses: actions/upload-artifact@v4 + with: + name: halo-linux-release + path: dist/halo-linux-release + retention-days: 3 + + android: + # The runner image has the Android SDK and NDK, a JDK and CMake. + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - name: Install the toolchain + run: | + mkdir -p "$RUNNER_TEMP/tools" + curl -fsSL -o ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip + unzip -q ninja.zip -d "$RUNNER_TEMP/tools" && rm ninja.zip + curl -fsSL https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz \ + | tar -xJ -C "$RUNNER_TEMP/tools" --strip-components=1 ccache-4.10.2-linux-x86_64/ccache + echo "$RUNNER_TEMP/tools" >> "$GITHUB_PATH" + + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-android-${{ github.sha }} + restore-keys: ccache-android- + + # musl and SDL, which configure.py fetches + - uses: actions/cache@v4 + with: + path: build/android/third_party + key: android-third-party-${{ hashFiles('tools/android_build.py') }} + + - uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ hashFiles('port/android/**/*.gradle*', 'port/android/gradle/wrapper/gradle-wrapper.properties') }} + + # the app's signing key, from the ANDROID_KEYSTORE_BASE64 secret (a + # keystore, base64) and ANDROID_KEYSTORE_PASSWORD (its password, and + # the key's, alias halo): every build signed with it installs over the + # one before, as the app's self-updater (Updater.java) needs. Without + # them each build is signed with the runner's own debug key. + - name: Signing key + env: + KEYSTORE: ${{ secrets.ANDROID_KEYSTORE_BASE64 }} + run: | + if [ -n "$KEYSTORE" ]; then + printf '%s' "$KEYSTORE" | base64 -d > "$RUNNER_TEMP/halo.keystore" + echo "HALO_ANDROID_KEYSTORE=$RUNNER_TEMP/halo.keystore" >> "$GITHUB_ENV" + else + echo "::warning::no ANDROID_KEYSTORE_BASE64 secret: the Android builds cannot update over each other" + fi + + - name: Debug + env: + HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} + run: python3 tools/ci_build.py android debug + + - name: Release + env: + HALO_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }} + run: python3 tools/ci_build.py android release + + - name: Compile cache + run: ccache --show-stats + + - uses: actions/upload-artifact@v4 + with: + name: halo-android-debug + path: dist/halo-android-debug + retention-days: 3 + + - uses: actions/upload-artifact@v4 + with: + name: halo-android-release + path: dist/halo-android-release + retention-days: 3 + + windows: + # The runner image has LLVM, Python and Visual Studio's x86 libraries and + # Windows SDK. + runs-on: windows-2025 + timeout-minutes: 40 + defaults: + run: + shell: bash + steps: + - name: Install the toolchain + shell: pwsh + run: | + Invoke-WebRequest -OutFile ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip + Expand-Archive ninja.zip -DestinationPath "$env:RUNNER_TEMP\tools" + Invoke-WebRequest -OutFile ccache.zip https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-windows-x86_64.zip + Expand-Archive ccache.zip -DestinationPath $env:RUNNER_TEMP + Copy-Item "$env:RUNNER_TEMP\ccache-4.10.2-windows-x86_64\ccache.exe" "$env:RUNNER_TEMP\tools" + Remove-Item ninja.zip, ccache.zip + "$env:RUNNER_TEMP\tools" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH + + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-windows-${{ github.sha }} + restore-keys: ccache-windows- + + # SDL's development package, which configure.py fetches + - uses: actions/cache@v4 + with: + path: build/windows/third_party + key: windows-third-party-${{ hashFiles('tools/windows_build.py') }} + + - name: Debug + run: python tools/ci_build.py windows debug + + - name: Release + run: python tools/ci_build.py windows release + + - name: Test light storage + run: python tools/test_light_storage.py + + - name: Compile cache + run: ccache --show-stats + + - uses: actions/upload-artifact@v4 + with: + name: halo-windows-debug + path: dist/halo-windows-debug + retention-days: 3 + + - uses: actions/upload-artifact@v4 + with: + name: halo-windows-release + path: dist/halo-windows-release + retention-days: 3 + + release: + # The six builds as a release, tagged build- and marked latest + # (every file keeps its name from release to release, for the README's + # latest/download links); then all but the newest five build-* releases + # go, tags and all. Only for main, and only when every platform built. + needs: [linux, android, windows] + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + outputs: + tag: ${{ steps.publish.outputs.tag }} + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - uses: actions/download-artifact@v4 + with: + pattern: halo-* + path: artifacts + + - name: Package + run: | + mkdir release + for build in artifacts/*/; do + name="$(basename "$build")" + (cd "$build" && zip -qr "$GITHUB_WORKSPACE/release/$name.zip" .) + done + ls -l release + + - name: Publish + id: publish + env: + MESSAGE: ${{ github.event.head_commit.message }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + tag="build-$GITHUB_RUN_NUMBER" + subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1)" + printf '%s\n\nCommit: %s\nBuild: %s\n\nThe release builds are for playing; the debug builds stop at the first failed assertion and log it.\n' \ + "$subject" "$COMMIT_URL" "$RUN_URL" > notes.md + gh release create "$tag" release/*.zip --target "$GITHUB_SHA" --latest \ + --title "Build $GITHUB_RUN_NUMBER (${GITHUB_SHA:0:7})" --notes-file notes.md + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Keep the last five + run: | + gh release list --limit 100 --json tagName,createdAt \ + --jq '[.[] | select(.tagName | startswith("build-"))] | sort_by(.createdAt) | reverse | .[5:] | .[].tagName' | + while read -r tag; do + gh release delete "$tag" --cleanup-tag --yes + done + + notify: + # A Discord embed for the run, to the DISCORD_WEBHOOK secret (nothing + # without it): green when every platform built, red when one did not, + # with each platform's result, this run's release (main only), and links + # to the latest release's builds. + needs: [linux, android, windows, release] + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Post to Discord + env: + WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + LINUX: ${{ needs.linux.result }} + ANDROID: ${{ needs.android.result }} + WINDOWS: ${{ needs.windows.result }} + RELEASE_TAG: ${{ needs.release.outputs.tag }} + SERVER: ${{ github.server_url }} + REPOSITORY: ${{ github.repository }} + BRANCH: ${{ github.ref_name }} + SHA: ${{ github.sha }} + ACTOR: ${{ github.actor }} + MESSAGE: ${{ github.event.head_commit.message }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + run: | + if [ -z "$WEBHOOK" ]; then + echo "no DISCORD_WEBHOOK secret: nothing to post" + exit 0 + fi + status() { + case "$1" in + success) echo "✅ Built" ;; + failure) echo "❌ Failed" ;; + cancelled) echo "⏹️ Cancelled" ;; + *) echo "➖ Skipped" ;; + esac + } + if [ "$LINUX" = success ] && [ "$ANDROID" = success ] && [ "$WINDOWS" = success ]; then + title="Build succeeded"; color=5763719 + else + title="Build failed"; color=15548997 + fi + latest="$SERVER/$REPOSITORY/releases/latest" + downloads="[Linux]($latest/download/halo-linux-release.zip) · [Windows]($latest/download/halo-windows-release.zip) · [Android]($latest/download/halo-android-release.zip) ([all builds]($latest))" + if [ -n "$RELEASE_TAG" ]; then + release="[$RELEASE_TAG]($SERVER/$REPOSITORY/releases/tag/$RELEASE_TAG)" + else + release="None (only a successful build of main makes one)" + fi + subject="$(printf '%s' "${MESSAGE:-manual run}" | head -n 1 | cut -c1-200)" + jq -n \ + --arg title "$title" --argjson color "$color" --arg url "$RUN_URL" \ + --arg description "[\`${SHA:0:7}\`]($COMMIT_URL) $subject — $ACTOR ($BRANCH)" \ + --arg release "$release" --arg downloads "$downloads" \ + --arg linux "$(status "$LINUX")" --arg windows "$(status "$WINDOWS")" --arg android "$(status "$ANDROID")" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{embeds: [{ + title: $title, url: $url, color: $color, description: $description, timestamp: $timestamp, + fields: [ + {name: "Linux", value: $linux, inline: true}, + {name: "Windows", value: $windows, inline: true}, + {name: "Android", value: $android, inline: true}, + {name: "Release", value: $release}, + {name: "Latest release builds", value: $downloads} + ] + }]}' > payload.json + # Discord rate limits webhooks (HTTP 429, with how many seconds to + # wait in retry_after): wait and try again, a few times. A post + # still limited, or a network or Discord error, is a warning rather + # than a failed run; any other answer (a deleted webhook) fails. (At + # most 5 tries of 15 seconds and 4 waits of 30: inside the job's 5 + # minutes.) + for attempt in 1 2 3 4 5; do + code=$(curl -sS --max-time 15 -o response.json -w '%{http_code}' \ + -H "Content-Type: application/json" -d @payload.json "$WEBHOOK") || code=000 + case "$code" in + 2??) + exit 0 + ;; + 429) + wait=$(jq -r '.retry_after // 5' response.json 2>/dev/null || echo 5) + wait=$(awk -v w="$wait" 'BEGIN { w += 0; if (w < 1) w = 1; if (w > 30) w = 30; printf "%d", w + 0.999 }') + echo "Discord is rate limiting the webhook; trying again in ${wait}s (attempt $attempt of 5)" + ;; + 000|5??) + wait=5 + echo "Discord answered HTTP $code; trying again in ${wait}s (attempt $attempt of 5)" + ;; + *) + echo "::error::Discord answered HTTP $code: $(cat response.json 2>/dev/null)" + exit 1 + ;; + esac + [ "$attempt" -lt 5 ] && sleep "$wait" + done + echo "::warning::The Discord post did not go through after 5 attempts (HTTP $code)"