Build RommPS in the release workflow, never publish without the store zip

1.3.0 went out without PPSA76677.zip because RommPS was built by hand after
the release, and the step was missed. The release workflow now builds the
payloads, then RommPS against the pinned PS5 stack (RADV cached), and
publishes only once both have built. PS4-only re-releases are unchanged.

- tools/ps5-stack.sh checks out the stack at its pins
- tools/package-app.sh is the shared from-scratch build and zip
- tools/release-app.sh stays as the fallback for a release missing its zip
- Store workflow: hourly check that the latest release has the zip and
  that ProsperoStore lists it

.github/workflows/app.yml

@@ -0,0 +1,133 @@
+# Builds RommPS, the PS5 app, into PPSA76677.zip: the file ProsperoStore
+# installs. release.yml runs it between building the payloads and publishing,
+# so a PS5 release can't go out without it (1.3.0 did, back when it was built
+# by hand, and the store sat on 1.2.1 for a day).
+#
+# The PS5 stack is checked out at its pins (tools/ps5-stack.sh) and RADV, the
+# slow part, is built once per Mesa and SDK revision and cached. The zip ends
+# up as the artifact "rommps"; this workflow never touches a release.
+name: RommPS
+
+on:
+  workflow_call:
+    inputs:
+      sha:
+        description: "Commit to build"
+        type: string
+        required: true
+      tag:
+        description: "The release, e.g. v1.3.1"
+        type: string
+        required: true
+      payloads:
+        description: "Artifact holding that release's romm-sync.elf"
+        type: string
+        required: true
+  # Test runs only, removed before merging: builds this branch with v1.3.0's
+  # payload, and stops at the artifact.
+  push:
+    branches: [ensure-store-zip-creation]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 300
+    env:
+      GH_TOKEN: ${{ github.token }}
+      PS5_STACK: ${{ github.workspace }}/ps5-stack
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.sha || github.sha }}
+          path: repo
+
+      - name: Resolve version
+        id: version
+        run: |
+          tag="${{ inputs.tag }}"
+          if [ -z "$tag" ]; then
+            IFS=. read -r major minor patch < <(jq -r .contentVersion repo/platform/ps5/app/rommps/ps5/sce_sys/param.json)
+            tag="v$((10#$major)).$((10#$minor)).$((10#$patch))"
+          fi
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+          echo "Building RommPS for $tag"
+
+      # RommPS carries the payload and starts it: the release's own romm-sync.elf.
+      - uses: actions/download-artifact@v4
+        if: inputs.payloads
+        with:
+          name: ${{ inputs.payloads }}
+          path: payloads
+
+      - name: The payload
+        env:
+          TAG: ${{ steps.version.outputs.tag }}
+          FROM_RELEASE: ${{ inputs.payloads == '' }}
+        run: |
+          mkdir -p repo/build/ps5
+          if [ "$FROM_RELEASE" = true ]; then
+            gh release download "$TAG" --repo "$GITHUB_REPOSITORY" -p romm-sync.elf -O repo/build/ps5/romm-sync.elf
+          else
+            cp payloads/romm-sync.elf repo/build/ps5/romm-sync.elf
+          fi
+
+      - name: Check out the PS5 stack
+        id: stack
+        run: |
+          repo/tools/ps5-stack.sh "$PS5_STACK"
+          pin() { sed -n "s/^$2=//p" "$PS5_STACK/PS5_Vulkan/tools/$1"; }
+          echo "radv=$(pin build-radv.sh mesa_revision)-$(pin setup-native-dependencies.sh sdk_revision)" >> "$GITHUB_OUTPUT"
+
+      - uses: docker/setup-buildx-action@v3
+
+      # The image tools/ps5-app-build.sh runs everything in. The cache only
+      # saves time, so failing to write it must not fail the build.
+      - uses: docker/build-push-action@v6
+        with:
+          context: repo/platform/ps5/app/docker
+          tags: rommps-ps5app:1
+          load: true
+          cache-from: type=gha,scope=ps5app
+          cache-to: type=gha,mode=max,scope=ps5app,ignore-error=true
+
+      - uses: actions/cache@v4
+        with:
+          path: ps5-stack/PS5_Vulkan/.deps/native/radv-release
+          key: radv-release-${{ steps.stack.outputs.radv }}
+
+      # What ps5/tools/bootstrap.sh does, minus the console: PS5_Vulkan's SDK,
+      # its native tool and libc.prx (make builds those first, then a sample a
+      # title doesn't need), and RADV, which build-radv.sh skips when the
+      # cached archive is already at its pins.
+      - name: Build the PS5 stack
+        run: |
+          repo/tools/ps5-app-build.sh bash -c '
+            set -euo pipefail
+            cd /stack/PS5_Vulkan
+            bash tools/setup-native-dependencies.sh > /dev/null
+            make > build-make.log 2>&1 || true
+            [[ -x build/host/ps5-native-tool && -f runtime/libc.prx ]] || { tail -50 build-make.log; exit 1; }
+            time bash tools/build-radv.sh release'
+
+      - name: Title art
+        run: repo/tools/ps5-app-build.sh bash /repo/platform/ps5/app/tools/title-art.sh
+
+      - name: Build and package RommPS
+        run: repo/tools/package-app.sh "${{ steps.version.outputs.tag }}"
+
+      - name: Logs
+        if: failure()
+        run: |
+          shopt -s globstar nullglob
+          for log in ps5-stack/PS5_Vulkan/build-make.log ps5-stack/PS5_Vulkan/.deps/work/**/*.log repo/platform/ps5/app/rommps/build/*.log; do
+            echo "::group::$log"; tail -80 "$log"; echo "::endgroup::"
+          done
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: rommps
+          path: repo/build/release/PPSA76677.zip
+          if-no-files-found: error

.github/workflows/release.yml

@@ -1,8 +1,20 @@
-# Builds the PS5 and PS4 payloads and publishes a GitHub release. Triggered either by:
+# Builds the PS5 and PS4 payloads and RommPS (PPSA76677.zip, the file
+# ProsperoStore installs), then publishes them as one GitHub release. Nothing is
+# published unless all of it built: 1.3.0 went out without the zip, back when
+# RommPS was built by hand, and the store sat on 1.2.1 for a day.
+#
+# Triggered either by:
 #
 #   - merging a PR into main from a branch named with a version, e.g.
 #     release/v1.0.2 or v1.0.2-some-fix (the tag is created on the merge commit)
 #   - pushing a version tag manually: git tag v1.0.0 && git push origin v1.0.0
+#
+# Run by hand (Actions > Release > Run workflow, or
+# gh workflow run release.yml -f tag=v1.3.0 -f ref=release/v1.3.0), it instead
+# rebuilds only the PS4 payload from ref and swaps it into the existing tag's
+# release: a PS4 fix that doesn't make the PS5 download anything. Those already
+# on that version reinstall it by hand; the in-app update only offers newer ones.
+# The PS5's files, PPSA76677.zip included, stay exactly as they are.
 name: Release
 
 on:
@@ -11,21 +23,33 @@ on:
   pull_request:
     types: [closed]
     branches: [main]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Existing release whose PS4 payload is replaced (v1.3.0)"
+        required: true
+      ref:
+        description: "Branch or commit the PS4 payload is built from"
+        required: true
 
 permissions:
   contents: write
 
 jobs:
-  release:
+  payloads:
     if: >-
       github.event_name == 'push' ||
       (github.event.pull_request.merged &&
        contains(github.event.pull_request.head.ref, 'v'))
     runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.version.outputs.tag }}
+      sha: ${{ steps.version.outputs.sha }}
     steps:
       - name: Resolve version
         id: version
         env:
+          GH_TOKEN: ${{ github.token }}
           HEAD_REF: ${{ github.event.pull_request.head.ref }}
         run: |
           if [ "$GITHUB_EVENT_NAME" = push ]; then
@@ -39,6 +63,12 @@ jobs:
             echo "Branch '$HEAD_REF' has no vX.Y.Z version; skipping release."
             exit 0
           fi
+          # A release branch merged again after its release went out (a fix
+          # shipped with the manual PS4 run below, say).
+          if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+            echo "Release $tag already exists; skipping."
+            exit 0
+          fi
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
           echo "sha=$sha" >> "$GITHUB_OUTPUT"
 
@@ -47,13 +77,18 @@ jobs:
         with:
           ref: ${{ steps.version.outputs.sha }}
 
-      # RommPS (built with tools/release-app.sh, outside this workflow) is
-      # offered as an update by the homebrew catalog only if its contentVersion
-      # goes up, so the tag has to match it: v1.1.0 is 01.001.000.
-      - name: Check RommPS's contentVersion
+      # The catalog offers RommPS as an update only if its contentVersion goes
+      # up, so the tag has to match it: v1.1.0 is 01.001.000. And RommPS starts
+      # the payload it carries, so it must ask for this one. Both are checked
+      # again when it's built (tools/package-app.sh); here they fail in a
+      # minute rather than after the long build.
+      - name: Check versions and release notes
         if: steps.version.outputs.tag
         env:
           TAG: ${{ steps.version.outputs.tag }}
+        # The notes are written by hand, in docs/releases/<tag>.md: the homebrew
+        # catalog shows them as RommPS's release notes, and drops GitHub's
+        # generated list.
         run: |
           IFS=. read -r major minor patch <<< "${TAG#v}"
           want=$(printf "%02d.%03d.%03d" "$major" "$minor" "$patch")
@@ -62,6 +97,14 @@ jobs:
             echo "::error::RommPS's contentVersion is $have, $TAG needs $want (platform/ps5/app/rommps/ps5/sce_sys/param.json)"
             exit 1
           fi
+          if ! grep -q "kMinPayload = \"${TAG#v}\"" platform/ps5/app/rommps/examples/rommps/kit/rommps_app.hpp; then
+            echo "::error::kMinPayload in rommps_app.hpp isn't ${TAG#v}"
+            exit 1
+          fi
+          if [ ! -s "docs/releases/$TAG.md" ]; then
+            echo "::error::docs/releases/$TAG.md is missing; write the release notes before tagging"
+            exit 1
+          fi
 
       - uses: docker/setup-buildx-action@v3
         if: steps.version.outputs.tag
@@ -119,24 +162,108 @@ jobs:
           }
           sign romm-sync.elf manifest
           sign romm-sync-ps4.elf manifest-ps4
-          cd dist && sha256sum * > SHA256SUMS
 
-      # Creates the tag on the target commit if it doesn't exist yet. A tag pushed
-      # by GITHUB_TOKEN does not re-trigger this workflow, so there's no double release.
-      - name: Publish release
+      - uses: actions/upload-artifact@v4
         if: steps.version.outputs.tag
-        env:
-          GH_TOKEN: ${{ github.token }}
-          TAG: ${{ steps.version.outputs.tag }}
-          SHA: ${{ steps.version.outputs.sha }}
-        # The notes are written by hand, in docs/releases/<tag>.md: the homebrew
-        # catalog shows them as RommPS's release notes, and drops GitHub's
-        # generated list.
+        with:
+          name: payloads
+          path: dist/
+          if-no-files-found: error
+
+  app:
+    needs: payloads
+    if: needs.payloads.outputs.tag
+    uses: ./.github/workflows/app.yml
+    with:
+      sha: ${{ needs.payloads.outputs.sha }}
+      tag: ${{ needs.payloads.outputs.tag }}
+      payloads: payloads
+
+  # Only once the payloads and RommPS have both built. gh uploads every file
+  # before the release is published, so nobody sees half of one. It creates
+  # the tag on the target commit too; a tag pushed by GITHUB_TOKEN does not
+  # re-trigger this workflow, so there's no double release.
+  publish:
+    needs: [payloads, app]
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+      TAG: ${{ needs.payloads.outputs.tag }}
+      SHA: ${{ needs.payloads.outputs.sha }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.payloads.outputs.sha }}
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: Publish release
         run: |
-          notes="docs/releases/$TAG.md"
-          if [ ! -s "$notes" ]; then
-            echo "::error::$notes is missing; write the release notes before tagging"
+          for file in romm-sync.elf manifest.json manifest.sig romm-sync-ps4.elf romm-sync-ps4.pkg \
+                      manifest-ps4.json manifest-ps4.sig PPSA76677.zip; do
+            [ -s "dist/$file" ] || { echo "::error::dist/$file is missing"; exit 1; }
+          done
+          (cd dist && sha256sum * > SHA256SUMS)
+          gh release create "$TAG" dist/* \
+            --target "$SHA" --title "$TAG" --notes-file "docs/releases/$TAG.md"
+
+  replace-ps4:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+      TAG: ${{ inputs.tag }}
+    steps:
+      - name: Check the release
+        run: |
+          if ! [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "::error::'$TAG' isn't a vX.Y.Z tag"
             exit 1
           fi
-          gh release create "$TAG" dist/* \
-            --target "$SHA" --title "$TAG" --notes-file "$notes"
+          gh release view "$TAG" >/dev/null || { echo "::error::no release $TAG to update"; exit 1; }
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: platform/ps4/docker
+          tags: romm-sync-ps4sdk:v0.9-crt1352-oo0.5.4
+          load: true
+          cache-from: type=gha,scope=ps4
+          cache-to: type=gha,mode=max,scope=ps4,ignore-error=true
+
+      - name: Build the PS4 payload
+        run: tools/ps4-build.sh ps4-pkg VERSION="${TAG#v}"
+
+      - name: Sign
+        env:
+          RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
+        run: |
+          if [ -z "$RELEASE_SIGNING_KEY" ]; then
+            echo "::error::RELEASE_SIGNING_KEY secret is not set; in-app updates need a signed release."
+            exit 1
+          fi
+          make tools
+          mkdir dist
+          cp build/ps4/romm-sync-ps4.elf build/ps4/romm-sync-ps4.pkg dist/
+          build/host/release-sign manifest "$TAG" dist/romm-sync-ps4.elf > dist/manifest-ps4.json
+          build/host/release-sign sign dist/manifest-ps4.json > dist/manifest-ps4.sig
+          build/host/release-sign check dist/manifest-ps4.json dist/manifest-ps4.sig
+
+      # The PS5's files and their checksums stay as they are.
+      - name: Replace the PS4 files
+        run: |
+          cd dist
+          ps4="romm-sync-ps4.elf romm-sync-ps4.pkg manifest-ps4.json manifest-ps4.sig"
+          gh release download "$TAG" --pattern SHA256SUMS --output old-sums
+          grep -vE '  (romm-sync-ps4\.(elf|pkg)|manifest-ps4\.(json|sig))$' old-sums > SHA256SUMS || true
+          sha256sum $ps4 >> SHA256SUMS
+          gh release upload "$TAG" $ps4 SHA256SUMS --clobber

.github/workflows/store.yml

@@ -0,0 +1,72 @@
+# Makes sure the latest release actually reaches ProsperoStore. release.yml
+# doesn't publish a PS5 release without PPSA76677.zip any more (1.3.0 went out
+# without one and the store sat on 1.2.1 for a day); this is the backstop for
+# a release published some other way, and for the store side.
+#
+# Every hour it looks at the latest release:
+#
+#   - no zip at all (published by hand, say)
+#   - the store still lists an older version 12 hours on (the catalog's own
+#     bot proposes new releases three times a day)
+#
+# A problem opens an issue (one per release) and fails the run; once it's
+# fixed, the issue is closed. PS4-only re-releases (release.yml's replace-ps4)
+# never touch the zip, so they never come up here.
+name: Store
+
+on:
+  schedule:
+    - cron: "17 * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+      ZIP: PPSA76677.zip
+      TITLE_ID: PPSA76677
+      STORE_API: https://prosperostore.net/api/v1/versions.json
+    steps:
+      - name: Check the latest release
+        run: |
+          minutes_since() { echo $(( ($(date +%s) - $(date -d "$1" +%s)) / 60 )); }
+          failed=0
+
+          # report <tag> <problem, or empty when there's none>
+          report() {
+            local title="$1 isn't on ProsperoStore" open
+            open=$(gh issue list --state open --search "in:title \"$title\"" --json number,title |
+                   jq -r --arg t "$title" '.[] | select(.title == $t) | .number')
+            if [[ -z "$2" ]]; then
+              for n in $open; do gh issue close "$n" --comment "Fixed."; done
+              echo "$1: fine"
+              return
+            fi
+            [[ -n "$open" ]] || gh issue create --title "$title" --body "$2"
+            echo "::error::$2"
+            failed=1
+          }
+
+          release=$(gh api "repos/$GH_REPO/releases/latest")
+          tag=$(jq -r .tag_name <<< "$release")
+          problem=
+          if ! jq -e --arg zip "$ZIP" 'any(.assets[]; .name == $zip)' <<< "$release" >/dev/null; then
+            problem="$tag is published without $ZIP, so the store has nothing to pick up. Run \`tools/release-app.sh $tag --upload\` from a checkout of $tag."
+          # The store's API being down isn't our problem; the next run asks again.
+          elif listed=$(curl -fsS --retry 3 "$STORE_API" | jq -r --arg id "$TITLE_ID" '.apps[$id].version // empty'); then
+            if [[ "$listed" != "${tag#v}" ]] && (( $(minutes_since "$(jq -r .published_at <<< "$release")") >= 720 )); then
+              problem="$ZIP is attached to $tag, but the store still lists ${listed:-nothing}. Open the catalog update by hand (docs/release-checklist.md)."
+            fi
+          else
+            echo "::warning::couldn't read $STORE_API"
+            exit "$failed" # no verdict, so its issue stays as it is
+          fi
+          report "$tag" "$problem"
+
+          exit "$failed"

docs/release-checklist.md

@@ -45,5 +45,9 @@ For each one, start from the old version actually installed and set up, then upd
 
 ## After
 
-- The release workflow published the payloads, then `tools/release-app.sh <tag> --upload` attached `PPSA76677.zip`.
-- The catalog bot's pull request for the new version shows up the next day.
+Every PS5 release, alone or with the PS4, ships `PPSA76677.zip`. Only a PS4-only re-release (the manual Release run) skips it, and that never touches the PS5's files.
+
+- The release workflow builds the payloads, then RommPS (`.github/workflows/app.yml`), and publishes the release only once both have built. No zip, no release: rerun the failed job.
+- The store: the catalog bot proposes the new version within 8 hours. For sooner, open the pull request yourself on the catalog (`apps/PPSA76677.json`: `version`, `artifact_url`, `sha256`, `icon_url`), from the account that owns the repository; it's merged and live within minutes.
+- The Store workflow checks every hour and opens an issue if the latest release has no zip, or the store still lists the old version 12 hours after publishing.
+- A release that went out without the zip anyway: `git checkout <tag> && tools/release-app.sh <tag> --upload` builds it here and attaches it.

src/covers.c

@@ -22,8 +22,8 @@
 
 /* An asset RomM doesn't have is remembered for a day (a .missing file), so it
  * isn't asked for again on every page. The folder is pruned, oldest first,
- * past its limit: room for a whole library's covers on the PS5 (66 KB each on
- * average), less where storage is tight. The full pass stops short of it. */
+ * past its limit: on the consoles, room for some 30,000 covers (66 KB each on
+ * average). The full pass stops short of it. */
 #define MISSING_SEC (24 * 60 * 60)
 #define FULL_EVERY_SEC (24 * 60 * 60)
 #define WARM_SLOTS 256
@@ -37,17 +37,26 @@ static long long g_cached_bytes = -1; /* -1 until counted */
 static char g_warm[WARM_SLOTS][512];
 static int g_nwarm;
 
-/* The full pass: every game's cover path, worked through by the workers. */
-static char **g_all;
-static int g_nall, g_next, g_done;
+/* The full pass: every game's cover path, worked through by the workers. It
+ * stays on disk, a line each in cache/cover-pass (twenty thousand paths are
+ * more than the PS4 can spare), with its size and counts in cover-pass.json
+ * and how far it got in cover-pass.pos, so a restart carries on from there. */
+static FILE *g_pass;
+static int g_nall, g_next, g_done, g_total; /* g_total: the pass's size, after it's gone */
+/* The library was being looked at: the pass waits, its pages' covers don't. */
+static time_t g_browse_at;
+#define BROWSE_PAUSE_SEC 15
 static int g_refresh; /* a full pass was asked for */
 static char g_state[16] = "idle";
 
 static int is_ps5(void) { return strcmp(plat_info()->name, "ps5") == 0; }
 
-static long long cache_limit(void) { return is_ps5() ? 1024LL * 1024 * 1024 : 300LL * 1024 * 1024; }
+static long long cache_limit(void) { return strcmp(plat_info()->name, "host") ? 2048LL * 1024 * 1024 : 300LL * 1024 * 1024; }
 
-static int workers(void) { return is_ps5() ? 4 : 2; }
+static int is_ps4(void) { return strcmp(plat_info()->name, "ps4") == 0; }
+
+/* The PS4 payload shares a small memory pool with GoldHEN: one cover at a time. */
+static int workers(void) { return is_ps5() ? 4 : is_ps4() ? 1 : 2; }
 
 static void cache_path(const char *p, const char *suffix, char *out, size_t n) {
     uint64_t h = 0xcbf29ce484222325ULL; /* FNV-1a of the asset path and query */
@@ -92,58 +101,74 @@ static char *svg_with_size(const char *svg, size_t len, size_t *out_len) {
     return outp;
 }
 
-typedef struct {
-    time_t mtime;
-    long long size;
-    char name[200];
-} cached_file;
+/* Pruning goes by age without a list of the files, so it takes the same
+ * little memory for a hundred covers as for twenty thousand: one pass sorts
+ * their sizes into AGE_BINS slices of the folder's age span, the next deletes
+ * the oldest slices, whole, until the rest fit. */
+#define AGE_BINS 256
 
-static int by_age(const void *a, const void *b) {
-    time_t x = ((const cached_file *)a)->mtime, y = ((const cached_file *)b)->mtime;
-    return x < y ? -1 : x > y;
-}
-
-/* The folder's size; and, past the limit, the oldest files deleted until it's
- * back under 95% of it. One pass over the folder, on the planner's thread. */
-static long long cache_scan(int prune) {
-    char dir[PATH_MAX_LEN], path[PATH_MAX_LEN];
-    snprintf(dir, sizeof dir, "%s/cache/assets", plat_data_dir());
+/* Calls fn on each cached file (the folder's own, not dotfiles). */
+static void each_cached(const char *dir, void (*fn)(const char *path, const struct stat *st, void *ud), void *ud) {
+    char path[PATH_MAX_LEN];
     DIR *d = opendir(dir);
-    if (!d) return 0;
-    cached_file *files = NULL;
-    int n = 0, cap = 0;
-    long long total = 0;
+    if (!d) return;
     struct dirent *de;
     while ((de = readdir(d))) {
         struct stat st;
-        if (de->d_name[0] == '.' || strlen(de->d_name) >= sizeof files->name) continue;
+        if (de->d_name[0] == '.') continue;
         path_join(path, sizeof path, dir, de->d_name);
-        if (stat(path, &st) != 0) continue;
-        total += st.st_size;
-        if (!prune) continue;
-        if (n == cap) {
-            cached_file *grown = realloc(files, (size_t)(cap = cap ? cap * 2 : 1024) * sizeof *files);
-            if (!grown) break;
-            files = grown;
-        }
-        files[n].mtime = st.st_mtime;
-        files[n].size = st.st_size;
-        str_copy(files[n].name, sizeof files[n].name, de->d_name);
-        n++;
+        if (stat(path, &st) == 0 && S_ISREG(st.st_mode)) fn(path, &st, ud);
     }
     closedir(d);
-    if (prune && total > cache_limit()) {
-        qsort(files, (size_t)n, sizeof *files, by_age);
-        for (int i = 0; i < n && total > cache_limit() / 20 * 19; i++) {
-            path_join(path, sizeof path, dir, files[i].name);
-            if (unlink(path) == 0) total -= files[i].size;
-        }
+}
+
+struct age_scan {
+    long long total, bins[AGE_BINS];
+    time_t oldest, newest;
+    int pass, cutoff; /* cutoff: the newest slice deleted */
+};
+
+static int age_bin(const struct age_scan *a, time_t t) {
+    const long long span = (long long)(a->newest - a->oldest) + 1;
+    const long long b = (long long)(t - a->oldest) * AGE_BINS / span;
+    return b < 0 ? 0 : b >= AGE_BINS ? AGE_BINS - 1 : (int)b;
+}
+
+static void age_visit(const char *path, const struct stat *st, void *ud) {
+    struct age_scan *a = ud;
+    if (a->pass == 0) {
+        a->total += st->st_size;
+        if (!a->oldest || st->st_mtime < a->oldest) a->oldest = st->st_mtime;
+        if (st->st_mtime > a->newest) a->newest = st->st_mtime;
+    } else if (a->pass == 1) {
+        a->bins[age_bin(a, st->st_mtime)] += st->st_size;
+    } else if (age_bin(a, st->st_mtime) <= a->cutoff && unlink(path) == 0) {
+        a->total -= st->st_size;
+    }
+}
+
+/* The folder's size; and, past the limit, the oldest files deleted until it's
+ * back under 95% of it. On the planner's thread. */
+static long long cache_scan(int prune) {
+    char dir[PATH_MAX_LEN];
+    snprintf(dir, sizeof dir, "%s/cache/assets", plat_data_dir());
+    struct age_scan a;
+    memset(&a, 0, sizeof a);
+    each_cached(dir, age_visit, &a);
+    if (prune && a.total > cache_limit()) {
+        a.pass = 1;
+        each_cached(dir, age_visit, &a);
+        long long over = a.total - cache_limit() / 20 * 19, freed = 0;
+        int last = 0;
+        while (last < AGE_BINS - 1 && (freed += a.bins[last]) < over) last++;
+        a.cutoff = last;
+        a.pass = 2;
+        each_cached(dir, age_visit, &a);
     }
-    free(files);
     pthread_mutex_lock(&g_lock);
-    g_cached_bytes = total;
+    g_cached_bytes = a.total;
     pthread_mutex_unlock(&g_lock);
-    return total;
+    return a.total;
 }
 
 static void cache_store(const char *path, const void *body, size_t len) {
@@ -229,6 +254,9 @@ int covers_fetch(const char *p, char **out, size_t *out_len, const char **type)
 }
 
 void covers_warm(const char *p) {
+    pthread_mutex_lock(&g_lock);
+    g_browse_at = time(NULL);
+    pthread_mutex_unlock(&g_lock);
     if (!valid_path(p) || have(p)) return;
     pthread_mutex_lock(&g_lock);
     for (int i = 0; i < g_nwarm; i++)
@@ -262,47 +290,18 @@ static int enabled(void) {
 /* The console is busy with something that matters more. */
 static int busy(void) { return sync_game_running() || sync_is_running() || library_busy(); }
 
-struct cover_list { char **list; int n, cap; };
-
-/* The covers seen first go first: the games on this console, then each
- * platform's tile, then the rest in RomM's order. */
-static void prioritize(struct cover_list *c, const char *heads) {
-    char **out = malloc(sizeof *out * (size_t)(c->n ? c->n : 1));
-    char *taken = calloc((size_t)(c->n ? c->n : 1), 1);
-    if (!out || !taken) { free(out); free(taken); return; }
-    int n = 0;
-    state_lock();
-    const cJSON *e;
-    cJSON_ArrayForEach(e, state_section("roms")) {
-        const char *cover = jget_str(e, "cover", "");
-        if (!cover[0] || !(file_exists(e->string) || dir_exists(e->string))) continue;
-        for (int i = 0; i < c->n; i++)
-            if (!taken[i] && !strcmp(c->list[i], cover)) { taken[i] = 1; out[n++] = c->list[i]; break; }
-    }
-    state_unlock();
-    for (int pass = 0; pass < 2; pass++)
-        for (int i = 0; i < c->n; i++)
-            if (!taken[i] && (pass || heads[i])) { taken[i] = 1; out[n++] = c->list[i]; }
-    free(taken);
-    free(c->list);
-    c->list = out;
+static void pass_path(char *out, size_t n, const char *suffix) {
+    snprintf(out, n, "%s/cache/cover-pass%s", plat_data_dir(), suffix);
 }
 
-static int collect_cover(const cJSON *it, const cJSON *page, int first, void *ctx) {
-    (void)page, (void)first;
-    if (!it) return 0;
-    const char *p = jget_str(it, "path_cover_small", "");
-    if (!valid_path(p)) return 0;
-    struct cover_list *c = ctx;
-    if (c->n >= 100000) return 0;
-    if (c->n == c->cap) {
-        c->cap = c->cap ? c->cap * 2 : 512;
-        char **grown = realloc(c->list, sizeof *grown * (size_t)c->cap);
-        if (!grown) return -1;
-        c->list = grown;
-    }
-    c->list[c->n++] = strdup(p);
-    return 0;
+/* Where the pass got to: every so many covers, not each one. Call with g_lock held. */
+static void save_pos(void) {
+    char path[PATH_MAX_LEN], pos[64];
+    long at = g_pass ? ftell(g_pass) : -1;
+    if (at < 0) return;
+    pass_path(path, sizeof path, ".pos");
+    int len = snprintf(pos, sizeof pos, "%d %ld\n", g_next, at);
+    write_file_atomic(path, pos, (size_t)len);
 }
 
 /* Each platform's game count when a pass last listed it: the daily pass
@@ -311,13 +310,140 @@ static int collect_cover(const cJSON *it, const cJSON *page, int first, void *ct
  * kept once its covers are done, so an unfinished one is listed again. */
 static cJSON *g_pass_counts;
 
-/* Every game's cover path, platform by platform; all: even the platforms
- * whose game count didn't change. 0 when RomM gave the whole list; -1 if it
- * couldn't be reached part way (what came is still used). */
+static void close_pass(void) {
+    if (g_pass) fclose(g_pass);
+    g_pass = NULL;
+    g_nall = g_next = 0;
+}
+
+/* A pass that's over (done, or stopped by a full cache) goes, files and all.
+ * Call with g_lock held. */
+static void drop_pass(void) {
+    char path[PATH_MAX_LEN];
+    close_pass();
+    static const char *const files[] = {"", ".json", ".pos"};
+    for (size_t i = 0; i < sizeof files / sizeof files[0]; i++) {
+        pass_path(path, sizeof path, files[i]);
+        unlink(path);
+    }
+}
+
+/* Opens the pass on disk where it was left (the top, for a new one). 0 if
+ * there's one to carry on with. Call with g_lock held. */
+static int open_pass(void) {
+    char path[PATH_MAX_LEN];
+    close_pass();
+    pass_path(path, sizeof path, ".json");
+    char *js = read_file(path, NULL);
+    cJSON *head = js ? cJSON_Parse(js) : NULL;
+    free(js);
+    pass_path(path, sizeof path, "");
+    g_pass = head ? fopen(path, "r") : NULL;
+    if (!g_pass) {
+        cJSON_Delete(head);
+        return -1;
+    }
+    setvbuf(g_pass, NULL, _IOFBF, 4096);
+    int next = 0;
+    long at = 0;
+    pass_path(path, sizeof path, ".pos");
+    char *pos = read_file(path, NULL);
+    if (!pos || sscanf(pos, "%d %ld", &next, &at) != 2 || next < 0 || at < 0 || fseek(g_pass, at, SEEK_SET) != 0) {
+        next = 0;
+        fseek(g_pass, 0, SEEK_SET);
+    }
+    free(pos);
+    g_nall = g_total = (int)jget_num(head, "total", 0);
+    g_next = g_done = next;
+    cJSON_Delete(g_pass_counts);
+    g_pass_counts = cJSON_DetachItemFromObjectCaseSensitive(head, "counts");
+    cJSON_Delete(head);
+    return 0;
+}
+
+/* While RomM lists the games: each platform's first covers (its tile) in one
+ * file, the rest in another. */
+struct pass_build { FILE *heads, *rest; int n, in_plat; };
+
+static int collect_cover(const cJSON *it, const cJSON *page, int first, void *ctx) {
+    (void)page, (void)first;
+    if (!it) return 0;
+    const char *p = jget_str(it, "path_cover_small", "");
+    if (!valid_path(p) || strlen(p) >= 500 || strchr(p, '\n')) return 0;
+    struct pass_build *b = ctx;
+    if (fprintf(b->in_plat++ < 3 ? b->heads : b->rest, "%s\n", p) < 0) return -1;
+    b->n++;
+    return 0;
+}
+
+static int by_str(const void *a, const void *b) { return strcmp(*(char *const *)a, *(char *const *)b); }
+
+/* Goes through one file's lines, leaving out those in mine (sorted): with
+ * out, copies the rest onto it, and says how many; without, marks in seen
+ * the ones of mine it came across. */
+static int merge_lines(FILE *out, const char *path, char **mine, int nmine, char *seen) {
+    char line[512];
+    FILE *in = fopen(path, "r");
+    int n = 0;
+    while (in && fgets(line, sizeof line, in)) {
+        line[strcspn(line, "\r\n")] = 0;
+        const char *key = line;
+        char **hit = nmine ? bsearch(&key, mine, (size_t)nmine, sizeof *mine, by_str) : NULL;
+        if (hit && seen) seen[hit - mine] = 1;
+        if (hit || !out) continue;
+        fprintf(out, "%s\n", line);
+        n++;
+    }
+    if (in) fclose(in);
+    return n;
+}
+
+/* The covers of the games on this console, sorted, without repeats. */
+static char **my_covers(int *n) {
+    char **mine = NULL;
+    int cap = 0;
+    *n = 0;
+    state_lock();
+    const cJSON *e;
+    cJSON_ArrayForEach(e, state_section("roms")) {
+        const char *cover = jget_str(e, "cover", "");
+        if (!valid_path(cover) || !(file_exists(e->string) || dir_exists(e->string))) continue;
+        if (*n == cap) {
+            char **grown = realloc(mine, sizeof *mine * (size_t)(cap = cap ? cap * 2 : 64));
+            if (!grown) break;
+            mine = grown;
+        }
+        if ((mine[*n] = strdup(cover))) (*n)++;
+    }
+    state_unlock();
+    if (*n) qsort(mine, (size_t)*n, sizeof *mine, by_str);
+    int kept = 0;
+    for (int i = 0; i < *n; i++) {
+        if (kept && !strcmp(mine[kept - 1], mine[i])) free(mine[i]);
+        else mine[kept++] = mine[i];
+    }
+    *n = kept;
+    return mine;
+}
+
+/* Every game's cover path, platform by platform, into a new pass on disk;
+ * all: even the platforms whose game count didn't change. The covers seen
+ * first go first: the games on this console, then each platform's tile, then
+ * the rest in RomM's order. 0 when RomM gave the whole list; -1 if it couldn't
+ * be reached part way (what came is still used). */
 static int list_all(int all) {
-    struct cover_list c = {0};
-    static char heads[100000]; /* a platform's first covers (its tile) */
-    memset(heads, 0, sizeof heads);
+    char heads[PATH_MAX_LEN], rest[PATH_MAX_LEN], fresh[PATH_MAX_LEN], path[PATH_MAX_LEN];
+    pass_path(heads, sizeof heads, ".heads");
+    pass_path(rest, sizeof rest, ".rest");
+    pass_path(fresh, sizeof fresh, ".new");
+    mkdir_parent(heads);
+    struct pass_build b = {fopen(heads, "w"), fopen(rest, "w"), 0, 0};
+    if (!b.heads || !b.rest) {
+        if (b.heads) fclose(b.heads);
+        if (b.rest) fclose(b.rest);
+        LOGW("covers: can't write %s", heads);
+        return -1;
+    }
     long st = 0;
     cJSON *plats = romm_call("GET", "/api/platforms", NULL, &st), *p;
     int ok = st == 200 && cJSON_IsArray(plats), same = 0;
@@ -328,38 +454,76 @@ static int list_all(int all) {
     cJSON_ArrayForEach(p, plats) {
         const double n = jget_num(p, "rom_count", 0);
         if (n <= 0) continue;
-        char id[24], path[160];
+        char id[24], list[160];
         snprintf(id, sizeof id, "%d", (int)jget_num(p, "id", 0));
         if (!all && jget_num(had, id, -1) == n) {
             cJSON_AddNumberToObject(counts, id, n);
             same++;
             continue;
         }
-        snprintf(path, sizeof path, "/api/roms?platform_ids=%s&order_by=name&order_dir=asc&with_char_index=false" ROMS_LEAN, id);
-        int before = c.n;
-        if (romm_list(path, 0, 100000, collect_cover, &c, &st) < 0) ok = 0;
+        snprintf(list, sizeof list, "/api/roms?platform_ids=%s&order_by=name&order_dir=asc&with_char_index=false" ROMS_LEAN, id);
+        b.in_plat = 0;
+        if (romm_list(list, 0, 100000, collect_cover, &b, &st) < 0) ok = 0;
         else cJSON_AddNumberToObject(counts, id, n);
-        for (int k = before; k < c.n && k < before + 3; k++) heads[k] = 1;
     }
     cJSON_Delete(had);
     cJSON_Delete(plats);
-    prioritize(&c, heads);
+    const int written = fclose(b.heads) == 0 && fclose(b.rest) == 0;
+
+    FILE *out = written ? fopen(fresh, "w") : NULL;
+    int total = 0, nmine = 0;
+    if (out) {
+        /* This console's games first, those RomM listed (a cover there since
+         * replaced has a new path), then the tiles and the rest without them. */
+        char **mine = my_covers(&nmine);
+        char *seen = calloc((size_t)nmine + 1, 1);
+        merge_lines(NULL, heads, mine, seen ? nmine : 0, seen);
+        merge_lines(NULL, rest, mine, seen ? nmine : 0, seen);
+        for (int i = 0; seen && i < nmine; i++)
+            if (seen[i]) {
+                fprintf(out, "%s\n", mine[i]);
+                total++;
+            }
+        total += merge_lines(out, heads, mine, seen ? nmine : 0, NULL);
+        total += merge_lines(out, rest, mine, seen ? nmine : 0, NULL);
+        for (int i = 0; i < nmine; i++) free(mine[i]);
+        free(mine);
+        free(seen);
+        if (fclose(out) != 0) total = -1;
+    }
+    unlink(heads);
+    unlink(rest);
+    cJSON *head = cJSON_CreateObject();
+    cJSON_AddNumberToObject(head, "total", total);
+    cJSON_AddItemToObject(head, "counts", counts);
+    char *js = cJSON_PrintUnformatted(head);
+    cJSON_Delete(head);
+
     pthread_mutex_lock(&g_lock);
-    for (int i = 0; i < g_nall; i++) free(g_all[i]);
-    free(g_all);
-    g_all = c.list;
-    g_nall = c.n;
-    g_next = g_done = 0;
-    cJSON_Delete(g_pass_counts);
-    g_pass_counts = counts;
+    drop_pass();
+    int opened = -1;
+    pass_path(path, sizeof path, "");
+    if (total >= 0 && js && rename(fresh, path) == 0) {
+        char meta[PATH_MAX_LEN];
+        pass_path(meta, sizeof meta, ".json");
+        if (write_file_atomic(meta, js, strlen(js)) == 0) opened = open_pass();
+    }
+    if (opened != 0) drop_pass();
     pthread_mutex_unlock(&g_lock);
-    LOGI("covers: %d games to check%s%s", c.n, ok ? "" : " (RomM couldn't be reached for all of them)",
+    unlink(fresh);
+    free(js);
+    if (opened != 0) {
+        LOGW("covers: couldn't keep the list of %d games to check", b.n);
+        return -1;
+    }
+    LOGI("covers: %d games to check%s%s", total, ok ? "" : " (RomM couldn't be reached for all of them)",
          same ? ", the platforms with as many games as last time skipped" : "");
     return ok ? 0 : -1;
 }
 
 /* The next cover to fetch: the pages being looked at first, then the full
- * pass. "" when there's nothing to do. */
+ * pass, which waits while the library is being looked at. "" when there's
+ * nothing to do. */
 static void next_job(char *out, size_t n, int *from_pass) {
     out[0] = 0;
     *from_pass = 0;
@@ -368,9 +532,16 @@ static void next_job(char *out, size_t n, int *from_pass) {
         str_copy(out, n, g_warm[0]);
         g_nwarm--;
         memmove(g_warm[0], g_warm[1], sizeof g_warm[0] * (size_t)g_nwarm);
-    } else if (g_next < g_nall) {
-        str_copy(out, n, g_all[g_next++]);
-        *from_pass = 1;
+    } else if (g_pass && g_next < g_nall && time(NULL) - g_browse_at >= BROWSE_PAUSE_SEC) {
+        char line[512];
+        if (fgets(line, sizeof line, g_pass)) {
+            line[strcspn(line, "\r\n")] = 0;
+            str_copy(out, n, line);
+            *from_pass = 1;
+            if (++g_next % 25 == 0) save_pos();
+        } else {
+            g_next = g_nall; /* shorter than it said: it's done anyway */
+        }
     }
     pthread_mutex_unlock(&g_lock);
 }
@@ -390,7 +561,10 @@ static void *worker(void *arg) {
         next_job(p, sizeof p, &from_pass);
         if (!p[0]) {
             pthread_mutex_lock(&g_lock);
-            if (g_next >= g_nall && g_nall) str_copy(g_state, sizeof g_state, "done");
+            if (g_next >= g_nall && g_nall) {
+                str_copy(g_state, sizeof g_state, "done");
+                drop_pass();
+            }
             struct timespec until;
             clock_gettime(CLOCK_REALTIME, &until);
             until.tv_sec += 30;
@@ -406,7 +580,7 @@ static void *worker(void *arg) {
             /* Stop short of the limit, so pruning never undoes the pass. */
             if (used >= 0 && used > cache_limit() * 9 / 10) {
                 pthread_mutex_lock(&g_lock);
-                g_next = g_nall;
+                drop_pass();
                 str_copy(g_state, sizeof g_state, "full");
                 pthread_mutex_unlock(&g_lock);
                 LOGW("covers: the cache is nearly full, the rest wait");
@@ -426,6 +600,14 @@ static void *worker(void *arg) {
             char why[160];
             if (covers_fetch(p, &body, &len, &type) == 0) {
                 free(body);
+            } else if (http_out_of_memory()) {
+                /* No room for it now: it goes first again once some is free,
+                 * rather than every cover after it failing the same way. */
+                covers_warm(p);
+                pthread_mutex_lock(&g_lock);
+                str_copy(g_state, sizeof g_state, "waiting");
+                pthread_mutex_unlock(&g_lock);
+                sleep(10);
             } else if (romm_no_answer(why, sizeof why)) {
                 /* Not skipped: it goes first again, once RomM answers (this
                  * worker's next try finds out). */
@@ -449,6 +631,9 @@ static void *worker(void *arg) {
 static void *planner(void *arg) {
     (void)arg;
     cache_scan(0);
+    pthread_mutex_lock(&g_lock);
+    if (open_pass() == 0 && g_next < g_nall) LOGI("covers: carrying on, %d of %d games left to check", g_nall - g_next, g_nall);
+    pthread_mutex_unlock(&g_lock);
     /* How often it looks for work; ROMM_SYNC_COVERS_TICK shortens it in tests. */
     const char *env = getenv("ROMM_SYNC_COVERS_TICK");
     unsigned tick = env && atoi(env) > 0 ? (unsigned)atoi(env) : 20;
@@ -511,7 +696,7 @@ cJSON *covers_status(void) {
     pthread_mutex_lock(&g_lock);
     cJSON_AddStringToObject(j, "state", g_state);
     cJSON_AddNumberToObject(j, "done", g_done);
-    cJSON_AddNumberToObject(j, "total", g_nall);
+    cJSON_AddNumberToObject(j, "total", g_total);
     cJSON_AddNumberToObject(j, "cached_mb", g_cached_bytes > 0 ? (double)(g_cached_bytes / (1024 * 1024)) : 0);
     pthread_mutex_unlock(&g_lock);
     cJSON_AddNumberToObject(j, "limit_mb", (double)(cache_limit() / (1024 * 1024)));

src/http.c

@@ -17,6 +17,9 @@
 #include "util.h"
 #ifdef ROMM_EMBED_CA
 #include "cacert_pem.h" /* generated from the SDK's Mozilla CA bundle */
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
 #endif
 
 static char g_ca_bundle[PATH_MAX_LEN];
@@ -24,6 +27,12 @@ static char g_ca_bundle[PATH_MAX_LEN];
  * which have no system CA store. */
 static char *g_ca_blob;
 static size_t g_ca_blob_len;
+#ifdef ROMM_EMBED_CA
+/* The same roots, parsed once and handed to every connection. Given as a
+ * blob, libcurl parses all ~140 of them again for each new connection and
+ * caches nothing, which the PS4's memory pool can't afford. */
+static X509_STORE *g_ca_store;
+#endif
 static int g_tls_verify = 1;
 static char g_user_agent[64];
 
@@ -42,6 +51,7 @@ static pthread_mutex_t g_share_locks[CURL_LOCK_DATA_LAST];
  * the freshest) or a new one, and gives it back; POOL_MAX are kept. */
 #define POOL_MAX 8
 static CURL *g_pool[POOL_MAX];
+static int pool_max(void) { return strcmp(plat_info()->name, "ps4") ? POOL_MAX : 3; }
 static int g_npool;
 static pthread_mutex_t g_pool_lock = PTHREAD_MUTEX_INITIALIZER;
 
@@ -57,7 +67,7 @@ static CURL *handle_take(void) {
 
 static void handle_give(CURL *c) {
     pthread_mutex_lock(&g_pool_lock);
-    const int kept = g_npool < POOL_MAX;
+    const int kept = g_npool < pool_max();
     if (kept) g_pool[g_npool++] = c;
     pthread_mutex_unlock(&g_pool_lock);
     if (!kept) curl_easy_cleanup(c);
@@ -69,6 +79,23 @@ static double mono_sec(void) {
     return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;
 }
 
+/* When a request last failed for want of memory: a reply we couldn't keep in
+ * memory (a download's write error is the disk's), or curl or OpenSSL coming
+ * up empty. */
+static double g_oom_at = -1;
+static pthread_mutex_t g_oom_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static int out_of_memory(CURLcode rc, int in_memory) {
+    return rc == CURLE_OUT_OF_MEMORY || rc == CURLE_SSL_CACERT_BADFILE || (in_memory && rc == CURLE_WRITE_ERROR);
+}
+
+int http_out_of_memory(void) {
+    pthread_mutex_lock(&g_oom_lock);
+    const int recent = g_oom_at >= 0 && mono_sec() - g_oom_at < 30;
+    pthread_mutex_unlock(&g_oom_lock);
+    return recent;
+}
+
 /* A server that doesn't answer at all (no connection) isn't asked again by
  * every thread, each waiting out the connect timeout: requests to it fail at
  * once, and one at a time checks whether it's back, after 5 s, then twice as
@@ -166,6 +193,37 @@ static void share_unlock(CURL *c, curl_lock_data data, void *ud) {
     if (data < CURL_LOCK_DATA_LAST) pthread_mutex_unlock(&g_share_locks[data]);
 }
 
+#ifdef ROMM_EMBED_CA
+/* What libcurl does with a CA blob (load_cacert_from_memory and the flags it
+ * sets), done once. NULL if not one certificate in it could be read. */
+static X509_STORE *ca_store(const char *pem, size_t len) {
+    X509_STORE *st = X509_STORE_new();
+    BIO *bio = st && len <= 0x7fffffff ? BIO_new_mem_buf(pem, (int)len) : NULL;
+    STACK_OF(X509_INFO) *infos = bio ? PEM_X509_INFO_read_bio(bio, NULL, NULL, NULL) : NULL;
+    int n = 0;
+    for (int i = 0; infos && i < sk_X509_INFO_num(infos); i++) {
+        X509_INFO *inf = sk_X509_INFO_value(infos, i);
+        if (inf->x509 && X509_STORE_add_cert(st, inf->x509)) n++;
+    }
+    sk_X509_INFO_pop_free(infos, X509_INFO_free);
+    BIO_free(bio);
+    if (!n) {
+        X509_STORE_free(st);
+        return NULL;
+    }
+    X509_STORE_set_flags(st, X509_V_FLAG_TRUSTED_FIRST | X509_V_FLAG_PARTIAL_CHAIN);
+    return st;
+}
+
+/* Every connection's SSL_CTX gets the one store; libcurl's own, empty, goes. */
+static CURLcode use_ca_store(CURL *c, void *ssl_ctx, void *ud) {
+    (void)c;
+    if (!X509_STORE_up_ref(ud)) return CURLE_OUT_OF_MEMORY;
+    SSL_CTX_set_cert_store(ssl_ctx, ud);
+    return CURLE_OK;
+}
+#endif
+
 void http_global_init(const char *ca_bundle, int tls_verify) {
     curl_global_init(CURL_GLOBAL_ALL);
     for (int i = 0; i < CURL_LOCK_DATA_LAST; i++) pthread_mutex_init(&g_share_locks[i], NULL);
@@ -193,6 +251,11 @@ void http_global_init(const char *ca_bundle, int tls_verify) {
         }
     }
     free(user);
+    const char *tls = curl_version_info(CURLVERSION_NOW)->ssl_version;
+    if (g_ca_blob && tls && !strncmp(tls, "OpenSSL", 7) && (g_ca_store = ca_store(g_ca_blob, g_ca_blob_len))) {
+        free(g_ca_blob);
+        g_ca_blob = NULL;
+    }
     LOGI("libcurl %s, CA bundle: built-in%s%s", curl_version_info(CURLVERSION_NOW)->version,
          g_ca_bundle[0] ? " + " : "", g_ca_bundle);
 #else
@@ -283,6 +346,14 @@ static CURL *easy_new(const char *url, struct curl_slist **hdrs, const char *aut
 #ifdef ROMM_NETLOG
     curl_easy_setopt(c, CURLOPT_DEBUGFUNCTION, curl_trace);
     curl_easy_setopt(c, CURLOPT_VERBOSE, 1L);
+#endif
+#ifdef ROMM_EMBED_CA
+    if (g_ca_store) {
+        curl_easy_setopt(c, CURLOPT_CAINFO, NULL);
+        curl_easy_setopt(c, CURLOPT_CAPATH, NULL);
+        curl_easy_setopt(c, CURLOPT_SSL_CTX_FUNCTION, use_ca_store);
+        curl_easy_setopt(c, CURLOPT_SSL_CTX_DATA, g_ca_store);
+    } else
 #endif
     if (g_ca_blob) {
         struct curl_blob blob = {g_ca_blob, g_ca_blob_len, CURL_BLOB_NOCOPY};
@@ -310,6 +381,11 @@ static int finish(CURL *c, CURLcode rc, membuf *m, http_resp *out, const char *u
     down_note(url, out->no_answer, probe);
     if (rc != CURLE_OK) {
         snprintf(out->error, sizeof out->error, "%s", curl_easy_strerror(rc));
+        if (out_of_memory(rc, m != NULL)) {
+            pthread_mutex_lock(&g_oom_lock);
+            g_oom_at = mono_sec();
+            pthread_mutex_unlock(&g_oom_lock);
+        }
         /* A download (CURLOPT_FAILONERROR) stopped by the server's answer:
          * say which, so a 404 isn't "HTTP response code said error". */
         if (rc == CURLE_HTTP_RETURNED_ERROR) {
@@ -321,8 +397,10 @@ static int finish(CURL *c, CURLcode rc, membuf *m, http_resp *out, const char *u
         long os_err = 0;
         curl_easy_getinfo(c, CURLINFO_OS_ERRNO, &os_err);
         const char *detail = errbuf && errbuf[0] ? errbuf : out->error;
-        if (os_err) LOGW("HTTP %s: %s (errno %ld: %s)", url, detail, os_err, strerror((int)os_err));
-        else LOGW("HTTP %s: %s", url, detail);
+        /* curl's words for a reply we had no room for don't say so. */
+        const char *oom = out_of_memory(rc, m != NULL) && rc != CURLE_OUT_OF_MEMORY ? " (out of memory)" : "";
+        if (os_err) LOGW("HTTP %s: %s%s (errno %ld: %s)", url, detail, oom, os_err, strerror((int)os_err));
+        else LOGW("HTTP %s: %s%s", url, detail, oom);
         free(m ? m->buf : NULL);
         return -1;
     }

src/http.h

@@ -55,6 +55,10 @@ int http_down(const char *url);
 typedef void (*http_back_fn)(const char *origin);
 void http_on_back(http_back_fn fn);
 
+/* 1 if a request failed for want of memory in the last 30 s: smaller
+ * requests or the same one again right away won't do better. */
+int http_out_of_memory(void);
+
 /* URL-encode a query/path component into a malloc'd string. */
 char *http_escape(const char *s);
 

src/romm.c

@@ -114,8 +114,10 @@ int romm_list(const char *path, int offset, int limit, romm_list_cb cb, void *ct
         if (status) *status = st;
         /* A page too big to take in (games with a lot of metadata, where memory
          * is tight): the same games again, fewer at a time. Not when RomM
-         * didn't answer at all: fewer games won't change that. */
-        if ((st == 0 || (st == 200 && !page)) && want > 1 && !(st == 0 && romm_no_answer(why, sizeof why))) {
+         * didn't answer at all, or memory is so short that a few games at a
+         * time don't fit either: fewer games won't change that. */
+        if ((st == 0 || (st == 200 && !page)) && want > 1 && !(st == 0 && romm_no_answer(why, sizeof why)) &&
+            !(want <= 4 && http_out_of_memory())) {
             cJSON_Delete(page);
             page_size = want / 2;
             LOGI("%s: trying %d games at a time", path, page_size);

src/sync.c

@@ -2491,7 +2491,8 @@ static void run_sync(const char *reason, int only_rom, sync_result *res) {
     scan_ctx c;
     set_phase("scanning local files");
     if (scan_prepare(&c) != 0) {
-        snprintf(res->error, sizeof res->error, "cannot reach RomM server");
+        snprintf(res->error, sizeof res->error, "%s",
+                 c.server_status == 0 && http_out_of_memory() ? "out of memory" : "cannot reach RomM server");
         res->unreachable = c.server_status == 0;
         scan_free(&c);
         return;

tools/package-app.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Builds RommPS from scratch and packages it as build/release/PPSA76677.zip,
+# the file ProsperoStore installs.
+#
+#   tools/package-app.sh v1.3.1
+#
+# Used by the release workflow (.github/workflows/app.yml) and by
+# tools/release-app.sh. It expects build/ps5/romm-sync.elf to be the payload
+# of that same release, since RommPS carries it and starts it, and the title
+# art (pic0/pic1.dds, from platform/ps5/app/tools/title-art.sh) in place.
+# The PS5 stack is PS5_STACK, or ../ps5-stack (tools/ps5-app-build.sh).
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+APP="$ROOT/platform/ps5/app/rommps"
+TAG="${1:?usage: tools/package-app.sh vX.Y.Z}"
+die() { echo "package-app: $*" >&2; exit 1; }
+
+# The catalog tells consoles about an update from contentVersion, so it must
+# match the tag: v1.1.0 is 01.001.000.
+[[ "$TAG" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]] || die "$TAG isn't a vX.Y.Z tag"
+want=$(printf "%02d.%03d.%03d" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")
+have=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["contentVersion"])' "$APP/ps5/sce_sys/param.json")
+[[ "$have" == "$want" ]] || die "contentVersion is $have, $TAG needs $want (ps5/sce_sys/param.json)"
+grep -q "kMinPayload = \"${TAG#v}\"" "$APP/examples/rommps/kit/rommps_app.hpp" ||
+    die "kMinPayload in rommps_app.hpp isn't ${TAG#v}"
+for art in pic0 pic1; do
+    [[ -s "$APP/ps5/sce_sys/$art.dds" ]] || die "$art.dds is missing: run platform/ps5/app/tools/title-art.sh"
+done
+[[ -s "$ROOT/build/ps5/romm-sync.elf" ]] || die "build/ps5/romm-sync.elf is missing"
+
+# From scratch: a dev build left behind gives ninja nothing to do, and its
+# notices still say "uncommitted changes", so the check below refuses it.
+rm -rf "$APP/build/ps5" "$APP/dist/PPSA76677"
+"$ROOT/tools/ps5-app-build.sh" bash -c 'cd /repo/platform/ps5/app/rommps && bash ps5/tools/build.sh rommps'
+# The notices name the commit of every part; one built with uncommitted changes
+# wouldn't be the source they point to.
+dirty=$(python3 -c 'import json,sys; print(" ".join(p["id"] for p in json.load(open(sys.argv[1])) if p["source"].get("dirty")))' \
+    "$APP/dist/PPSA76677/licenses/components.json")
+[[ -z "$dirty" ]] || die "built from uncommitted changes in: $dirty"
+cmp -s "$ROOT/build/ps5/romm-sync.elf" "$APP/dist/PPSA76677/assets/rommps/romm-sync.elf" ||
+    die "the romm-sync.elf in RommPS isn't build/ps5/romm-sync.elf"
+
+mkdir -p "$ROOT/build/release"
+rm -f "$ROOT/build/release/PPSA76677.zip"
+(cd "$APP/dist" && zip -qrX "$ROOT/build/release/PPSA76677.zip" PPSA76677 -x 'PPSA76677/screenshots/*')
+(cd "$ROOT" && shasum -a 256 build/release/PPSA76677.zip)

tools/ps5-stack.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Checks out the PS5 stack RommPS builds against, at the commits a release
+# uses, for the release workflow (or anyone without ../ps5-stack):
+#
+#   tools/ps5-stack.sh <dir>     PS5_Vulkan, PS5_Mesa and PS5_PayloadSDK in <dir>
+#
+# PS5_Vulkan is pinned here. Everything else is pinned where it's used, and
+# read from there so there's one place to bump: Mesa by PS5_Vulkan's
+# tools/build-radv.sh, the payload SDK by PS5_Vulkan's
+# tools/setup-native-dependencies.sh and by RommPS's ps5/tools/setup-sdk.sh.
+# Every script that uses them does `git archive <pin>`, so only those commits
+# are fetched, not the histories (Mesa's alone is a gigabyte).
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+STACK="${1:?usage: tools/ps5-stack.sh <dir>}"
+
+# 5b5e4fc: what 1.3.0 was built with. RADV comes out of this one's pin of PS5_Mesa.
+vulkan_revision=5b5e4fc2d80fdb67a4f61ba9e6a8a424026bb8a5
+
+# fetch <repo> <commit>...: a repository holding just those commits, the first checked out
+fetch() {
+    local repo="$1" dir="$STACK/$1"
+    shift
+    [[ -d $dir/.git ]] || git init -q "$dir"
+    git -C "$dir" remote get-url origin >/dev/null 2>&1 ||
+        git -C "$dir" remote add origin "https://github.com/mihawk-99/$repo.git"
+    git -C "$dir" fetch -q --depth 1 origin "$@"
+    git -C "$dir" -c advice.detachedHead=false checkout -q "$1"
+    echo "$repo at ${1:0:11}"
+}
+pin() { sed -n "s/^$2=//p" "$1"; }
+
+fetch PS5_Vulkan "$vulkan_revision"
+fetch PS5_Mesa "$(pin "$STACK/PS5_Vulkan/tools/build-radv.sh" mesa_revision)"
+fetch PS5_PayloadSDK "$(pin "$ROOT/platform/ps5/app/rommps/ps5/tools/setup-sdk.sh" sdk_revision)" \
+    "$(pin "$STACK/PS5_Vulkan/tools/setup-native-dependencies.sh" sdk_revision)"

tools/release-app.sh

@@ -1,56 +1,40 @@
 #!/usr/bin/env bash
-# Builds RommPS for a release and attaches PPSA76677.zip to its GitHub release.
+# Builds RommPS for a release here, and attaches PPSA76677.zip to its GitHub release.
 #
 #   tools/release-app.sh v1.1.0            build and package it, in build/release/
 #   tools/release-app.sh v1.1.0 --upload   and attach it to the release v1.1.0
 #
-# The release workflow can't build RommPS: it needs the PS5 stack beside this
-# repository (tools/ps5-app-build.sh). So it is built here, from the tagged
-# commit, once the workflow has published the payloads. The homebrew catalog picks the release up from the asset's name,
-# so it stays PPSA76677.zip, and an asset already published is never replaced.
+# The release workflow builds and attaches the zip itself (.github/workflows/app.yml);
+# a PS5 release isn't published without it. This is the fallback for a release
+# that went out without one anyway, which is how the store sat on 1.2.1 for a
+# day after 1.3.0. The homebrew catalog picks the release up from the asset's
+# name, so it stays PPSA76677.zip, and an asset already published is never replaced.
 set -euo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-APP="$ROOT/platform/ps5/app/rommps"
 TAG="${1:?usage: tools/release-app.sh vX.Y.Z [--upload]}"
 die() { echo "release-app: $*" >&2; exit 1; }
 
-# The catalog tells consoles about an update from contentVersion, so it must
-# match the tag: v1.1.0 is 01.001.000.
-[[ "$TAG" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]] || die "$TAG isn't a vX.Y.Z tag"
-want=$(printf "%02d.%03d.%03d" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")
-have=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["contentVersion"])' "$APP/ps5/sce_sys/param.json")
-[[ "$have" == "$want" ]] || die "contentVersion is $have, $TAG needs $want (ps5/sce_sys/param.json)"
-
 cd "$ROOT"
 git diff --quiet && git diff --cached --quiet || die "commit or stash your changes first"
 [[ "$(git rev-parse HEAD)" == "$(git rev-parse "$TAG^{commit}" 2>/dev/null)" ]] || die "check out $TAG first"
-for art in pic0 pic1; do # the home screen art, made by platform/ps5/app/tools/title-art.sh
-    [[ -s "$APP/ps5/sce_sys/$art.dds" ]] || die "$art.dds is missing: run platform/ps5/app/tools/title-art.sh"
-done
+if [[ "${2:-}" == --upload ]] && gh release view "$TAG" --json assets -q '.assets[].name' | grep -qx PPSA76677.zip; then
+    die "$TAG already has PPSA76677.zip; publish a new release instead of replacing it"
+fi
 
 # RommPS carries the payload and starts it when it isn't running: the
 # release's own romm-sync.elf, the same file the in-app updater installs.
-grep -q "kMinPayload = \"${TAG#v}\"" "$APP/examples/rommps/kit/rommps_app.hpp" ||
-    die "kMinPayload in rommps_app.hpp isn't ${TAG#v}"
 mkdir -p build/ps5
 gh release download "$TAG" -p romm-sync.elf -O build/ps5/romm-sync.elf --clobber
 (cd build/ps5 && gh release download "$TAG" -p SHA256SUMS -O - | grep " romm-sync.elf$" | shasum -a 256 -c -) ||
     die "build/ps5/romm-sync.elf isn't $TAG's"
 
-tools/ps5-app-build.sh bash -c 'cd /repo/platform/ps5/app/rommps && bash ps5/tools/build.sh rommps'
-# The notices name the commit of every part; one built with uncommitted changes
-# wouldn't be the source they point to.
-dirty=$(python3 -c 'import json,sys; print(" ".join(p["id"] for p in json.load(open(sys.argv[1])) if p["source"].get("dirty")))' \
-    "$APP/dist/PPSA76677/licenses/components.json")
-[[ -z "$dirty" ]] || die "built from uncommitted changes in: $dirty"
-
-mkdir -p build/release
-rm -f build/release/PPSA76677.zip
-(cd "$APP/dist" && zip -qrX "$ROOT/build/release/PPSA76677.zip" PPSA76677 -x 'PPSA76677/screenshots/*')
-shasum -a 256 build/release/PPSA76677.zip
+tools/package-app.sh "$TAG"
 
 if [[ "${2:-}" == --upload ]]; then
-    gh release view "$TAG" --json assets -q '.assets[].name' | grep -qx PPSA76677.zip &&
-        die "$TAG already has PPSA76677.zip; publish a new release instead of replacing it"
     gh release upload "$TAG" build/release/PPSA76677.zip
+    local_sum=$(shasum -a 256 build/release/PPSA76677.zip | cut -d' ' -f1)
+    remote_sum=$(gh release download "$TAG" -p PPSA76677.zip -O - | shasum -a 256 | cut -d' ' -f1)
+    [[ "$local_sum" == "$remote_sum" ]] || die "the PPSA76677.zip on $TAG isn't the one built here"
+    echo "$TAG has PPSA76677.zip ($local_sum). The catalog bot proposes it within 8 hours;"
+    echo "for sooner, open the update yourself (docs/release-checklist.md)."
 fi