Fixed recompStore qr code

Revised readme for clarity and simplification

.github/workflows/release.yml

@@ -3,6 +3,12 @@
 #   - merging a PR into main from a branch named with a version, e.g.
 #     release/v1.0.2 or v1.0.2-some-fix (the tag is created on the merge commit)
 #   - pushing a version tag manually: git tag v1.0.0 && git push origin v1.0.0
+#
+# Run by hand (Actions > Release > Run workflow, or
+# gh workflow run release.yml -f tag=v1.3.0 -f ref=release/v1.3.0), it instead
+# rebuilds only the PS4 payload from ref and swaps it into the existing tag's
+# release: a PS4 fix that doesn't make the PS5 download anything. Those already
+# on that version reinstall it by hand; the in-app update only offers newer ones.
 name: Release
 
 on:
@@ -11,6 +17,14 @@ on:
   pull_request:
     types: [closed]
     branches: [main]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Existing release whose PS4 payload is replaced (v1.3.0)"
+        required: true
+      ref:
+        description: "Branch or commit the PS4 payload is built from"
+        required: true
 
 permissions:
   contents: write
@@ -26,6 +40,7 @@ jobs:
       - name: Resolve version
         id: version
         env:
+          GH_TOKEN: ${{ github.token }}
           HEAD_REF: ${{ github.event.pull_request.head.ref }}
         run: |
           if [ "$GITHUB_EVENT_NAME" = push ]; then
@@ -39,6 +54,12 @@ jobs:
             echo "Branch '$HEAD_REF' has no vX.Y.Z version; skipping release."
             exit 0
           fi
+          # A release branch merged again after its release went out (a fix
+          # shipped with the manual PS4 run below, say).
+          if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+            echo "Release $tag already exists; skipping."
+            exit 0
+          fi
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
           echo "sha=$sha" >> "$GITHUB_OUTPUT"
 
@@ -140,3 +161,61 @@ jobs:
           fi
           gh release create "$TAG" dist/* \
             --target "$SHA" --title "$TAG" --notes-file "$notes"
+
+  replace-ps4:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+      TAG: ${{ inputs.tag }}
+    steps:
+      - name: Check the release
+        run: |
+          if ! [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "::error::'$TAG' isn't a vX.Y.Z tag"
+            exit 1
+          fi
+          gh release view "$TAG" >/dev/null || { echo "::error::no release $TAG to update"; exit 1; }
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: platform/ps4/docker
+          tags: romm-sync-ps4sdk:v0.9-crt1352-oo0.5.4
+          load: true
+          cache-from: type=gha,scope=ps4
+          cache-to: type=gha,mode=max,scope=ps4,ignore-error=true
+
+      - name: Build the PS4 payload
+        run: tools/ps4-build.sh ps4-pkg VERSION="${TAG#v}"
+
+      - name: Sign
+        env:
+          RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
+        run: |
+          if [ -z "$RELEASE_SIGNING_KEY" ]; then
+            echo "::error::RELEASE_SIGNING_KEY secret is not set; in-app updates need a signed release."
+            exit 1
+          fi
+          make tools
+          mkdir dist
+          cp build/ps4/romm-sync-ps4.elf build/ps4/romm-sync-ps4.pkg dist/
+          build/host/release-sign manifest "$TAG" dist/romm-sync-ps4.elf > dist/manifest-ps4.json
+          build/host/release-sign sign dist/manifest-ps4.json > dist/manifest-ps4.sig
+          build/host/release-sign check dist/manifest-ps4.json dist/manifest-ps4.sig
+
+      # The PS5's files and their checksums stay as they are.
+      - name: Replace the PS4 files
+        run: |
+          cd dist
+          ps4="romm-sync-ps4.elf romm-sync-ps4.pkg manifest-ps4.json manifest-ps4.sig"
+          gh release download "$TAG" --pattern SHA256SUMS --output old-sums
+          grep -vE '  (romm-sync-ps4\.(elf|pkg)|manifest-ps4\.(json|sig))$' old-sums > SHA256SUMS || true
+          sha256sum $ps4 >> SHA256SUMS
+          gh release upload "$TAG" $ps4 SHA256SUMS --clobber

README.md

@@ -28,12 +28,74 @@ A [RomM](https://romm.app) companion client for jailbroken PS5 consoles (etaHEN
 - **Easter Eggs**
   - Check in the Settings, scroll the settings list, see if you find the X. This **hint** won't be here forever.
 
-## Supported Emulators and Games
-- RommPS has built-in profiles for all known emulators on the PS5, and will set the correct default game, save/save state, and bios directories for them. Since there is a huge influx of new emulators popping up, I will be doing my very best to keep up with them and add profiles accordingly.
-- You can also manually set the directories for each emulator/platform via the PS5 app or or the Web UI.
+## Requirements
 
-### Note
-- RommPS relies on your emulator **NOT** being sandboxed. Some emulators require a separate payload that must be launched to be able to access outside of the sandbox, so make sure you've followed their instructions to properly install. Sandboxed emulators will not be able to read games or save files.
+- A PS5 running etaHEN or onionHEN, or a PS4 running GoldHEN
+- RomM 5.2.0 or newer (may work on older versions)
+- Whichever other emulators you want
+  
+## Install
+
+### PS5
+
+| ProsperoStore | RecompStore |
+| :---: | :---: |
+| <img src="/assets/prospero-store-rommps.png" width="300" alt="ProsperoStore Install QR"> | <img src="/assets/recomp-store-rommps.png" width="300" alt="RecompStore Install QR"> |
+
+### Manual Install
+Download `PPSA76677.zip` from the releases page, unzip it and copy the `PPSA76677` folder to `/data/homebrew/` on the console. Launch RommPS and it will deploy the background service, and set itself to autostart with etaHen or onionHen. 
+
+If you'd rather run just the payload, without the app:
+1. Download the latest `romm-sync.elf` from the releases page.
+2. Send it to the console (or use whichever favorite tool you wish):
+   ```sh
+   nc -w 1 <ps5-ip> 9021 < romm-sync.elf
+   ```
+3. Set it up at `http://<ps5-ip>:8780` from another device.
+
+### PS4
+
+> **Experimental.** The PS4 is currently considered experimental
+Needs GoldHEN 2.4b18.5 or newer, whose payload menu runs ELF payloads.
+R
+1. Download the latest `romm-sync-ps4.pkg` from the releases page and install it with your package installer.
+2. Open **RommPS** from the home screen. It copies its payload, `romm-sync-ps4.elf`, to `/data/payloads/`.
+3. In GoldHEN Settings, open the payloader, pick `romm-sync-ps4.elf` and run it. Add it to the AutoRun queue there and it starts on every jailbreak.
+4. Open **RommPS** again: it shows the web UI. It's also at `http://<ps4-ip>:8780` from another device.
+
+The app only installs the payload and shows the web UI; the payload does the syncing, in the background, whether or not the app is open. Without the package, copy `romm-sync-ps4.elf` to `/data/payloads/` yourself (GoldHEN's FTP server, or `make install-ps4 PS4_HOST=<ps4-ip>`).
+
+**Note:** Don't send the payload to GoldHEN's network loader on port 9090: GoldHEN 2.4 crashes on ELFs sent there. Make sure the console's date and time are set (GoldHEN can keep them updated), or HTTPS won't work.
+
+### How it runs
+
+RommPS starts with your HEN, and setup turns this on for you. Both HENs start `payloads/romm-sync.elf` when `romm-sync.elf.auto_start` sits next to it, and RommPS puts both in every HEN's folder on the console (`/data/etaHEN/payloads` and `/data/OnionHEN/payloads`), so switching HENs keeps working. If a payload manager like pldmgr has its own copy, that copy is kept up to date too. If you build it yourself, `make install PS5_HOST=<ps5-ip>` copies it over FTP (add `PS5_FTP=ftp://<ps5-ip>:1337 PS5_HEN_DIR=/data/OnionHEN` for onionHEN's FTP plugin).
+
+## Updates
+
+- On the PS5, just update RommPS: from ProsperoStore (or other homebrew storefronts eventually), or by copying the new zip's folder over the old one. The background service comes with it, and RommPS swaps it in the next time you open it, once nothing is syncing or downloading. An older version never replaces a newer one. 
+- On the PS4, the payload updates itself from Settings.
+
+## Setup Wizard
+
+The setup wizard walks you through five steps:
+
+1. Connecting to your server.
+2. Approving the console in RomM, by scanning a QR code or entering a code.
+3. Choosing which emulators to sync.
+4. Checking what the first sync will do.
+5. Choosing when to sync in the background.
+
+**Back up your saves before the first sync.** Copy your emulator save folders to a computer or USB drive. RommPS keeps a local copy before replacing any file, but your own backup is the safest option.
+
+## Emulators
+RommPS has built-in profiles for most known emulators on the PS5, and will set the correct default game, save/save state, and bios directories for them. Since there is a huge influx of new emulators popping up, I will be doing my very best to keep up with them and add profiles accordingly.  
+
+Note: RommPS relies on your emulator **NOT** being sandboxed. Some emulators require a separate payload that must be launched to be able to access outside of the sandbox, so make sure you've followed their instructions to properly install. Sandboxed emulators will not be able to read games or save files.
+
+> The emulators listed below are those I've tested and have created profiles for to auto-detect them. You can also manually set the directories for each emulator/platform via the PS5 app or or the Web UI; you can use whichever emulators you wish.
+
+### PS5
 
 | Emulator | Systems | Game Downloads | Saves | Comments |
 | --- | --- | --- | --- | --- |
@@ -47,138 +109,27 @@ A [RomM](https://romm.app) companion client for jailbroken PS5 consoles (etaHEN
 | [ProsperoEden](https://github.com/blackbearreloaded/ProsperoEden) | Switch | Manual Support | Researching | - |
 | [PS5X360](https://github.com/BrinooTk/PS5X360) | Xbox 360 | Manual Support | Researching | - |
 
-To give some meaning to the table above. **Full support** means RommPS can detect and properly set game/save directories for that emulator. **Manual Support** means you can use it to sync games and/or saves, but you'll need to point it to the directories yourself. This typically means that full support is coming shortly. Broken or N/A means there was a problem keeping us from sync games and saves, or the emulator just fall out doesn't work or crashes. **Researching** means I am seeing what is possible when it comes to that sync type for the given emulator or game system.
-
-### Important: Save States
-Some users have recently reported issues with being able to pull down save states from their servers. They don't see them on the destination save folder for that emulator, or if they do, they don't work. Save states typically cannot be used across different emulators (cores), and even within the same core, the definition of a save state may different with file version, and version of the app. 
-
-**Simply put**:
-- Emulator core must be the same.
-- Different schema versions of save states are sometimes incompatible with each other.
-
-For the 1.2.0 release I will be including some fixes as well as some conversion logic to try and convert save state formats for max compatibility whenever possible. This is also why we have an upload only or upload and download option for save states. Maybe you don't care about syncing, but want to back them up, in case something happens. That's a good use case. I want to be blunt with you: Save states are kind of useless. Yes, they're great for recovering your progress after you feel asleep and forgot to save, but you truly should be saving progress with the in-game saves, not save status.
-
-### PS4 (experimental) ###
+### PS4
 
 | Emulator | Systems |
 | --- | --- |
 | RetroArch | The RetroArch systems above, wherever the PS4 build has the core |
 | pSNES, pNES, pGEN, pGBA, pFBN | SNES; NES; Genesis, Master System, Game Gear, Sega CD, SG-1000; GBA, GBC, GB; arcade |
 
-Only RetroArch has been tested on the PS4 so far.
-
 ### Save states
-Save states only load in the emulator core that made them, sometimes only in the same version of it. RommPS backs every state up to RomM, and with **Save states: Back up and restore** in Settings it brings one down only to an emulator that can load it:
-- **RetroArch:** states from the same core, from RomM's web player or another RetroArch.
-- **PSXS5:** PCSX-ReARMed or Beetle states, whichever core PSXS5 is set to. They're converted to and from the web player's format.
-- **SwanStationPS5, PS5N64:** only their own states.
-
-A state from elsewhere never replaces one of yours: it goes into the next free slot.
+Some users have recently reported issues with being able to pull down save states from their servers. They don't see them on the destination save folder for that emulator, or if they do, they don't work. Save states typically cannot be used across different emulators (cores), and even within the same core, the definition of a save state may different with the file version, and version of the app. In my opinion, save states should be for quick resume or to save you when you forgot to save your game, so they truly don't need to be synced across devices.
 
 ## PS4/PS5 Games
 
 If you are interested in downloading PS4 and PS5 games from your RomM server, RomM has direct FPKGi support, simply add it as a library. This sync client does NOT handle PS4/PS5 save syncing at this time, but is being researched for potential future inclusion.
 
-## Requirements
-
-- A PS5 running etaHEN or onionHEN, or a PS4 running GoldHEN
-- RomM 5.2.0 or newer (5.3.0 if you want to do shared memory card syncing instead of per-game)
-- RetroArch installed
-- Whichever other emulators you want
-
 ## Web UI
-- RommPS exposes a separate Web UI at http://your-ps5-ip:8780 which allows you to access most of the features present in the web app, but can make things like sync profiled editing, and adding multiple downloads of games easier from your computer instead.
 
-## Install
-
-### Install via ProsperoStore
-
-![ProsperoStoreQRCode](assets/prospero-store-rommps.png)
-
-Install **RommPS** from [ProsperoStore](https://homebrew.page) and open it. It'll set up the background daemon, and take you through a setup wizard. That's it: RommPS carries the RomM Sync payload and starts it for you. From then on it starts with your HEN, so after a jailbreak or waking the console it's ready to go.
-
-Not using the store? Download `PPSA76677.zip` from the releases page, unzip it and copy the `PPSA76677` folder to `/data/homebrew/` on the console.
-
-If you'd rather run just the payload, without the app:
-
-1. Download the latest `romm-sync.elf` from the releases page.
-2. Send it to the console (or use whichever favorite tool you wish):
-   ```sh
-   nc -w 1 <ps5-ip> 9021 < romm-sync.elf
-   ```
-3. Set it up at `http://<ps5-ip>:8780` from another device.
-
-On the PS5 the payload doesn't update itself: new versions come with RommPS. Without the app, download the new `romm-sync.elf` yourself.
-
-RomM Sync starts with your HEN, and setup turns this on for you. Both HENs start `payloads/romm-sync.elf` when `romm-sync.elf.auto_start` sits next to it, and RomM Sync puts both in every HEN's folder on the console (`/data/etaHEN/payloads` and `/data/OnionHEN/payloads`), so switching HENs keeps working. If a payload manager like pldmgr has its own copy, that copy is kept up to date too. If you build it yourself, `make install PS5_HOST=<ps5-ip>` copies it over FTP (add `PS5_FTP=ftp://<ps5-ip>:1337 PS5_HEN_DIR=/data/OnionHEN` for onionHEN's FTP plugin).
-
-### PS4
-
-> **Experimental.** The PS4 build has only been tested with RetroArch so far. Standalone emulators may work, but haven't been confirmed. I spent all my focus on the PS5 build, so I didn't get to test as much, so consider it unstable and there may be gremlins for now.
-
-Needs GoldHEN 2.4b18.5 or newer, whose payload menu runs ELF payloads.
-
-1. Download the latest `romm-sync-ps4.pkg` from the releases page and install it with your package installer.
-2. Open **RomM Sync** from the home screen. It copies its payload, `romm-sync-ps4.elf`, to `/data/payloads/`.
-3. In GoldHEN Settings, open the payloader, pick `romm-sync-ps4.elf` and run it. Add it to the AutoRun queue there and it starts on every jailbreak.
-4. Open **RomM Sync** again: it shows the web UI. It's also at `http://<ps4-ip>:8780` from another device.
-
-The app only installs the payload and shows the web UI; the payload does the syncing, in the background, whether or not the app is open. Without the package, copy `romm-sync-ps4.elf` to `/data/payloads/` yourself (GoldHEN's FTP server, or `make install-ps4 PS4_HOST=<ps4-ip>`).
-
-Don't send the payload to GoldHEN's network loader on port 9090: GoldHEN 2.4 crashes on ELFs sent there. Make sure the console's date and time are set (GoldHEN can keep them updated), or HTTPS won't work. RetroArch is found in `/data/retroarch`, with its cores in `/data/self/retroarch/cores`, as its `retroarch.cfg` says.
-
-## Updates
-
-On the PS5, just update RommPS: from ProsperoStore (or other homebrew storefronts eventually), or by copying the new zip's folder over the old one. The background service comes with it, and RommPS swaps it in the next time you open it, once nothing is syncing or downloading. An older version never replaces a newer one.
-
-On the PS4, the payload updates itself from Settings.
-
-## Setup
-
-The setup walks you through five steps:
-
-1. Connecting to your server.
-2. Approving the console in RomM, by scanning a QR code or entering a code.
-3. Choosing which emulators to sync.
-4. Checking what the first sync will do.
-5. Choosing when to sync in the background.
-
-**Back up your saves before the first sync.** Copy your emulator save folders to a computer or USB drive. RomM Sync keeps a local copy before replacing any file, but your own backup is the safest option.
-
-## PS2 Memory Cards
-
-LRPS2 keeps every game on one shared memory card by default. RomM stores saves per game, so pick one of these during setup:
-- **One card per game.** Works with any RomM version. In RetroArch, open the LRPS2 core options and turn off _Shared Memory Cards_. After that, each game's card syncs like any other save. Saves already on the shared card stay there.
-- **Back up the shared card.** Needs RomM 5.3.0 or newer. The whole card is uploaded when it changes, and you can restore it from Settings in the web UI. It isn't merged with other devices.
-
-PS5SX2 used 2 shared memory cards in slots 1 and 2. While it does support per-game memory cards, they need to be specified in a per-game ini file. In 1.2.0, I plan on adding the ability to directly modify those .ini files in PS5SX2 to write the proper per-game memory card path, effectively making RommPS handle per-game memory cards for it gracefully. Until then, I suggest using whichever emulator fits your preferred method (PS5SX2 for shared mem cards or use LRPS2 in retroarch for per-game support). If you have both installed, RommPS will sync game and saves, but they will be separate and different saves (PS5SX2 will download your shared memory cards, LRPS2 will grab the memory card per game). I know...it's confusing, but this is just a side-effect of the boom in emulators being released.
-
-## Other "Memory Card" Consoles
-PSP, GameCube and Wii saves are synced as one zip per game, the same format other RomM clients use. Depending on the client, they may or may not support unzipping the saves (I don't truly know), but this client does!
-
-My opinion? One card per game is better since 8MB memory cards fill up quick.
+RommPS exposes a separate Web UI at http://your-ps5-ip:8780 where you can access all the same features present in the native app.
 
 ## Emulator Profiles
 
-I've included direct support for most of the emulators that are out right now, except for the ones that won't even launch... you can sync games, states, and saves anywhere you want, but the presets should do it for most people.
-
-Settings in the web UI has a JSON editor for emulator profiles, and a table showing which folders are used for each platform. A profile lists an emulator's folders and the RomM platforms it plays:
-
-```json
-{
-  "id": "retroarch",
-  "name": "RetroArch",
-  "root": "/data/homebrew/PPSA99169",
-  "retroarch_cfg": "{root}/config/retroarch.cfg",
-  "rom_dir": "{root}/content/{platform}",
-  "save_dir": "{root}/savefiles",
-  "state_dir": "{root}/savestates",
-  "bios_dir": "{root}/system",
-  "save_exts": ".srm,.sav",
-  "platforms": [{ "romm": ["snes", "sfam"], "dir": "snes", "emulator": "snes9x", "core_name": "Snes9x" }]
-}
-```
-
+I've included direct support for most of the emulators that are out right now, except for the ones that won't even launch... you can sync games, states, and saves anywhere you want, but the presets should do it for most people. You can also configure the paths for all platforms manually. The web UI has a JSON editor for emulator profiles.
 
 ## Building
 

src/covers.c

@@ -22,8 +22,8 @@
 
 /* An asset RomM doesn't have is remembered for a day (a .missing file), so it
  * isn't asked for again on every page. The folder is pruned, oldest first,
- * past its limit: room for a whole library's covers on the PS5 (66 KB each on
- * average), less where storage is tight. The full pass stops short of it. */
+ * past its limit: on the consoles, room for some 30,000 covers (66 KB each on
+ * average). The full pass stops short of it. */
 #define MISSING_SEC (24 * 60 * 60)
 #define FULL_EVERY_SEC (24 * 60 * 60)
 #define WARM_SLOTS 256
@@ -37,17 +37,26 @@ static long long g_cached_bytes = -1; /* -1 until counted */
 static char g_warm[WARM_SLOTS][512];
 static int g_nwarm;
 
-/* The full pass: every game's cover path, worked through by the workers. */
-static char **g_all;
-static int g_nall, g_next, g_done;
+/* The full pass: every game's cover path, worked through by the workers. It
+ * stays on disk, a line each in cache/cover-pass (twenty thousand paths are
+ * more than the PS4 can spare), with its size and counts in cover-pass.json
+ * and how far it got in cover-pass.pos, so a restart carries on from there. */
+static FILE *g_pass;
+static int g_nall, g_next, g_done, g_total; /* g_total: the pass's size, after it's gone */
+/* The library was being looked at: the pass waits, its pages' covers don't. */
+static time_t g_browse_at;
+#define BROWSE_PAUSE_SEC 15
 static int g_refresh; /* a full pass was asked for */
 static char g_state[16] = "idle";
 
 static int is_ps5(void) { return strcmp(plat_info()->name, "ps5") == 0; }
 
-static long long cache_limit(void) { return is_ps5() ? 1024LL * 1024 * 1024 : 300LL * 1024 * 1024; }
+static long long cache_limit(void) { return strcmp(plat_info()->name, "host") ? 2048LL * 1024 * 1024 : 300LL * 1024 * 1024; }
 
-static int workers(void) { return is_ps5() ? 4 : 2; }
+static int is_ps4(void) { return strcmp(plat_info()->name, "ps4") == 0; }
+
+/* The PS4 payload shares a small memory pool with GoldHEN: one cover at a time. */
+static int workers(void) { return is_ps5() ? 4 : is_ps4() ? 1 : 2; }
 
 static void cache_path(const char *p, const char *suffix, char *out, size_t n) {
     uint64_t h = 0xcbf29ce484222325ULL; /* FNV-1a of the asset path and query */
@@ -92,58 +101,74 @@ static char *svg_with_size(const char *svg, size_t len, size_t *out_len) {
     return outp;
 }
 
-typedef struct {
-    time_t mtime;
-    long long size;
-    char name[200];
-} cached_file;
+/* Pruning goes by age without a list of the files, so it takes the same
+ * little memory for a hundred covers as for twenty thousand: one pass sorts
+ * their sizes into AGE_BINS slices of the folder's age span, the next deletes
+ * the oldest slices, whole, until the rest fit. */
+#define AGE_BINS 256
 
-static int by_age(const void *a, const void *b) {
-    time_t x = ((const cached_file *)a)->mtime, y = ((const cached_file *)b)->mtime;
-    return x < y ? -1 : x > y;
-}
-
-/* The folder's size; and, past the limit, the oldest files deleted until it's
- * back under 95% of it. One pass over the folder, on the planner's thread. */
-static long long cache_scan(int prune) {
-    char dir[PATH_MAX_LEN], path[PATH_MAX_LEN];
-    snprintf(dir, sizeof dir, "%s/cache/assets", plat_data_dir());
+/* Calls fn on each cached file (the folder's own, not dotfiles). */
+static void each_cached(const char *dir, void (*fn)(const char *path, const struct stat *st, void *ud), void *ud) {
+    char path[PATH_MAX_LEN];
     DIR *d = opendir(dir);
-    if (!d) return 0;
-    cached_file *files = NULL;
-    int n = 0, cap = 0;
-    long long total = 0;
+    if (!d) return;
     struct dirent *de;
     while ((de = readdir(d))) {
         struct stat st;
-        if (de->d_name[0] == '.' || strlen(de->d_name) >= sizeof files->name) continue;
+        if (de->d_name[0] == '.') continue;
         path_join(path, sizeof path, dir, de->d_name);
-        if (stat(path, &st) != 0) continue;
-        total += st.st_size;
-        if (!prune) continue;
-        if (n == cap) {
-            cached_file *grown = realloc(files, (size_t)(cap = cap ? cap * 2 : 1024) * sizeof *files);
-            if (!grown) break;
-            files = grown;
-        }
-        files[n].mtime = st.st_mtime;
-        files[n].size = st.st_size;
-        str_copy(files[n].name, sizeof files[n].name, de->d_name);
-        n++;
+        if (stat(path, &st) == 0 && S_ISREG(st.st_mode)) fn(path, &st, ud);
     }
     closedir(d);
-    if (prune && total > cache_limit()) {
-        qsort(files, (size_t)n, sizeof *files, by_age);
-        for (int i = 0; i < n && total > cache_limit() / 20 * 19; i++) {
-            path_join(path, sizeof path, dir, files[i].name);
-            if (unlink(path) == 0) total -= files[i].size;
-        }
+}
+
+struct age_scan {
+    long long total, bins[AGE_BINS];
+    time_t oldest, newest;
+    int pass, cutoff; /* cutoff: the newest slice deleted */
+};
+
+static int age_bin(const struct age_scan *a, time_t t) {
+    const long long span = (long long)(a->newest - a->oldest) + 1;
+    const long long b = (long long)(t - a->oldest) * AGE_BINS / span;
+    return b < 0 ? 0 : b >= AGE_BINS ? AGE_BINS - 1 : (int)b;
+}
+
+static void age_visit(const char *path, const struct stat *st, void *ud) {
+    struct age_scan *a = ud;
+    if (a->pass == 0) {
+        a->total += st->st_size;
+        if (!a->oldest || st->st_mtime < a->oldest) a->oldest = st->st_mtime;
+        if (st->st_mtime > a->newest) a->newest = st->st_mtime;
+    } else if (a->pass == 1) {
+        a->bins[age_bin(a, st->st_mtime)] += st->st_size;
+    } else if (age_bin(a, st->st_mtime) <= a->cutoff && unlink(path) == 0) {
+        a->total -= st->st_size;
+    }
+}
+
+/* The folder's size; and, past the limit, the oldest files deleted until it's
+ * back under 95% of it. On the planner's thread. */
+static long long cache_scan(int prune) {
+    char dir[PATH_MAX_LEN];
+    snprintf(dir, sizeof dir, "%s/cache/assets", plat_data_dir());
+    struct age_scan a;
+    memset(&a, 0, sizeof a);
+    each_cached(dir, age_visit, &a);
+    if (prune && a.total > cache_limit()) {
+        a.pass = 1;
+        each_cached(dir, age_visit, &a);
+        long long over = a.total - cache_limit() / 20 * 19, freed = 0;
+        int last = 0;
+        while (last < AGE_BINS - 1 && (freed += a.bins[last]) < over) last++;
+        a.cutoff = last;
+        a.pass = 2;
+        each_cached(dir, age_visit, &a);
     }
-    free(files);
     pthread_mutex_lock(&g_lock);
-    g_cached_bytes = total;
+    g_cached_bytes = a.total;
     pthread_mutex_unlock(&g_lock);
-    return total;
+    return a.total;
 }
 
 static void cache_store(const char *path, const void *body, size_t len) {
@@ -229,6 +254,9 @@ int covers_fetch(const char *p, char **out, size_t *out_len, const char **type)
 }
 
 void covers_warm(const char *p) {
+    pthread_mutex_lock(&g_lock);
+    g_browse_at = time(NULL);
+    pthread_mutex_unlock(&g_lock);
     if (!valid_path(p) || have(p)) return;
     pthread_mutex_lock(&g_lock);
     for (int i = 0; i < g_nwarm; i++)
@@ -262,47 +290,18 @@ static int enabled(void) {
 /* The console is busy with something that matters more. */
 static int busy(void) { return sync_game_running() || sync_is_running() || library_busy(); }
 
-struct cover_list { char **list; int n, cap; };
-
-/* The covers seen first go first: the games on this console, then each
- * platform's tile, then the rest in RomM's order. */
-static void prioritize(struct cover_list *c, const char *heads) {
-    char **out = malloc(sizeof *out * (size_t)(c->n ? c->n : 1));
-    char *taken = calloc((size_t)(c->n ? c->n : 1), 1);
-    if (!out || !taken) { free(out); free(taken); return; }
-    int n = 0;
-    state_lock();
-    const cJSON *e;
-    cJSON_ArrayForEach(e, state_section("roms")) {
-        const char *cover = jget_str(e, "cover", "");
-        if (!cover[0] || !(file_exists(e->string) || dir_exists(e->string))) continue;
-        for (int i = 0; i < c->n; i++)
-            if (!taken[i] && !strcmp(c->list[i], cover)) { taken[i] = 1; out[n++] = c->list[i]; break; }
-    }
-    state_unlock();
-    for (int pass = 0; pass < 2; pass++)
-        for (int i = 0; i < c->n; i++)
-            if (!taken[i] && (pass || heads[i])) { taken[i] = 1; out[n++] = c->list[i]; }
-    free(taken);
-    free(c->list);
-    c->list = out;
+static void pass_path(char *out, size_t n, const char *suffix) {
+    snprintf(out, n, "%s/cache/cover-pass%s", plat_data_dir(), suffix);
 }
 
-static int collect_cover(const cJSON *it, const cJSON *page, int first, void *ctx) {
-    (void)page, (void)first;
-    if (!it) return 0;
-    const char *p = jget_str(it, "path_cover_small", "");
-    if (!valid_path(p)) return 0;
-    struct cover_list *c = ctx;
-    if (c->n >= 100000) return 0;
-    if (c->n == c->cap) {
-        c->cap = c->cap ? c->cap * 2 : 512;
-        char **grown = realloc(c->list, sizeof *grown * (size_t)c->cap);
-        if (!grown) return -1;
-        c->list = grown;
-    }
-    c->list[c->n++] = strdup(p);
-    return 0;
+/* Where the pass got to: every so many covers, not each one. Call with g_lock held. */
+static void save_pos(void) {
+    char path[PATH_MAX_LEN], pos[64];
+    long at = g_pass ? ftell(g_pass) : -1;
+    if (at < 0) return;
+    pass_path(path, sizeof path, ".pos");
+    int len = snprintf(pos, sizeof pos, "%d %ld\n", g_next, at);
+    write_file_atomic(path, pos, (size_t)len);
 }
 
 /* Each platform's game count when a pass last listed it: the daily pass
@@ -311,13 +310,140 @@ static int collect_cover(const cJSON *it, const cJSON *page, int first, void *ct
  * kept once its covers are done, so an unfinished one is listed again. */
 static cJSON *g_pass_counts;
 
-/* Every game's cover path, platform by platform; all: even the platforms
- * whose game count didn't change. 0 when RomM gave the whole list; -1 if it
- * couldn't be reached part way (what came is still used). */
+static void close_pass(void) {
+    if (g_pass) fclose(g_pass);
+    g_pass = NULL;
+    g_nall = g_next = 0;
+}
+
+/* A pass that's over (done, or stopped by a full cache) goes, files and all.
+ * Call with g_lock held. */
+static void drop_pass(void) {
+    char path[PATH_MAX_LEN];
+    close_pass();
+    static const char *const files[] = {"", ".json", ".pos"};
+    for (size_t i = 0; i < sizeof files / sizeof files[0]; i++) {
+        pass_path(path, sizeof path, files[i]);
+        unlink(path);
+    }
+}
+
+/* Opens the pass on disk where it was left (the top, for a new one). 0 if
+ * there's one to carry on with. Call with g_lock held. */
+static int open_pass(void) {
+    char path[PATH_MAX_LEN];
+    close_pass();
+    pass_path(path, sizeof path, ".json");
+    char *js = read_file(path, NULL);
+    cJSON *head = js ? cJSON_Parse(js) : NULL;
+    free(js);
+    pass_path(path, sizeof path, "");
+    g_pass = head ? fopen(path, "r") : NULL;
+    if (!g_pass) {
+        cJSON_Delete(head);
+        return -1;
+    }
+    setvbuf(g_pass, NULL, _IOFBF, 4096);
+    int next = 0;
+    long at = 0;
+    pass_path(path, sizeof path, ".pos");
+    char *pos = read_file(path, NULL);
+    if (!pos || sscanf(pos, "%d %ld", &next, &at) != 2 || next < 0 || at < 0 || fseek(g_pass, at, SEEK_SET) != 0) {
+        next = 0;
+        fseek(g_pass, 0, SEEK_SET);
+    }
+    free(pos);
+    g_nall = g_total = (int)jget_num(head, "total", 0);
+    g_next = g_done = next;
+    cJSON_Delete(g_pass_counts);
+    g_pass_counts = cJSON_DetachItemFromObjectCaseSensitive(head, "counts");
+    cJSON_Delete(head);
+    return 0;
+}
+
+/* While RomM lists the games: each platform's first covers (its tile) in one
+ * file, the rest in another. */
+struct pass_build { FILE *heads, *rest; int n, in_plat; };
+
+static int collect_cover(const cJSON *it, const cJSON *page, int first, void *ctx) {
+    (void)page, (void)first;
+    if (!it) return 0;
+    const char *p = jget_str(it, "path_cover_small", "");
+    if (!valid_path(p) || strlen(p) >= 500 || strchr(p, '\n')) return 0;
+    struct pass_build *b = ctx;
+    if (fprintf(b->in_plat++ < 3 ? b->heads : b->rest, "%s\n", p) < 0) return -1;
+    b->n++;
+    return 0;
+}
+
+static int by_str(const void *a, const void *b) { return strcmp(*(char *const *)a, *(char *const *)b); }
+
+/* Goes through one file's lines, leaving out those in mine (sorted): with
+ * out, copies the rest onto it, and says how many; without, marks in seen
+ * the ones of mine it came across. */
+static int merge_lines(FILE *out, const char *path, char **mine, int nmine, char *seen) {
+    char line[512];
+    FILE *in = fopen(path, "r");
+    int n = 0;
+    while (in && fgets(line, sizeof line, in)) {
+        line[strcspn(line, "\r\n")] = 0;
+        const char *key = line;
+        char **hit = nmine ? bsearch(&key, mine, (size_t)nmine, sizeof *mine, by_str) : NULL;
+        if (hit && seen) seen[hit - mine] = 1;
+        if (hit || !out) continue;
+        fprintf(out, "%s\n", line);
+        n++;
+    }
+    if (in) fclose(in);
+    return n;
+}
+
+/* The covers of the games on this console, sorted, without repeats. */
+static char **my_covers(int *n) {
+    char **mine = NULL;
+    int cap = 0;
+    *n = 0;
+    state_lock();
+    const cJSON *e;
+    cJSON_ArrayForEach(e, state_section("roms")) {
+        const char *cover = jget_str(e, "cover", "");
+        if (!valid_path(cover) || !(file_exists(e->string) || dir_exists(e->string))) continue;
+        if (*n == cap) {
+            char **grown = realloc(mine, sizeof *mine * (size_t)(cap = cap ? cap * 2 : 64));
+            if (!grown) break;
+            mine = grown;
+        }
+        if ((mine[*n] = strdup(cover))) (*n)++;
+    }
+    state_unlock();
+    if (*n) qsort(mine, (size_t)*n, sizeof *mine, by_str);
+    int kept = 0;
+    for (int i = 0; i < *n; i++) {
+        if (kept && !strcmp(mine[kept - 1], mine[i])) free(mine[i]);
+        else mine[kept++] = mine[i];
+    }
+    *n = kept;
+    return mine;
+}
+
+/* Every game's cover path, platform by platform, into a new pass on disk;
+ * all: even the platforms whose game count didn't change. The covers seen
+ * first go first: the games on this console, then each platform's tile, then
+ * the rest in RomM's order. 0 when RomM gave the whole list; -1 if it couldn't
+ * be reached part way (what came is still used). */
 static int list_all(int all) {
-    struct cover_list c = {0};
-    static char heads[100000]; /* a platform's first covers (its tile) */
-    memset(heads, 0, sizeof heads);
+    char heads[PATH_MAX_LEN], rest[PATH_MAX_LEN], fresh[PATH_MAX_LEN], path[PATH_MAX_LEN];
+    pass_path(heads, sizeof heads, ".heads");
+    pass_path(rest, sizeof rest, ".rest");
+    pass_path(fresh, sizeof fresh, ".new");
+    mkdir_parent(heads);
+    struct pass_build b = {fopen(heads, "w"), fopen(rest, "w"), 0, 0};
+    if (!b.heads || !b.rest) {
+        if (b.heads) fclose(b.heads);
+        if (b.rest) fclose(b.rest);
+        LOGW("covers: can't write %s", heads);
+        return -1;
+    }
     long st = 0;
     cJSON *plats = romm_call("GET", "/api/platforms", NULL, &st), *p;
     int ok = st == 200 && cJSON_IsArray(plats), same = 0;
@@ -328,38 +454,76 @@ static int list_all(int all) {
     cJSON_ArrayForEach(p, plats) {
         const double n = jget_num(p, "rom_count", 0);
         if (n <= 0) continue;
-        char id[24], path[160];
+        char id[24], list[160];
         snprintf(id, sizeof id, "%d", (int)jget_num(p, "id", 0));
         if (!all && jget_num(had, id, -1) == n) {
             cJSON_AddNumberToObject(counts, id, n);
             same++;
             continue;
         }
-        snprintf(path, sizeof path, "/api/roms?platform_ids=%s&order_by=name&order_dir=asc&with_char_index=false" ROMS_LEAN, id);
-        int before = c.n;
-        if (romm_list(path, 0, 100000, collect_cover, &c, &st) < 0) ok = 0;
+        snprintf(list, sizeof list, "/api/roms?platform_ids=%s&order_by=name&order_dir=asc&with_char_index=false" ROMS_LEAN, id);
+        b.in_plat = 0;
+        if (romm_list(list, 0, 100000, collect_cover, &b, &st) < 0) ok = 0;
         else cJSON_AddNumberToObject(counts, id, n);
-        for (int k = before; k < c.n && k < before + 3; k++) heads[k] = 1;
     }
     cJSON_Delete(had);
     cJSON_Delete(plats);
-    prioritize(&c, heads);
+    const int written = fclose(b.heads) == 0 && fclose(b.rest) == 0;
+
+    FILE *out = written ? fopen(fresh, "w") : NULL;
+    int total = 0, nmine = 0;
+    if (out) {
+        /* This console's games first, those RomM listed (a cover there since
+         * replaced has a new path), then the tiles and the rest without them. */
+        char **mine = my_covers(&nmine);
+        char *seen = calloc((size_t)nmine + 1, 1);
+        merge_lines(NULL, heads, mine, seen ? nmine : 0, seen);
+        merge_lines(NULL, rest, mine, seen ? nmine : 0, seen);
+        for (int i = 0; seen && i < nmine; i++)
+            if (seen[i]) {
+                fprintf(out, "%s\n", mine[i]);
+                total++;
+            }
+        total += merge_lines(out, heads, mine, seen ? nmine : 0, NULL);
+        total += merge_lines(out, rest, mine, seen ? nmine : 0, NULL);
+        for (int i = 0; i < nmine; i++) free(mine[i]);
+        free(mine);
+        free(seen);
+        if (fclose(out) != 0) total = -1;
+    }
+    unlink(heads);
+    unlink(rest);
+    cJSON *head = cJSON_CreateObject();
+    cJSON_AddNumberToObject(head, "total", total);
+    cJSON_AddItemToObject(head, "counts", counts);
+    char *js = cJSON_PrintUnformatted(head);
+    cJSON_Delete(head);
+
     pthread_mutex_lock(&g_lock);
-    for (int i = 0; i < g_nall; i++) free(g_all[i]);
-    free(g_all);
-    g_all = c.list;
-    g_nall = c.n;
-    g_next = g_done = 0;
-    cJSON_Delete(g_pass_counts);
-    g_pass_counts = counts;
+    drop_pass();
+    int opened = -1;
+    pass_path(path, sizeof path, "");
+    if (total >= 0 && js && rename(fresh, path) == 0) {
+        char meta[PATH_MAX_LEN];
+        pass_path(meta, sizeof meta, ".json");
+        if (write_file_atomic(meta, js, strlen(js)) == 0) opened = open_pass();
+    }
+    if (opened != 0) drop_pass();
     pthread_mutex_unlock(&g_lock);
-    LOGI("covers: %d games to check%s%s", c.n, ok ? "" : " (RomM couldn't be reached for all of them)",
+    unlink(fresh);
+    free(js);
+    if (opened != 0) {
+        LOGW("covers: couldn't keep the list of %d games to check", b.n);
+        return -1;
+    }
+    LOGI("covers: %d games to check%s%s", total, ok ? "" : " (RomM couldn't be reached for all of them)",
          same ? ", the platforms with as many games as last time skipped" : "");
     return ok ? 0 : -1;
 }
 
 /* The next cover to fetch: the pages being looked at first, then the full
- * pass. "" when there's nothing to do. */
+ * pass, which waits while the library is being looked at. "" when there's
+ * nothing to do. */
 static void next_job(char *out, size_t n, int *from_pass) {
     out[0] = 0;
     *from_pass = 0;
@@ -368,9 +532,16 @@ static void next_job(char *out, size_t n, int *from_pass) {
         str_copy(out, n, g_warm[0]);
         g_nwarm--;
         memmove(g_warm[0], g_warm[1], sizeof g_warm[0] * (size_t)g_nwarm);
-    } else if (g_next < g_nall) {
-        str_copy(out, n, g_all[g_next++]);
-        *from_pass = 1;
+    } else if (g_pass && g_next < g_nall && time(NULL) - g_browse_at >= BROWSE_PAUSE_SEC) {
+        char line[512];
+        if (fgets(line, sizeof line, g_pass)) {
+            line[strcspn(line, "\r\n")] = 0;
+            str_copy(out, n, line);
+            *from_pass = 1;
+            if (++g_next % 25 == 0) save_pos();
+        } else {
+            g_next = g_nall; /* shorter than it said: it's done anyway */
+        }
     }
     pthread_mutex_unlock(&g_lock);
 }
@@ -390,7 +561,10 @@ static void *worker(void *arg) {
         next_job(p, sizeof p, &from_pass);
         if (!p[0]) {
             pthread_mutex_lock(&g_lock);
-            if (g_next >= g_nall && g_nall) str_copy(g_state, sizeof g_state, "done");
+            if (g_next >= g_nall && g_nall) {
+                str_copy(g_state, sizeof g_state, "done");
+                drop_pass();
+            }
             struct timespec until;
             clock_gettime(CLOCK_REALTIME, &until);
             until.tv_sec += 30;
@@ -406,7 +580,7 @@ static void *worker(void *arg) {
             /* Stop short of the limit, so pruning never undoes the pass. */
             if (used >= 0 && used > cache_limit() * 9 / 10) {
                 pthread_mutex_lock(&g_lock);
-                g_next = g_nall;
+                drop_pass();
                 str_copy(g_state, sizeof g_state, "full");
                 pthread_mutex_unlock(&g_lock);
                 LOGW("covers: the cache is nearly full, the rest wait");
@@ -426,6 +600,14 @@ static void *worker(void *arg) {
             char why[160];
             if (covers_fetch(p, &body, &len, &type) == 0) {
                 free(body);
+            } else if (http_out_of_memory()) {
+                /* No room for it now: it goes first again once some is free,
+                 * rather than every cover after it failing the same way. */
+                covers_warm(p);
+                pthread_mutex_lock(&g_lock);
+                str_copy(g_state, sizeof g_state, "waiting");
+                pthread_mutex_unlock(&g_lock);
+                sleep(10);
             } else if (romm_no_answer(why, sizeof why)) {
                 /* Not skipped: it goes first again, once RomM answers (this
                  * worker's next try finds out). */
@@ -449,6 +631,9 @@ static void *worker(void *arg) {
 static void *planner(void *arg) {
     (void)arg;
     cache_scan(0);
+    pthread_mutex_lock(&g_lock);
+    if (open_pass() == 0 && g_next < g_nall) LOGI("covers: carrying on, %d of %d games left to check", g_nall - g_next, g_nall);
+    pthread_mutex_unlock(&g_lock);
     /* How often it looks for work; ROMM_SYNC_COVERS_TICK shortens it in tests. */
     const char *env = getenv("ROMM_SYNC_COVERS_TICK");
     unsigned tick = env && atoi(env) > 0 ? (unsigned)atoi(env) : 20;
@@ -511,7 +696,7 @@ cJSON *covers_status(void) {
     pthread_mutex_lock(&g_lock);
     cJSON_AddStringToObject(j, "state", g_state);
     cJSON_AddNumberToObject(j, "done", g_done);
-    cJSON_AddNumberToObject(j, "total", g_nall);
+    cJSON_AddNumberToObject(j, "total", g_total);
     cJSON_AddNumberToObject(j, "cached_mb", g_cached_bytes > 0 ? (double)(g_cached_bytes / (1024 * 1024)) : 0);
     pthread_mutex_unlock(&g_lock);
     cJSON_AddNumberToObject(j, "limit_mb", (double)(cache_limit() / (1024 * 1024)));

src/http.c

@@ -17,6 +17,9 @@
 #include "util.h"
 #ifdef ROMM_EMBED_CA
 #include "cacert_pem.h" /* generated from the SDK's Mozilla CA bundle */
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
 #endif
 
 static char g_ca_bundle[PATH_MAX_LEN];
@@ -24,6 +27,12 @@ static char g_ca_bundle[PATH_MAX_LEN];
  * which have no system CA store. */
 static char *g_ca_blob;
 static size_t g_ca_blob_len;
+#ifdef ROMM_EMBED_CA
+/* The same roots, parsed once and handed to every connection. Given as a
+ * blob, libcurl parses all ~140 of them again for each new connection and
+ * caches nothing, which the PS4's memory pool can't afford. */
+static X509_STORE *g_ca_store;
+#endif
 static int g_tls_verify = 1;
 static char g_user_agent[64];
 
@@ -42,6 +51,7 @@ static pthread_mutex_t g_share_locks[CURL_LOCK_DATA_LAST];
  * the freshest) or a new one, and gives it back; POOL_MAX are kept. */
 #define POOL_MAX 8
 static CURL *g_pool[POOL_MAX];
+static int pool_max(void) { return strcmp(plat_info()->name, "ps4") ? POOL_MAX : 3; }
 static int g_npool;
 static pthread_mutex_t g_pool_lock = PTHREAD_MUTEX_INITIALIZER;
 
@@ -57,7 +67,7 @@ static CURL *handle_take(void) {
 
 static void handle_give(CURL *c) {
     pthread_mutex_lock(&g_pool_lock);
-    const int kept = g_npool < POOL_MAX;
+    const int kept = g_npool < pool_max();
     if (kept) g_pool[g_npool++] = c;
     pthread_mutex_unlock(&g_pool_lock);
     if (!kept) curl_easy_cleanup(c);
@@ -69,6 +79,23 @@ static double mono_sec(void) {
     return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;
 }
 
+/* When a request last failed for want of memory: a reply we couldn't keep in
+ * memory (a download's write error is the disk's), or curl or OpenSSL coming
+ * up empty. */
+static double g_oom_at = -1;
+static pthread_mutex_t g_oom_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static int out_of_memory(CURLcode rc, int in_memory) {
+    return rc == CURLE_OUT_OF_MEMORY || rc == CURLE_SSL_CACERT_BADFILE || (in_memory && rc == CURLE_WRITE_ERROR);
+}
+
+int http_out_of_memory(void) {
+    pthread_mutex_lock(&g_oom_lock);
+    const int recent = g_oom_at >= 0 && mono_sec() - g_oom_at < 30;
+    pthread_mutex_unlock(&g_oom_lock);
+    return recent;
+}
+
 /* A server that doesn't answer at all (no connection) isn't asked again by
  * every thread, each waiting out the connect timeout: requests to it fail at
  * once, and one at a time checks whether it's back, after 5 s, then twice as
@@ -166,6 +193,37 @@ static void share_unlock(CURL *c, curl_lock_data data, void *ud) {
     if (data < CURL_LOCK_DATA_LAST) pthread_mutex_unlock(&g_share_locks[data]);
 }
 
+#ifdef ROMM_EMBED_CA
+/* What libcurl does with a CA blob (load_cacert_from_memory and the flags it
+ * sets), done once. NULL if not one certificate in it could be read. */
+static X509_STORE *ca_store(const char *pem, size_t len) {
+    X509_STORE *st = X509_STORE_new();
+    BIO *bio = st && len <= 0x7fffffff ? BIO_new_mem_buf(pem, (int)len) : NULL;
+    STACK_OF(X509_INFO) *infos = bio ? PEM_X509_INFO_read_bio(bio, NULL, NULL, NULL) : NULL;
+    int n = 0;
+    for (int i = 0; infos && i < sk_X509_INFO_num(infos); i++) {
+        X509_INFO *inf = sk_X509_INFO_value(infos, i);
+        if (inf->x509 && X509_STORE_add_cert(st, inf->x509)) n++;
+    }
+    sk_X509_INFO_pop_free(infos, X509_INFO_free);
+    BIO_free(bio);
+    if (!n) {
+        X509_STORE_free(st);
+        return NULL;
+    }
+    X509_STORE_set_flags(st, X509_V_FLAG_TRUSTED_FIRST | X509_V_FLAG_PARTIAL_CHAIN);
+    return st;
+}
+
+/* Every connection's SSL_CTX gets the one store; libcurl's own, empty, goes. */
+static CURLcode use_ca_store(CURL *c, void *ssl_ctx, void *ud) {
+    (void)c;
+    if (!X509_STORE_up_ref(ud)) return CURLE_OUT_OF_MEMORY;
+    SSL_CTX_set_cert_store(ssl_ctx, ud);
+    return CURLE_OK;
+}
+#endif
+
 void http_global_init(const char *ca_bundle, int tls_verify) {
     curl_global_init(CURL_GLOBAL_ALL);
     for (int i = 0; i < CURL_LOCK_DATA_LAST; i++) pthread_mutex_init(&g_share_locks[i], NULL);
@@ -193,6 +251,11 @@ void http_global_init(const char *ca_bundle, int tls_verify) {
         }
     }
     free(user);
+    const char *tls = curl_version_info(CURLVERSION_NOW)->ssl_version;
+    if (g_ca_blob && tls && !strncmp(tls, "OpenSSL", 7) && (g_ca_store = ca_store(g_ca_blob, g_ca_blob_len))) {
+        free(g_ca_blob);
+        g_ca_blob = NULL;
+    }
     LOGI("libcurl %s, CA bundle: built-in%s%s", curl_version_info(CURLVERSION_NOW)->version,
          g_ca_bundle[0] ? " + " : "", g_ca_bundle);
 #else
@@ -283,6 +346,14 @@ static CURL *easy_new(const char *url, struct curl_slist **hdrs, const char *aut
 #ifdef ROMM_NETLOG
     curl_easy_setopt(c, CURLOPT_DEBUGFUNCTION, curl_trace);
     curl_easy_setopt(c, CURLOPT_VERBOSE, 1L);
+#endif
+#ifdef ROMM_EMBED_CA
+    if (g_ca_store) {
+        curl_easy_setopt(c, CURLOPT_CAINFO, NULL);
+        curl_easy_setopt(c, CURLOPT_CAPATH, NULL);
+        curl_easy_setopt(c, CURLOPT_SSL_CTX_FUNCTION, use_ca_store);
+        curl_easy_setopt(c, CURLOPT_SSL_CTX_DATA, g_ca_store);
+    } else
 #endif
     if (g_ca_blob) {
         struct curl_blob blob = {g_ca_blob, g_ca_blob_len, CURL_BLOB_NOCOPY};
@@ -310,6 +381,11 @@ static int finish(CURL *c, CURLcode rc, membuf *m, http_resp *out, const char *u
     down_note(url, out->no_answer, probe);
     if (rc != CURLE_OK) {
         snprintf(out->error, sizeof out->error, "%s", curl_easy_strerror(rc));
+        if (out_of_memory(rc, m != NULL)) {
+            pthread_mutex_lock(&g_oom_lock);
+            g_oom_at = mono_sec();
+            pthread_mutex_unlock(&g_oom_lock);
+        }
         /* A download (CURLOPT_FAILONERROR) stopped by the server's answer:
          * say which, so a 404 isn't "HTTP response code said error". */
         if (rc == CURLE_HTTP_RETURNED_ERROR) {
@@ -321,8 +397,10 @@ static int finish(CURL *c, CURLcode rc, membuf *m, http_resp *out, const char *u
         long os_err = 0;
         curl_easy_getinfo(c, CURLINFO_OS_ERRNO, &os_err);
         const char *detail = errbuf && errbuf[0] ? errbuf : out->error;
-        if (os_err) LOGW("HTTP %s: %s (errno %ld: %s)", url, detail, os_err, strerror((int)os_err));
-        else LOGW("HTTP %s: %s", url, detail);
+        /* curl's words for a reply we had no room for don't say so. */
+        const char *oom = out_of_memory(rc, m != NULL) && rc != CURLE_OUT_OF_MEMORY ? " (out of memory)" : "";
+        if (os_err) LOGW("HTTP %s: %s%s (errno %ld: %s)", url, detail, oom, os_err, strerror((int)os_err));
+        else LOGW("HTTP %s: %s%s", url, detail, oom);
         free(m ? m->buf : NULL);
         return -1;
     }

src/http.h

@@ -55,6 +55,10 @@ int http_down(const char *url);
 typedef void (*http_back_fn)(const char *origin);
 void http_on_back(http_back_fn fn);
 
+/* 1 if a request failed for want of memory in the last 30 s: smaller
+ * requests or the same one again right away won't do better. */
+int http_out_of_memory(void);
+
 /* URL-encode a query/path component into a malloc'd string. */
 char *http_escape(const char *s);
 

src/romm.c

@@ -114,8 +114,10 @@ int romm_list(const char *path, int offset, int limit, romm_list_cb cb, void *ct
         if (status) *status = st;
         /* A page too big to take in (games with a lot of metadata, where memory
          * is tight): the same games again, fewer at a time. Not when RomM
-         * didn't answer at all: fewer games won't change that. */
-        if ((st == 0 || (st == 200 && !page)) && want > 1 && !(st == 0 && romm_no_answer(why, sizeof why))) {
+         * didn't answer at all, or memory is so short that a few games at a
+         * time don't fit either: fewer games won't change that. */
+        if ((st == 0 || (st == 200 && !page)) && want > 1 && !(st == 0 && romm_no_answer(why, sizeof why)) &&
+            !(want <= 4 && http_out_of_memory())) {
             cJSON_Delete(page);
             page_size = want / 2;
             LOGI("%s: trying %d games at a time", path, page_size);

src/sync.c

@@ -2491,7 +2491,8 @@ static void run_sync(const char *reason, int only_rom, sync_result *res) {
     scan_ctx c;
     set_phase("scanning local files");
     if (scan_prepare(&c) != 0) {
-        snprintf(res->error, sizeof res->error, "cannot reach RomM server");
+        snprintf(res->error, sizeof res->error, "%s",
+                 c.server_status == 0 && http_out_of_memory() ? "out of memory" : "cannot reach RomM server");
         res->unreachable = c.server_status == 0;
         scan_free(&c);
         return;