test(connection): align pill expectations with connection redesign

Store: repositories behind a login, and a GitHub token in source settings

.github/workflows/checks.yml

@@ -5,6 +5,11 @@ on:
     branches: [main]
   pull_request:
 
+# A newer push to the same pull request supersedes a run still in progress.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   checks:
     runs-on: ubuntu-latest
@@ -17,7 +22,9 @@ jobs:
         with:
           node-version: "24"
       - name: Install zsh
-        run: sudo apt-get update -qq && sudo apt-get install -y -qq zsh
+        # The runner image's third-party apt sources (packages.microsoft.com) sometimes
+        # fail to refresh; Ubuntu's own lists still update, and the install fails if not.
+        run: (sudo apt-get update -qq || true) && sudo apt-get install -y -qq zsh
       - name: Script syntax
         run: |
           sh -n ui/local-bin/ssh
@@ -84,6 +91,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install zsh
-        run: sudo apt-get update -qq && sudo apt-get install -y -qq zsh
+        # The runner image's third-party apt sources (packages.microsoft.com) sometimes
+        # fail to refresh; Ubuntu's own lists still update, and the install fails if not.
+        run: (sudo apt-get update -qq || true) && sudo apt-get install -y -qq zsh
       - name: End-to-end tests
         run: scripts/e2e.sh

.github/workflows/release.yml

@@ -2,14 +2,22 @@ name: release
 
 # Pushing a v* tag builds Frame Control for macOS, Windows and Linux and attaches
 # the installers to that tag's GitHub release (created as a draft if missing).
-# Pull requests that touch the app build the same installers as artifacts.
+# Pull requests that change how the installers are built (the Electron app, the
+# macOS helper it compiles, or this workflow) build the same installers as
+# artifacts. ui/, scripts/, frame/ and apk-catalog/ are copied in as they are and
+# are tested by the checks workflow; run this one by hand to package them early.
 on:
   push:
     tags: ["v*"]
   pull_request:
-    paths: ["app/**", "ui/**", "scripts/**", "frame/**", "apk-catalog/**", ".github/workflows/release.yml"]
+    paths: ["app/**", "mac/frame-mac-view/**", ".github/workflows/release.yml"]
   workflow_dispatch:
 
+# A newer push to the same pull request supersedes a build still in progress.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: write
 
@@ -94,7 +102,9 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh release upload "${GITHUB_REF_NAME}" ${{ matrix.files }} --clobber
+      # Tagged builds are attached to the release above; only other builds need artifacts.
       - uses: actions/upload-artifact@v4
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
         with:
           name: frame-control-${{ matrix.os }}
           path: |

.gitignore

@@ -6,8 +6,6 @@ compat-db/.env.lakebed.server
 compat-db/.lakebed/
 tests/smoke/results/
 mac/bin/
-site/.wrangler/
-site/.dev.vars
 
 # Downloaded at build time (frame/kdeconnect/fetch.py, app/build/fetch-deps.js)
 frame/kdeconnect/packages/

README.md

@@ -118,11 +118,8 @@ a computer. Build it from [`ios/`](ios) in Xcode; see [docs/iphone.md](docs/ipho
 The app brings its own Python and `adb`; SSH is built into macOS and Windows.
 From 0.4 it updates itself: when a new version is published, a banner offers
 **Update and restart**. It sends anonymous usage statistics, which you can turn
-off. Sharing compatibility results and error details is opt-in, and so is the
-mailing list it offers once on first launch (just an email address, **No
-thanks** to skip). Signups go to the website’s private Cloudflare D1 list;
-see [site/README.md](site/README.md#mailing-list) for backend setup and
-[docs/privacy.md](docs/privacy.md) for what is sent.
+off. Sharing compatibility results and error details is opt-in. See
+[docs/privacy.md](docs/privacy.md).
 Google doesn't publish `adb` for arm64 Linux, so that build uses your
 distribution's. If you already have `adb`, the app uses yours.
 
@@ -192,7 +189,7 @@ entry to `~/.ssh/config` and keys at `~/.ssh/id_ed25519_frame` and
 
 This is a first public test, so reports are really useful, especially from
 Windows and Linux. The quickest way is **Report a problem** in the app (the
-warning-sign button at the top, or **Help → Report a Problem…**). It adds
+speech-bubble button at the top, or **Help → Report a Problem…**). It adds
 diagnostics with personal details removed, shows you exactly what's included,
 and sends it privately to the maintainer; nothing is published. Without the app,
 use the [feedback form](https://frame-control.pages.dev/feedback/). Please include:
@@ -227,7 +224,7 @@ Frame's software fits together, all checked against a real headset and labelled
 | [Frame Control for iPhone](docs/iphone.md) | The iPhone and iPad app, how it runs the server on the Frame, pairing |
 | [Recovery and OS images](docs/recovery-and-images.md) | Where to download the Frame's OS, what's inside, testing without the headset |
 | [AI agents and assistant](docs/agents.md) | Key-free MCP tools, human approvals, and an opt-in assistant panel |
-| [Testing](docs/testing.md) | Unit tests, end-to-end tests against a fake Frame in Docker, and the headset smoke test |
+| [Testing](docs/testing.md) | Unit tests, end-to-end tests against a fake Frame in Docker, the headset smoke test and a Windows test VM |
 | [Open questions](docs/open-questions.md) | What's still unchecked |
 
 <details>

app/main.js

@@ -602,9 +602,6 @@ function buildMenu() {
           if (win && url && linkPage === win.webContents) win.webContents.send("report:open");
           else shell.openExternal("https://frame-control.pages.dev/feedback/");  // the page isn't up
         } },
-        { label: "Join the Mailing List…", click: () => {
-          if (win && url && linkPage === win.webContents) win.webContents.send("mailing:open");
-        } },
         { label: "Release Notes", click: () => shell.openExternal(updater.RELEASES) },
         { label: "Project on GitHub", click: () => shell.openExternal("https://github.com/saphid/steam-frame") },
       ],

app/preload.js

@@ -40,11 +40,6 @@ contextBridge.exposeInMainWorld("frameApp", {
     ipcRenderer.removeAllListeners("report:open");
     ipcRenderer.on("report:open", () => cb());
   },
-  // Help → Join the Mailing List… opens the page's mailing list setting (ui/frame_contact.py).
-  onJoinMailingList: (cb) => {
-    ipcRenderer.removeAllListeners("mailing:open");
-    ipcRenderer.on("mailing:open", () => cb());
-  },
   // Sign in with Meta (app/meta-signin.js): Meta's login page in its own window. The
   // answer is the server's (connected, display name), never the token.
   meta: {

docs/apk-repos.md

@@ -58,8 +58,61 @@ form shows the pinned one ("Trusted on first use: …") so you can compare it.
 Re-adding an existing URL preserves its pin; changing it requires deliberately
 removing and re-adding it.
 
+### Repositories behind a login
+
+Some self-hosted repositories sit behind HTTP basic auth. As in F-Droid's own
+client, the login can be part of the address
+(`https://user:pass@example.org/fdroid/repo`), or given separately: in the
+store's **Add a source** form under "This repository needs a login", in the
+server API (`username` and `password` beside `url`), or on the command line,
+where the password is read from standard input so it stays out of shell history
+and `ps`:
+
+```sh
+printf '%s\n' "$PASSWORD" | python3 ui/apk_sources/fdroid.py add 'https://example.org/fdroid/repo' --username me --password-stdin
+```
+
+How the login is kept and used:
+
+- **Kept out of the address.** The `user:pass@` part is removed before anything
+  is saved, so the saved repository URL, its id, the source list, job results
+  and the page never contain it. The store only says "Uses a login".
+- **Stored separately.** Logins live in
+  `frame_host.data_dir('source-credentials.json')`
+  (`~/Library/Application Support/Frame Control/source-credentials.json` on
+  macOS), written with mode 0600 and keyed by source id. They are stored in
+  plain text there, protected by file permissions only (on Windows,
+  the folder's normal ACLs). Removing the repository deletes its login, and
+  the file goes when the last login is removed. Each change holds a lock
+  file (`source-credentials.lock`), so the app and the command line can't undo
+  each other's saves or removals.
+- **HTTPS only.** An `http://` address with a login is refused before any request.
+- **Inside the repository only.** The `Authorization` header is decided
+  separately for every request, redirects included: index, `entry.jar`,
+  APKs and the store's artwork proxy. It is sent only when the request's
+  scheme, host and port match the repository's and its path is inside the
+  repository's directory by whole path segments. A redirect to another host
+  or port, or to another path on the same server, gets no login, and that
+  request's own `Authorization` header is removed. Paths with `.` or `..`
+  segments (plain or percent-encoded), encoded `/` or `\`, double encoding
+  or `;` get no login, since a server could normalise them to a path
+  outside the repository. Index file names like that are refused outright.
+- **Never echoed.** Logins are not written to logs, error messages, API
+  responses or the cache. The CLI prints only the saved source.
+- **Checked before saving.** As with any repository, the whole index must
+  download and verify before anything is saved. HTTP 401 (or 403 when a login
+  was sent) stops the add with "check the username and password" and saves
+  nothing: no repository, no login, no rollback state. A 403 with a login is
+  not treated as rate limiting.
+- **Same verification.** The login changes only who may download. Signature,
+  fingerprint pin or TOFU, index hash and rollback checks apply exactly as for
+  any other repository.
+
+Re-adding the same address with a new login replaces the saved one. Re-adding
+it without a login keeps the saved one.
+
 The API for the search/server integration is in `ui/apk_sources/fdroid.py`:
-`add_repo(url, fingerprint=None, name=None)`, `remove_repo(source_id)`,
+`add_repo(url, fingerprint=None, name=None, username=None, password=None)`, `remove_repo(source_id)`,
 `set_enabled(source_id, enabled)`, and `user_repos()`. The module also exposes
 `sources`, `search`, `details`, and `download` from the shared source contract.
 This change supplies the CLI and API; the integrated source-management UI is

docs/apk-sources.md

@@ -45,10 +45,23 @@ search. Results outside the curated list stay page-only, even if a repository
 claims an open-source license. This prevents an arbitrary tagged mirror from
 becoming a trusted downloader. Extend the curated JSON after provenance review.
 
-Set optional `FRAME_GITHUB_TOKEN` in the process environment for a higher API
-quota. Tokens are sent only to `api.github.com`, never written to the cache,
-never sent to asset hosts, and removed on redirects. The adapter does not
-read `gh` credentials automatically. Metadata is cached for one hour under
+For a higher API quota, add a GitHub access token in the store's source
+settings (the GitHub row's token field) or through the server API
+(`POST /api/sources` with `action: "set-token"` or `"clear-token"` and
+`source: "github"`). `FRAME_GITHUB_TOKEN` in the process environment still
+works and wins over a saved token. A saved token is kept with repository
+logins in `source-credentials.json` (mode 0600, see
+[APK repositories](apk-repos.md#repositories-behind-a-login)). The source list
+reports only whether a token is set, where it came from and whether GitHub
+rejected it, never its value. Tokens are sent as `Authorization: Bearer` only to
+`api.github.com`, never written to the cache, never sent to asset hosts (public
+release downloads don't need them), and removed on redirects. If GitHub answers
+401 to the token, the request is retried once without it, as Obtainium does.
+The rejection is logged without the token and shown in source settings. The
+token is not tried again until it is saved again or the app restarts. Rate-limit
+backoff is kept per login (anonymous, or each token), so saving a token after
+anonymous requests were limited lets the next request go out with it. The
+adapter does not read `gh` credentials automatically. Metadata is cached for one hour under
 `frame_host.cache_dir('apk-sources', 'publisher')`. A cold details request
 fetches at most ten releases. Rate-limit errors are surfaced without retry
 loops. Asset IDs and release tags are not Android version codes: metadata

docs/devices.md

@@ -3,8 +3,9 @@
 Frame Control can manage more than one Steam Frame, and each headset can be
 reached at more than one address: a LAN IP at home, another at the office, its
 mDNS name (`frame.local`), its Tailscale IP or MagicDNS name. The **Devices**
-tab (key 5) lists them, and the connection pill in the header shows what the
-app is doing to reach the one in use, step by step, as it happens.
+tab (key 5) lists them. The connection pill in the header says whether the one in
+use is connected and how (Tailscale, or the network's name); click it to add,
+edit or reorder that headset's addresses, or to see each step of connecting.
 
 The code is in three modules, all stdlib-only Python on your computer:
 
@@ -57,7 +58,7 @@ and ranks them:
 5. addresses that only ever worked on other networks;
 6. Tailscale addresses while Tailscale is off.
 
-Your order on the Devices tab breaks ties. The best-ranked address that answers
+Your order (on the Devices tab, or in the pill's dialog) breaks ties. The best-ranked address that answers
 wins; one that answers first waits up to 0.35 s for a better-ranked one that is
 still trying. If SSH to the winner fails in a way another address could fix
 (a different device answered there, or the link dropped), the next one that
@@ -72,6 +73,16 @@ status --json`, including the Mac app's own CLI) with buttons to add their
 MagicDNS name or IP. **Find on this network** asks mDNS for SteamOS devkit
 services and checks `ALIAS.local` and `frame.local`.
 
+**The pill's dialog** lists the same addresses, with what each one answered, and
+can add, edit, reorder and remove them without leaving the page you're on. When
+the headset reports a LAN IP on the same network as this computer and that IP
+isn't saved, it offers to add it. The offer puts the address first in the list,
+so on that network it wins over the Tailscale name; away from home the Tailscale
+name still leads. A new or edited address is tested straight away. While
+connected, **Reconnect** applies your changes now rather than at the next
+connection: it tries the addresses again, ranked as above, and the best-ranked
+one that answers promptly wins. It doesn't pick a particular address.
+
 ## Networks
 
 A network is told apart by its default gateway: the router's IP address plus its

docs/frame-control.md

@@ -23,9 +23,9 @@ the catalogue, display settings, reports), **Tools** (sending files and text,
 Flatpaks, remote and power), **Devices** (your headsets and their addresses),
 and **Control** (the searchable Control Center, metrics, supported settings and
 headset guidance). Keys 1–6 switch between them. Files can be dropped anywhere in the window. A
-connection pill in the header always shows which headset, which network this
-computer is on, the address in use or being tried, and each step of connecting
-as it happens; click it for the whole timeline. When the Frame can't be
+connection pill in the header always shows which headset and whether it's
+connected, and how (Tailscale, or the network's name). Click it to add, edit or
+reorder the headset's addresses, or to see each step of connecting. When the Frame can't be
 reached, a banner says why in plain words, what was tried, and counts down to
 the next try, filling everything in once it answers. Flatpak and Android installs run in the background; the bottom bar
 counts them while they run.

docs/privacy.md

@@ -132,7 +132,7 @@ runs.
 
 ## Report a problem
 
-**Report a problem** is the warning-sign button in the header, also in the
+**Report a problem** is the speech-bubble button in the header, also in the
 Privacy panel and under **Help → Report a Problem…**. It sends the report
 privately to Frame Control's PostHog project as a `problem_report` event, the
 same way as the analytics above, so only the maintainer can read it and
@@ -218,27 +218,21 @@ The maintainer reads reports on the Frame Control dashboard in PostHog, or
 with `python3 ui/frame_report.py inbox [days]`, which uses the same personal
 API key as `frame_compat_db.py sync`.
 
-## Mailing list and contact email (optional)
+## Contact email (optional)
 
 Frame Control never needs an email address. If you'd like to leave one, there
 are two separate choices, both off until you tick them:
 
 | Choice | What it's for |
 |---|---|
-| **Join the mailing list: new features and updates** | Occasional emails about new features and updates |
+| **Email me about Frame Control updates** | Occasional notices about new releases and updates |
 | **The maintainer may contact me with follow-up questions** | Questions about problem reports you send, mostly |
 
-The first time Frame Control opens, a bar at the top of the page asks
-"Want to hear about new features? Join the Frame Control mailing list." with
-an empty email field, **Join** and **No thanks**. It only asks about the
-mailing list; follow-up questions are a separate choice in **Privacy &
-updates**. If the first-run privacy notice is showing, it waits until that's
-closed, so the two never appear together. It doesn't need the Frame to be set
-up. **No thanks** hides it for good, and it isn't shown again even if you
-ignore it. **Mailing list and contact email** in **Privacy & updates** (also
-**Help → Join the Mailing List…** in the desktop app) is where you join later,
-or change or remove the address and either choice, at any time. Unticking the
-mailing list is unsubscribing.
+You're asked once, in a bar at the top of the page, after the Frame has
+connected for the first time, and never in the same visit as the first-run
+privacy notice. **No thanks** hides it for good, and it isn't
+shown again even if you ignore it. **Contact email** in **Privacy & updates**
+is where you add, change or remove the address and either choice at any time.
 
 **What's sent, and where.** The address and the two choices go privately to
 Frame Control's PostHog project, the same place as problem reports, as a
@@ -270,40 +264,10 @@ this computer and is sent when PostHog can be reached. The earlier event
 stays in PostHog until its data retention removes it; to have it deleted
 sooner, ask the maintainer (for example in a problem report).
 
-**The Cloudflare mailing list.** When you explicitly join or change an existing
-mailing-list choice, the app also sends that choice to
-`https://framecontrol.fun/api/mailing-list`, on the website's Cloudflare Pages
-project. This request contains only the email address, an explicit `updates`
-boolean, a random capability used to change this copy's subscription, and a
-revision number used to order retries. It contains no analytics id or machine
-details. Follow-up-only choices and skipping the offer send nothing to this
-endpoint. Existing saved choices aren't automatically copied to a new backend.
-
-Cloudflare D1 privately stores the current address and revision under a hash of
-the random capability, with a change timestamp. Changing the address replaces
-the previous one. Unsubscribing sets the address to null; only the capability
-hash, revision and timestamp remain, so a delayed older signup can't restore
-it. Duplicate addresses from separate app copies are deduplicated when exported;
-unsubscribing in the app withdraws this copy's subscription. The website reuses
-its short-lived salted IP hashes to limit requests, without storing raw IPs in
-the list. Cloudflare necessarily processes the connection's IP address.
-
-The mailing request has its own durable local queue in `contact/contact.json`.
-It retries in the background independently of PostHog, including after restarting.
-Saving, skipping and continuing to use the app never wait for either service.
-An offline removal clears the queued address immediately and leaves a withdrawal
-to send when the endpoint is available. Until it arrives, the server still has
-the previous choice. If Cloudflare storage hasn't been provisioned, the endpoint
-returns 503 and the same queue retains the choice safely.
-
-No newsletter sending service is configured and no confirmation or newsletter
-emails are sent by this feature. Cloudflare is the private signup backend;
-[site/README.md](../site/README.md#mailing-list) describes provisioning and a
-maintainer-only export of current opt-ins. The existing
-`python3 ui/frame_report.py contacts [updates|followup]` command still lists
-PostHog consent records; use the Cloudflare export for the mailing list. A future
-sending service must honor current withdrawals, provide an unsubscribe option
-in every email, and be named here before addresses are transferred to it.
+Nothing sends email yet: this only records who agreed to what. The
+maintainer lists the addresses with
+`python3 ui/frame_report.py contacts [updates|followup]`, which uses the same
+personal API key as `inbox`.
 
 ## Turning it all off
 

docs/streaming.md

@@ -21,6 +21,20 @@ The confidence labels are the same as in [ssh.md](ssh.md).
 | **ADB + scrcpy (Lepton only)** | A mirror of the Android container | **Guess** | `brew install scrcpy android-platform-tools`, then `adb connect frame.local:5555` while Lepton Development is running ([adb_lepton](https://partner.steamgames.com/doc/steamhardware/steamframe/adb_lepton)), then `scrcpy`. This only shows Android apps, not SteamOS. |
 | VNC server on the Frame (krfb / wayvnc) | A mirror of the Plasma desktop | **Inferred (SteamOS)** | Deck users run krfb in Desktop Mode ([one.vg](https://one.vg/blog/remote-control-your-steam-deck)). On the Frame, the in-headset desktop is a virtual screen, and krfb isn't known to be preinstalled. RDP and Steam Link cover this case, so it's not recommended. |
 
+**RDP from Windows, verified 2026-09-30:** Windows 11 25H2's Remote Desktop
+(`mstsc`) against BUILD_ID 20260925.6191901. xrdp picks TLS, not NLA, so
+Remote Desktop never asks for a user or password. It warns that the certificate
+(`www.xrdp.org`) can't be verified. After **Yes**, xrdp shows its own "Login to
+frame" box with the username blank. Any user but `steamos` gets "User does not
+exist, or could not be authenticated". Signing in as `steamos` with the
+Developer Mode password opens a Plasma (X11) desktop within about 6 seconds.
+It's a new session on display `:10`, separate from what the headset shows,
+and it uses about 1.3 GB of the Frame's memory. Closing Remote Desktop leaves
+it running, and the next sign-in reconnects to it. To end it, find its
+session with `loginctl list-sessions` over SSH (its leader is `xrdp-sesexec`)
+and run `loginctl terminate-session <id>`; the headset's own session keeps
+running.
+
 **Recommendation for A:** start with Steam Link for macOS, because Valve
 documents it. Use Windows App (RDP) when you want a proper Linux desktop on the
 Mac with keyboard, mouse, and clipboard.

docs/testing.md

@@ -8,6 +8,7 @@ A fourth, a SteamOS VM, may come later ([issue #6](https://github.com/saphid/ste
 | Unit tests (`tests/*.py`) | `python3 -m unittest discover -s tests` | Nothing | Parsing, validation, request guards; SSH and HTTP are mocked |
 | Fake Frame (`tests/e2e`) | `scripts/e2e.sh` | Linux with Docker | The real server and scripts against a container that behaves like a Frame |
 | Headset smoke test | `scripts/frame-smoke.sh` | A Frame on the `frame` alias | Install, launch and remove on the real device, recorded with its BUILD_ID |
+| [Windows test VM](#windows-test-vm) | `scripts/windows-vm.sh` | A Linux machine with KVM and Docker | The Windows build on a real Windows desktop, against a real Frame when needed |
 
 ## Unit tests
 
@@ -186,6 +187,86 @@ refuses ids with a hyphen (`missing/invalid arguments`), which the fake had
 accepted. The fake now refuses them the same way, and Frame Control makes ids
 Steam accepts.
 
+## Windows test VM
+
+The unit tests run on Windows in CI, but the app itself doesn't. Anything that
+depends on the Windows desktop (Remote Desktop, the installer, the bundled
+Python, file dialogs) needs a real Windows machine. This is a Windows 11 VM in a
+[dockur/windows](https://github.com/dockur/windows) container on a Linux machine
+with KVM, driven from the Mac with `scripts/windows-vm.sh`.
+
+Setting it up, once, on the Linux machine:
+
+- **Windows comes from Microsoft.** The container downloads the Windows 11
+  image from Microsoft on first start. Windows runs unactivated, which is fine
+  for testing. Don't use activation workarounds or third-party Windows images.
+- **Publish its ports on loopback only.** Map `127.0.0.1:2222:22` (SSH),
+  `127.0.0.1:8006:8006` (the web console) and, if you need it,
+  `127.0.0.1:13389:3389`. The Mac reaches them through `ssh -J`. Use
+  `restart: "no"` and `stop_grace_period: 2m` so it only runs when someone is
+  testing, and a `docker stop` shuts Windows down cleanly.
+- **Give it SSH on first sign-in.** The container runs `/oem/install.bat`
+  once. Have it add the OpenSSH Server capability, start `sshd`, set PowerShell
+  as its default shell, and put a dedicated public key (for example
+  `~/.ssh/id_ed25519_winvm` on the Mac) in
+  `C:\ProgramData\ssh\administrators_authorized_keys`.
+- Give it 4 cores, 8 GB of memory and a 32 GB disk. That's enough for the app
+  and the tests.
+
+Using it, with `WINVM_HOST` set to the Linux machine's ssh alias:
+
+```sh
+scripts/windows-vm.sh up                        # start it and wait for SSH (1-2 minutes)
+scripts/windows-vm.sh put Frame-Control-Setup-x64.exe
+scripts/windows-vm.sh ps 'Start-Process "$env:USERPROFILE\Downloads\Frame-Control-Setup-x64.exe" /S -Wait'
+scripts/windows-vm.sh shot screen.png            # what's on its screen
+scripts/windows-vm.sh click 723 359              # screen pixels, as in the screenshot
+scripts/windows-vm.sh keys s t e a m o s ret     # QEMU key names
+scripts/windows-vm.sh down                       # shut Windows down
+```
+
+- **Clicks go through a scheduled task.** Commands over SSH run in a
+  session with no desktop, so `click` and `scroll` write the position to a
+  file, and a scheduled task running as the signed-in user replays it. The
+  VM's screen must be signed in; it is after `up`. Clicks and scrolls sent at
+  the same time run one after another. QEMU's own `mouse_move` is relative
+  and drifts, so the script doesn't use it.
+- **Screenshots may not show the pointer.** Check the result of a click (a
+  menu that opens, a button that changes) rather than the pointer's position.
+- **Windows' `ssh` waits for stdin.** The script closes it for every command.
+  Do the same if you run `ssh` in the VM by hand.
+- **Starting the app.** Run Frame Control in the signed-in session, not over
+  SSH. Use its Start-menu shortcut via `click`, or a scheduled task like the
+  one `click` uses.
+
+**Testing against a real Frame.** The VM reaches the headset on the LAN like
+any other computer. Run **Set Up Connection** in the VM's Frame Control once;
+that makes the VM's keys. So the VM doesn't keep access between test runs,
+afterwards take the lines it added out of the headset's
+`~/.ssh/authorized_keys`: the ones matching the VM's
+`~/.ssh/id_ed25519_frame.pub` and, if pairing made one,
+`~/.ssh/id_rsa_frame_devkit.pub`. Check that `ssh -o BatchMode=yes frame true`
+in the VM now fails. Then, around each run:
+
+```sh
+scripts/windows-vm.sh frame-key add       # let the VM's key into the headset
+# ... test ...
+scripts/windows-vm.sh frame-key remove    # and take it out again
+```
+
+`add` appends the VM's key tagged `windows-vm-test`, unless the headset
+already trusts that key through another line. `remove` deletes only the exact
+line `add` wrote, so it doesn't undo what Set Up Connection did, and it leaves
+the file alone if it can't rewrite it. Follow the shared-device
+procedure in [Headset smoke test](#headset-smoke-test) before installing or
+launching anything on the headset.
+
+**Verified 2026-09-30:** Windows 11 Pro 25H2 (build 26200) against a Frame on
+BUILD_ID 20260925.6191901. The script was used to install Frame Control 0.4.0,
+connect it to the Frame and follow Remote Desktop through to the Frame's
+desktop ([what it does](streaming.md#a-see-and-control-the-frame-from-the-mac)).
+`frame-key` left `authorized_keys` byte-for-byte as it was.
+
 ## Owned media player
 
 `tests/test_media.py` covers layout evidence and overrides, OU eye ordering,

docs/vr-utilities.md

@@ -26,7 +26,7 @@ it twice reuses the existing process.
 | Compositor CPU | OpenVR compositor render CPU milliseconds, not game CPU time. |
 | System CPU | `/proc/stat` busy-time delta across the sample, with guest time counted once and iowait treated as idle. |
 | GPU clock | `3d00000.gpu/cur_freq`, converted from Hz to MHz; frequency is not load. |
-| Hottest sensor / battery | Existing thermal-zone and battery sysfs reads from `frame_status.py`. |
+| Hottest sensor / battery | Existing thermal-zone and battery sysfs reads from `frame_status.py`. In the headset HUD only: the app shows them once, in the battery menu at the top. |
 
 OpenVR uses background application mode, which does not start SteamVR or keep
 it running. This mode also returned live timing in a read-only device probe.

scripts/e2e.sh

@@ -23,7 +23,8 @@ started=$SECONDS
 
 print "==> Building fakeframe-frame (from $base) and fakeframe-host"
 # Quiet when it works; if a build fails, build again with the full log so CI shows why.
-build() { docker build -q "$@" >/dev/null || { docker build --progress=plain "$@"; exit 2 } }
+# That second build also rides out a flaky package mirror: only its failure is fatal.
+build() { docker build -q "$@" >/dev/null || docker build --progress=plain "$@" || exit 2 }
 build --build-arg BASE="$base" -t fakeframe-frame -f tests/fakeframe/Containerfile tests/fakeframe
 build -t fakeframe-host -f tests/fakeframe/host.Containerfile tests/fakeframe
 

scripts/windows-vm.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env zsh
+# Mac or Linux: drive a Windows 11 test VM for Frame Control's Windows build.
+# The VM is a dockur/windows container on another machine; this reaches it over
+# SSH through that machine. See docs/testing.md#windows-test-vm.
+#
+# Usage: scripts/windows-vm.sh <command> [args]
+#   up | down | status      start it (waits for SSH), shut Windows down cleanly, show state
+#   ps '<PowerShell>'       run PowerShell as the VM's user
+#   put <file> [<dir>]      copy a file in (default: the user's Downloads)
+#   shot <out.png>          save the VM's screen
+#   click <x> <y>           click screen pixel x,y in the signed-in session
+#   scroll <x> <y> <n>      turn the wheel n notches at x,y (negative scrolls down)
+#   keys <key>...           type QEMU key names: a, shift-a, ret, tab, esc, spc ...
+#   frame-key add|remove    let the VM's Frame Control key into the headset (`frame`
+#                           alias) for a test run, then take it out again
+#
+# Set WINVM_HOST to the ssh alias of the machine running the container. Optional:
+# WINVM_CONTAINER (frame-winvm), WINVM_USER (frame), WINVM_PORT (2222: the VM's
+# sshd, published on that machine's loopback), WINVM_KEY (~/.ssh/id_ed25519_winvm).
+set -euo pipefail
+setopt extendedglob
+
+host=${WINVM_HOST:?set WINVM_HOST to the ssh alias of the machine running the VM}
+ctr=${WINVM_CONTAINER:-frame-winvm}
+user=${WINVM_USER:-frame}
+port=${WINVM_PORT:-2222}
+key=${WINVM_KEY:-$HOME/.ssh/id_ed25519_winvm}
+opts=(-o ConnectTimeout=20 -o StrictHostKeyChecking=accept-new
+      -o UserKnownHostsFile=${TMPDIR:-/tmp}/windows-vm-known_hosts -i $key -J $host)
+TAG=windows-vm-test  # comment on the VM's key in the headset's authorized_keys
+
+die() { print -u2 "windows-vm: $*"; exit 1 }
+int() { [[ $1 == (-|)<-99999> ]] || die "not a whole number (up to 99999): $1" }
+# These go into commands run by a shell on the other machine, so keep them plain.
+for v in $host $ctr $user; do [[ $v == [A-Za-z0-9_.]##[A-Za-z0-9_.-]# ]] || die "not a plain name: $v"; done
+[[ $port == <1-65535> ]] || die "WINVM_PORT isn't a port: $port"
+
+# Windows' OpenSSH waits for stdin to close, so it always gets /dev/null. The
+# script travels UTF-16 base64-encoded, so no quoting survives two shells.
+vm_ps() {
+  local b64=$(print -rn -- "\$ProgressPreference = 'SilentlyContinue'"$'\n'"$1" |
+              iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n')
+  ssh $opts -p $port $user@127.0.0.1 "powershell -NoProfile -NonInteractive -OutputFormat Text -EncodedCommand $b64" </dev/null
+}
+
+# QEMU's monitor inside the container: one command per line on stdin.
+monitor() { ssh $host "docker exec -i $ctr nc -q 1 -U /run/shm/monitor.sock" >/dev/null }
+
+# Input has to come from the signed-in desktop session, not SSH's session 0, so
+# a scheduled task running as the user replays one click or wheel turn written
+# to input.txt, then deletes the file to say it's done. Any error stops it
+# before that, so the caller times out instead of reporting a click.
+INPUT_PS1='$ErrorActionPreference = "Stop"
+$a = (Get-Content "$PSScriptRoot\input.txt").Trim() -split "\s+"
+Add-Type -Namespace WinVm -Name Input -MemberDefinition @"
+[DllImport("user32.dll")] public static extern bool SetProcessDPIAware();
+[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
+[DllImport("user32.dll")] public static extern void mouse_event(uint flags, int dx, int dy, int data, System.IntPtr extra);
+"@
+[WinVm.Input]::SetProcessDPIAware() | Out-Null
+if (-not [WinVm.Input]::SetCursorPos([int]$a[0], [int]$a[1])) { throw "SetCursorPos failed" }
+Start-Sleep -Milliseconds 150
+if ($a[2] -eq "click") {
+  [WinVm.Input]::mouse_event(0x2, 0, 0, 0, [IntPtr]::Zero); Start-Sleep -Milliseconds 60
+  [WinVm.Input]::mouse_event(0x4, 0, 0, 0, [IntPtr]::Zero)
+} else { [WinVm.Input]::mouse_event(0x800, 0, 0, 120 * [int]$a[3], [IntPtr]::Zero) }
+Remove-Item "$PSScriptRoot\input.txt"'
+
+pointer() {  # x y click|wheel [notches]
+  local b64=$(print -rn -- $INPUT_PS1 | base64 | tr -d '\n')
+  vm_ps '$ErrorActionPreference = "Stop"
+$d = Join-Path $env:LOCALAPPDATA "windows-vm"; $req = "$d\input.txt"; $mine = $false
+# Giving up: end the helper first, or it could wake up and act in a later request.
+function Abandon {
+  Stop-ScheduledTask -TaskName WindowsVmInput -ErrorAction SilentlyContinue
+  foreach ($i in 1..25) {
+    if ((Get-ScheduledTask -TaskName WindowsVmInput -ErrorAction SilentlyContinue).State -ne "Running") { break }
+    Start-Sleep -Milliseconds 200
+  }
+  Remove-Item $req -ErrorAction SilentlyContinue
+}
+trap { [Console]::Error.WriteLine("windows-vm: $_"); if ($mine) { Abandon }; exit 1 }
+# One request at a time: the helper, the task and input.txt are shared. Windows
+# frees the mutex when this process ends, however it ends.
+$lock = New-Object Threading.Mutex($false, "windows-vm-input")
+try { $mine = $lock.WaitOne(15000) } catch [Threading.AbandonedMutexException] { $mine = $true }
+if (-not $mine) { [Console]::Error.WriteLine("windows-vm: another click or scroll is still in progress"); exit 1 }
+New-Item -ItemType Directory -Force $d | Out-Null
+[IO.File]::WriteAllText($req, "'"$*"'")
+[IO.File]::WriteAllText("$d\input.ps1", [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String("'$b64'")))
+$act = New-ScheduledTaskAction -Execute powershell.exe -Argument "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$d\input.ps1`""
+$who = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive
+Register-ScheduledTask -TaskName WindowsVmInput -Action $act -Principal $who -Force | Out-Null
+Start-ScheduledTask -TaskName WindowsVmInput
+foreach ($i in 1..50) { if (-not (Test-Path $req)) { exit 0 }; Start-Sleep -Milliseconds 200 }
+Abandon
+[Console]::Error.WriteLine("windows-vm: no input after 10 s: is $env:USERNAME signed in on the VM screen, and is x,y on it?"); exit 1'
+}
+
+(( $# )) || die "usage: see the top of $0"
+cmd=$1; shift
+case $cmd in
+  up)
+    ssh $host "docker start $ctr" >/dev/null
+    for i in {1..60}; do
+      vm_ps 'exit 0' 2>/dev/null && { print "up"; exit 0 }
+      sleep 5
+    done
+    die "Windows didn't answer on SSH within 5 minutes" ;;
+  down)   # the container turns SIGTERM into an ACPI shutdown and waits for Windows
+    ssh $host "docker stop -t 150 $ctr" >/dev/null && print "down" ;;
+  status) ssh $host "docker ps -a --filter 'name=^$ctr\$' --format '{{.Names}}: {{.Status}}'" ;;
+  ps)     (( $# == 1 )) || die "usage: ps '<PowerShell>'"; vm_ps "$1" ;;
+  put)
+    [[ -f ${1:-} ]] || die "usage: put <file> [<dir>]"
+    scp -q $opts -P $port $1 "$user@127.0.0.1:${2:-C:/Users/$user/Downloads}/${1:t}" </dev/null ;;
+  shot)
+    (( $# == 1 )) || die "usage: shot <out.png>"
+    print "screendump /tmp/windows-vm-shot.ppm" | monitor
+    sleep 1
+    ssh $host "docker exec $ctr sh -c 'cat /tmp/windows-vm-shot.ppm && rm /tmp/windows-vm-shot.ppm'" | python3 -c '
+import re, struct, sys, zlib
+d = sys.stdin.buffer.read()
+m = re.match(rb"P6\s+(\d+)\s+(\d+)\s+255\s", d) or sys.exit("windows-vm: no screen dump")
+w, h = int(m[1]), int(m[2]); px = d[m.end():]
+raw = b"".join(b"\0" + px[y * w * 3:(y + 1) * w * 3] for y in range(h))
+chunk = lambda t, b: struct.pack(">I", len(b)) + t + b + struct.pack(">I", zlib.crc32(t + b))
+open(sys.argv[1], "wb").write(b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
+                              + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b""))
+' $1
+    print $1 ;;
+  click)  (( $# == 2 )) || die "usage: click <x> <y>"; int $1; int $2; pointer $1 $2 click ;;
+  scroll) (( $# == 3 )) || die "usage: scroll <x> <y> <n>"; int $1; int $2; int $3; pointer $1 $2 wheel $3 ;;
+  keys)
+    (( $# )) || die "usage: keys <key>..."
+    for k; do [[ $k == [a-z0-9_.,/=-]## ]] || die "not a QEMU key name: $k"; done
+    for k; do print "sendkey $k"; done | monitor ;;
+  frame-key)
+    [[ ${1:-} == (add|remove) ]] || die "usage: frame-key add|remove"
+    pub=(${=$(vm_ps 'Get-Content (Join-Path $env:USERPROFILE ".ssh\id_ed25519_frame.pub")' | tr -d '\r')})
+    [[ ${pub[1]:-} == ssh-ed25519 && ${pub[2]:-} == [A-Za-z0-9+/=]## ]] ||
+      die "the VM has no Frame Control key yet: run Set Up Connection in the app first"
+    # $1: add|remove, $2: the key's base64, $3: the exact line this script owns.
+    ssh frame "sh -s -- $1 '$pub[2]' '$pub[1] $pub[2] $TAG'" <<'EOF'
+f=~/.ssh/authorized_keys
+if [ "$1" = add ]; then
+  if grep -qxF "$3" "$f"; then exit 0; fi
+  if grep -qF "$2" "$f"; then echo "the headset already trusts this key through another entry; left as it is"; exit 0; fi
+  [ -z "$(tail -c 1 "$f")" ] || echo >> "$f"   # a last line without a newline would swallow ours
+  echo "$3" >> "$f"
+else
+  t=$(mktemp "$f.XXXXXX") || exit 1
+  grep -vxF "$3" "$f" > "$t"   # 0: lines left, 1: none left, more: couldn't read or write
+  if [ $? -gt 1 ] || ! chmod 600 "$t" || ! mv "$t" "$f"; then
+    rm -f "$t"; echo "couldn't rewrite $f; it's unchanged" >&2; exit 1
+  fi
+fi
+EOF
+    print "frame-key $1: done" ;;
+  *) die "unknown command: $cmd (see the top of $0)" ;;
+esac

site/README.md

@@ -4,7 +4,6 @@ The Frame Control website, <https://frame-control.pages.dev>, on Cloudflare Page
 
 - `public/`: static pages. `/` is the landing page, `/feedback/` the feedback form, `/privacy/` the privacy note.
 - `functions/api/feedback.js`: `POST /api/feedback`, which turns the form into a GitHub issue labelled `feedback`.
-- `functions/api/mailing-list.js`: `POST /api/mailing-list`, private mailing-list opt-ins in Cloudflare D1.
 - `lib/feedback.js`: validation and issue formatting, tested by `test/feedback.test.mjs`.
 - `public/js/site.js`: settings, including the Ko-fi page name for the donate buttons.
 
@@ -34,86 +33,3 @@ npx wrangler pages deploy --branch main      # production
 ```
 
 Point `GITHUB_REPO` in `.dev.vars` at a scratch repo when testing locally so test issues don't land on the real tracker.
-
-## Mailing list
-
-The app offers signup on first launch, after its privacy notice, because the
-platform installers have no shared interactive UI. Joining later and removing
-the address live in **Privacy & updates**. The email field is empty, **No thanks**
-is permanent, and neither the prompt nor skipping contacts this endpoint.
-
-The backend lives on the existing Cloudflare Pages project. It needs one private
-D1 database, bound as `MAILING_LIST`. No new third-party account or API key is
-needed. With an authenticated Wrangler session, provision it once:
-
-```sh
-cd site
-npx wrangler login
-npx wrangler d1 create frame-control-mailing-list --binding MAILING_LIST --update-config
-npx wrangler d1 execute frame-control-mailing-list --remote --file migrations/mailing-list/0001_subscribers.sql
-```
-
-The create command adds the real database ID and binding to `wrangler.toml`.
-Commit that non-secret configuration. If the database already exists, use its
-ID from `npx wrangler d1 list` instead of creating another one. Apply the schema
-above before deploying. Deploy this site's functions and privacy page with
-`npx wrangler pages deploy --branch main`; the normal PR does not deploy them.
-For a Pages preview, bind a separate test database rather than the production
-list. Without the binding or schema, the endpoint returns 503 and the app keeps
-the pending choice locally, rather than claiming it was delivered.
-
-The contract is JSON `{token, rev, email, updates}`: `updates` must explicitly
-be `true` with a valid email, or `false` with an empty email to withdraw. `token`
-is a random 256-bit capability stored only in this copy's local contact file,
-separate from analytics. Only its SHA-256 hash is stored on the server. Updates
-use a conditional SQLite upsert so retries are idempotent and older requests
-can't undo newer choices. Withdrawal clears the address, keeping a revision
-tombstone. Only the current address is stored, never an address history.
-Browser-origin requests are refused, bodies are limited to 2 KiB, responses
-are never cached, and the site's existing `FEEDBACK_RL`/`GITHUB_TOKEN` bindings
-provide a best-effort limit of 30 requests per IP per hour using salted hashes.
-KV limits are eventually consistent, so use Cloudflare WAF limits if needed for
-larger bursts. Database failures never log email addresses or capability tokens.
-
-The small client integration point is `ui/frame_mailing.py:submit`, with
-`FRAME_CONTROL_MAILING_LIST_URL` overriding the default
-`https://framecontrol.fun/api/mailing-list`. Saving is immediate; the existing
-contact worker retries mailing changes separately from PostHog every ten
-minutes and after a new choice. A removal wipes any queued address immediately.
-There is no app-wide secret in the client and no public list/export endpoint.
-
-For a local test, after adding the binding, initialize only the local database:
-
-```sh
-npx wrangler d1 execute frame-control-mailing-list --local --file migrations/mailing-list/0001_subscribers.sql
-npx wrangler pages dev --port 8788
-```
-
-Start the app with
-`FRAME_CONTROL_MAILING_LIST_URL=http://127.0.0.1:8788/api/mailing-list` and a
-disposable `FRAME_CONTROL_DATA_DIR`. Tests use temporary SQLite databases and
-local HTTP servers, never the production list.
-
-### Maintainer export
-
-Only an authenticated Cloudflare account can read the list:
-
-```sh
-npx wrangler d1 execute frame-control-mailing-list --remote --json --command "SELECT DISTINCT lower(email) AS email FROM mailing_list WHERE email IS NOT NULL ORDER BY email"
-```
-
-Treat the output as private. Export current opt-ins immediately before sending;
-do not use the older PostHog `contacts updates` output as the delivery list.
-An app removal withdraws this copy's subscription; the same address explicitly
-subscribed from another copy remains subscribed there. Addresses are unverified:
-this collection endpoint sends no confirmation or newsletter messages. A future
-sender should verify ownership before sending updates and include an unsubscribe
-link in every message, apply withdrawals, and update the privacy note before
-transferring addresses. Choosing or creating a sending service is not required
-to collect signups on Cloudflare.
-
-Cloudflare's own Email Service is not a newsletter sender: its
-[FAQ](https://developers.cloudflare.com/email-service/reference/faq/#can-i-use-email-service-for-marketing-emails)
-explicitly limits it to transactional email and excludes marketing email.
-Use Pages/D1 for collecting this list; do not route feature-news campaigns
-through that transactional service.

site/functions/api/mailing-list.js

@@ -1,76 +0,0 @@
-// Private signup storage on the existing Cloudflare Pages site. No email sender
-// or third-party mailing account is needed to collect explicit opt-ins.
-import { hashIp } from "../../lib/feedback.js";
-
-export const UPDATE = `INSERT INTO mailing_list (subscriber_id, rev, email)
-  VALUES (?, ?, ?)
-  ON CONFLICT(subscriber_id) DO UPDATE SET rev = excluded.rev,
-    email = excluded.email, updated_at = CURRENT_TIMESTAMP
-  WHERE excluded.rev > mailing_list.rev`;
-
-const json = (status, data) => new Response(JSON.stringify(data), {
-  status, headers: { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" },
-});
-
-export async function onRequestPost({ request, env }) {
-  // This endpoint is called by the app's Python server, never a browser form.
-  // Reject cross-site browser submissions instead of enabling CORS.
-  if (request.headers.has("origin")) return json(403, { error: "Use Frame Control's mailing list setting." });
-  if (!env.MAILING_LIST) return json(503, { error: "Mailing list storage is not configured yet." });
-  if (!request.headers.get("content-type")?.startsWith("application/json")) {
-    return json(415, { error: "Send JSON." });
-  }
-  let input;
-  try {
-    const reader = request.body?.getReader();
-    if (!reader) return json(400, { error: "Send JSON." });
-    const chunks = [];
-    let length = 0;
-    while (true) {
-      const { value, done } = await reader.read();
-      if (done) break;
-      length += value.length;
-      if (length > 2048) {
-        await reader.cancel();
-        return json(413, { error: "Request too large." });
-      }
-      chunks.push(value);
-    }
-    const bytes = new Uint8Array(length);
-    let offset = 0;
-    for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; }
-    input = JSON.parse(new TextDecoder().decode(bytes));
-  } catch {
-    return json(400, { error: "Send JSON." });
-  }
-  if (!input || typeof input.token !== "string" || !/^[a-f0-9]{64}$/.test(input.token) ||
-      !Number.isSafeInteger(input.rev) || input.rev < 1 ||
-      typeof input.email !== "string" || typeof input.updates !== "boolean") {
-    return json(400, { error: "Invalid mailing list choice." });
-  }
-  const email = input.email.trim();
-  if ((input.updates && (!email || email.length > 254 || !/^[^@\s]+@[^@\s]+\.[^@\s.]+$/.test(email))) ||
-      (!input.updates && email)) {
-    return json(400, { error: "An email needs explicit opt-in; withdraw with an empty email." });
-  }
-  try {
-    // Reuse the site's existing rate-limit namespace and secret salt. Never
-    // store raw IP addresses, email addresses, or capabilities in rate-limit keys.
-    if (env.FEEDBACK_RL && env.GITHUB_TOKEN) {
-      const who = await hashIp(request.headers.get("cf-connecting-ip") || "unknown", env.GITHUB_TOKEN);
-      const key = `mailing:ip:${who}:${Math.floor(Date.now() / 3600e3)}`;
-      const count = Number(await env.FEEDBACK_RL.get(key)) || 0;
-      if (count >= 30) return json(429, { error: "Please try again later." });
-      await env.FEEDBACK_RL.put(key, String(count + 1), { expirationTtl: 3900 });
-    }
-    const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(input.token));
-    const id = [...new Uint8Array(digest)].map(b => b.toString(16).padStart(2, "0")).join("");
-    await env.MAILING_LIST.prepare(UPDATE).bind(id, input.rev, email || null).run();
-    return json(200, { ok: true });
-  } catch {
-    // No address, token, request body, or storage error details in logs/responses.
-    return json(503, { error: "Couldn't save just now. Frame Control will retry." });
-  }
-}
-
-export const onRequest = () => json(405, { error: "POST only." });

site/migrations/mailing-list/0001_subscribers.sql

@@ -1,8 +0,0 @@
--- One current choice per random app capability, never a history of addresses.
--- Keep a tombstone on withdrawal so an older retry cannot add the address back.
-CREATE TABLE IF NOT EXISTS mailing_list (
-  subscriber_id TEXT PRIMARY KEY,
-  rev INTEGER NOT NULL CHECK (rev > 0),
-  email TEXT,
-  updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
-);

site/public/privacy/index.html

@@ -38,11 +38,10 @@
     <div class="head">
       <span class="kicker">Privacy</span>
       <h1>Privacy</h1>
-      <p>Short version: the app sends anonymous usage statistics you can turn off, your email only if you join the mailing list, and the website keeps only what you choose to send.</p>
+      <p>Short version: the app collects nothing, and the website keeps only what you choose to send.</p>
     </div>
     <div class="faq notes">
-      <section><h2>The app</h2><p>Frame Control talks only to your headset (over SSH on your network), to GitHub for releases, and to Steam and F-Droid for game and app listings. It has no accounts. It sends anonymous usage statistics, which you can turn off in Privacy &amp; updates; <a href="https://github.com/saphid/frame-control/blob/main/docs/privacy.md">privacy.md</a> lists exactly what.</p></section>
-      <section><h2>The mailing list</h2><p>The first time the app opens, it offers to add you to the Frame Control mailing list for news about new features and updates. It's optional: the email field starts empty, <b>No thanks</b> skips it for good, and the app works the same either way. Only if you join, your address and consent are sent privately to the maintainer’s PostHog project and to this website’s Cloudflare Pages endpoint. Cloudflare D1 stores the current mailing-list address with a random subscription identifier and revision; no analytics identifier or machine details go to the mailing-list endpoint. Removing it clears that address from the list when the app can reach the service. No newsletter sending service is configured yet. Unsubscribe or remove the address any time in Privacy &amp; updates.</p></section>
+      <section><h2>The app</h2><p>Frame Control talks only to your headset (over SSH on your network), to GitHub for releases, and to Steam and F-Droid for game and app listings. It has no analytics and no accounts.</p></section>
       <section><h2>The feedback form</h2><p>What you type becomes a public GitHub issue on <a href="https://github.com/saphid/frame-control/issues">saphid/frame-control</a>. To stop abuse, the form keeps a one-way hash of your IP address for about an hour to count submissions. The address itself isn't stored or published.</p></section>
       <section><h2>This website</h2><p>Hosted on Cloudflare Pages. No cookies, no analytics, no trackers. The download section asks GitHub for the latest version number. Donations go through Ko-fi, under Ko-fi's own privacy policy.</p></section>
     </div>

site/test/mailing-list.test.mjs

@@ -1,101 +0,0 @@
-import assert from "node:assert/strict";
-import { test } from "node:test";
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { spawnSync } from "node:child_process";
-import { onRequestPost, onRequest } from "../functions/api/mailing-list.js";
-
-const choice = (over = {}) => ({ token: "a".repeat(64), rev: 1, email: "me@example.com", updates: true, ...over });
-const request = (body, headers = {}) => new Request("https://framecontrol.fun/api/mailing-list", {
-  method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify(body),
-});
-const call = (body, env = {}, headers) => onRequestPost({ request: request(body, headers), env });
-
-// Execute the actual D1 SQLite statements and migration, rather than mirroring
-// their conflict handling in a mock. No Cloudflare account or network needed.
-function database(t) {
-  const dir = mkdtempSync(join(tmpdir(), "frame-mailing-"));
-  t.after(() => rmSync(dir, { recursive: true, force: true }));
-  const file = join(dir, "list.sqlite");
-  const sql = (query, args = []) => {
-    const result = spawnSync("python3", ["-c", `
-import json, sqlite3, sys
-db = sqlite3.connect(sys.argv[1])
-query, args = json.loads(sys.stdin.read())
-rows = db.execute(query, args).fetchall()
-db.commit()
-print(json.dumps(rows))
-`, file], { input: JSON.stringify([query, args]), encoding: "utf8" });
-    assert.equal(result.status, 0, result.stderr);
-    return JSON.parse(result.stdout);
-  };
-  sql(readFileSync(new URL("../migrations/mailing-list/0001_subscribers.sql", import.meta.url), "utf8"));
-  return { sql, prepare: query => ({ bind: (...args) => ({ run: async () => sql(query, args) }) }) };
-}
-
-test("stores one current choice; retries and delayed joins cannot undo a withdrawal", async t => {
-  const db = database(t), env = { MAILING_LIST: db };
-  assert.equal((await call(choice(), env)).status, 200);
-  assert.equal((await call(choice(), env)).status, 200);
-  assert.deepEqual(db.sql("SELECT rev, email FROM mailing_list"), [[1, "me@example.com"]]);
-  await call(choice({ rev: 3, email: "", updates: false }), env);
-  await call(choice({ rev: 2, email: "old@example.com" }), env);
-  assert.deepEqual(db.sql("SELECT rev, email FROM mailing_list"), [[3, null]]);
-  await call(choice({ rev: 4, email: "new@example.com" }), env);
-  assert.deepEqual(db.sql("SELECT rev, email FROM mailing_list"), [[4, "new@example.com"]]);
-  assert.notEqual(db.sql("SELECT subscriber_id FROM mailing_list")[0][0], choice().token);
-});
-
-test("each app capability only changes its own row", async t => {
-  const db = database(t), env = { MAILING_LIST: db };
-  await call(choice(), env);
-  await call(choice({ token: "b".repeat(64), email: "other@example.com" }), env);
-  await call(choice({ rev: 2, email: "", updates: false }), env);
-  assert.deepEqual(db.sql("SELECT email FROM mailing_list WHERE email IS NOT NULL"), [["other@example.com"]]);
-});
-
-test("requires a valid email and explicit boolean consent before storing anything", async () => {
-  let writes = 0;
-  const env = { MAILING_LIST: { prepare() { writes++; throw new Error(); } } };
-  for (const invalid of [null, {}, choice({ updates: "true" }), choice({ updates: false }),
-    choice({ email: "", updates: true }), choice({ email: "invalid" }),
-    choice({ token: "guess" }), choice({ token: ["a".repeat(64)] }), choice({ rev: 0 }), choice({ rev: 1.5 }),
-    choice({ email: "x".repeat(255) + "@example.com" })]) {
-    assert.equal((await call(invalid, env)).status, 400);
-  }
-  assert.equal(writes, 0);
-});
-
-test("rejects browser origins, oversized bodies, malformed JSON and wrong methods", async () => {
-  const env = { MAILING_LIST: {} };
-  assert.equal((await call(choice(), env, { origin: "https://elsewhere.example" })).status, 403);
-  assert.equal((await call(choice({ padding: "x".repeat(3000) }), env)).status, 413);
-  assert.equal((await call(choice(), env, { "content-type": "text/plain" })).status, 415);
-  const bad = new Request("https://framecontrol.fun/api/mailing-list", {
-    method: "POST", body: "{bad", headers: { "content-type": "application/json" },
-  });
-  assert.equal((await onRequestPost({ request: bad, env })).status, 400);
-  assert.equal(onRequest().status, 405);
-});
-
-test("unconfigured or unavailable storage fails safely without personal data in errors", async () => {
-  assert.equal((await call(choice())).status, 503);
-  const result = await call(choice(), { MAILING_LIST: { prepare() { throw new Error("private database details"); } } });
-  assert.equal(result.status, 503);
-  const body = await result.text();
-  assert.doesNotMatch(body, /me@example|aaaa|private database/);
-});
-
-test("uses the existing salted rate-limit store and refuses bursts", async () => {
-  const keys = [], writes = [];
-  const env = {
-    GITHUB_TOKEN: "test-salt",
-    FEEDBACK_RL: { get: async key => { keys.push(key); return "30"; }, put: async () => {} },
-    MAILING_LIST: { prepare: () => { writes.push(true); } },
-  };
-  assert.equal((await call(choice(), env, { "cf-connecting-ip": "192.0.2.8" })).status, 429);
-  assert.equal(writes.length, 0);
-  assert.equal(keys.length, 1);
-  assert.doesNotMatch(keys[0], /192\.0\.2\.8|example|test-salt/);
-});

site/wrangler.toml

@@ -10,7 +10,3 @@ GITHUB_REPO = "saphid/frame-control"
 [[kv_namespaces]]
 binding = "FEEDBACK_RL"
 id = "4852de5aae9e4d9f989f6dc2bc3b2b6b"
-
-# Mailing-list D1 binding: provision once with the command in site/README.md.
-# Wrangler adds [[d1_databases]] with binding = "MAILING_LIST" and the real ID.
-# Until then /api/mailing-list returns 503 and the app safely queues choices.

tests/control_center.cjs

@@ -64,7 +64,7 @@ function sharedPage({productionApi=false} = {}) {
       const request=deferred();request.path=path;request.opts=opts;requests.push(request);return request.promise;
     };
     Object.assign(context,{renderPill(){},renderConnDlg(){},renderBanner(){},viaText(){},netLabel(){},
-      resetShots(){},loadDevices(){},renderDevices(){},renderTests(){},reloadAll(){},checkContactPrompt(){}});
+      resetShots(){},closeDevPop(){},loadDevices(){},renderDevices(){},renderTests(){},reloadAll(){},checkContactPrompt(){}});
     vm.runInContext(`const UI_KEY = 'test-only';
       const link = {s:null,device:'frame-a',phase:'connected',rev:0,net:null};
       const disp = {}, gm = {seq:0}, cat = {}, wi = {};
@@ -450,7 +450,7 @@ test('every linked workflow selects its real page and section, preserving shortc
       if(!ids.has(id))return null;
       if(!sections.has(id))sections.set(id,{hidden:false,scrollIntoView(){scrolled=id;}});
       return sections.get(id);
-    },live:false,toggleLive(){},loadMacView(){},loadPanelSwitcher(){}};
+    },live:false,toggleLive(){},scrollToY(){},loadMacView(){},loadPanelSwitcher(){}};
   const androidStart=html.indexOf('function showAndroidTab(');
   const androidEnd=html.indexOf("document.querySelectorAll('[data-store-tab]').forEach(button => button.onclick",androidStart);
   vm.createContext(context);vm.runInContext(html.slice(androidStart,androidEnd)+html.slice(start,end)+'\nPAGES.push("devices", "control");',context);

tests/page/source_login.mjs

@@ -0,0 +1,79 @@
+// The store's "Add a source" login handling and GitHub token field, run in node against
+// the real functions from ui/index.html. A login typed into the address must leave the
+// address field (into the login fields) before anything is sent, and stay out of it.
+import { readFileSync } from "fs";
+const src = readFileSync(new URL("../../ui/index.html", import.meta.url), "utf8");
+const lines = src.split("\n");
+const fail = msg => { console.log("FAIL " + msg); process.exit(1); };
+
+function grab(name) {
+  const start = lines.findIndex(l => new RegExp(`^(?:(?:async )?function ${name}\\(|const ${name} = )`).test(l));
+  if (start < 0) fail(`no top-level ${name} in index.html`);
+  let depth = 0;
+  for (let i = start; i < lines.length; i++) {
+    for (const c of lines[i]) depth += c === "{" ? 1 : c === "}" ? -1 : 0;
+    if (depth <= 0) return lines.slice(start, i + 1).join("\n") + "\n";
+  }
+  fail(`unbalanced ${name}`);
+}
+const { moveLogin, sourceToken } = new Function(
+  "const esc = s => String(s).replace(/[&<>\"']/g, c => `&#${c.charCodeAt(0)};`); const storeName = s => s;\n" +
+  grab("decodeLenient") + grab("moveLogin") + grab("sourceToken") + "return { moveLogin, sourceToken };")();
+
+function form(url, username = "", password = "") {
+  const details = { open: false };
+  const field = value => ({ value, closest: () => details });
+  return { details, elements: { url: field(url), username: field(username), password: field(password) } };
+}
+
+let f = form("https://alice:s3cr%40t@repo.example/fdroid/repo");
+moveLogin(f);
+if (f.elements.url.value !== "https://repo.example/fdroid/repo") fail("address still holds the login: " + f.elements.url.value);
+if (f.elements.username.value !== "alice" || f.elements.password.value !== "s3cr@t") fail("login not moved into its fields");
+if (!f.details.open) fail("login fields not shown");
+
+f = form("https://alice:fromurl@repo.example/repo", "typed", "typedpw");
+moveLogin(f);
+if (f.elements.url.value.includes("fromurl") || f.elements.username.value !== "typed" || f.elements.password.value !== "typedpw")
+  fail("typed login replaced, or address kept the login");
+
+// Undecodable logins and unusual spellings the URL parser accepts still lose their login; undecodable bytes stay as typed.
+for (const [url, address, user, pass] of [
+  ["https://alice:100%done@repo.example/private/repo", "https://repo.example/private/repo", "alice", "100%done"],
+  ["https:\t//alice:s3cret@repo.example/repo", "https://repo.example/repo", "alice", "s3cret"],
+  ["ht\ntps://alice:s3cret@repo.example/repo", "https://repo.example/repo", "alice", "s3cret"],
+  ["https:alice:s3cret@repo.example/repo", "https://repo.example/repo", "alice", "s3cret"],
+  ["\u0000https://alice:s3cret@repo.example/repo", "https://repo.example/repo", "alice", "s3cret"],
+  ["https://alice:%FF@repo.example/private/repo", "https://repo.example/private/repo", "alice", "%FF"],
+  ["https://al%20ice:p%40ss:word@repo.example/r", "https://repo.example/r", "al ice", "p@ss:word"],
+  ["https://alice:p%FF\u00e4@repo.example/repo", "https://repo.example/repo", "alice", "p%FF\u00e4"],
+  ["fdroidrepos://alice:pw@repo.example/repo?fingerprint=AB", "fdroidrepos://repo.example/repo?fingerprint=AB", "alice", "pw"],
+  ["https://alice@repo.example/repo", "https://repo.example/repo", "alice", ""],
+]) {
+  f = form(url);
+  moveLogin(f);
+  if (f.elements.url.value !== address) fail(`address kept the login: ${url} -> ${f.elements.url.value}`);
+  if (f.elements.username.value !== user || f.elements.password.value !== pass)
+    fail(`login for ${url}: ${f.elements.username.value} / ${f.elements.password.value}`);
+}
+
+// An address the parser rejects that may hold a login is refused and left as typed, never sent.
+for (const url of ["https://alice\\name:pw@repo.example/repo", "https:\\\\alice:s3cret@repo.example/repo", "fdroidrepos://alice:p\\w@repo.example/repo", "https://alice:s3cret@repo example/repo", "https:alice:s3cret@repo example/repo", "https://repo.example:bad\\a@b"]) {
+  f = form(url);
+  if (moveLogin(f) !== false) fail("accepted an invalid address with a login: " + url);
+  if (f.elements.username.value || f.elements.password.value) fail("guessed a login from an invalid address: " + url);
+}
+
+for (const url of ["https://repo.example/repo", "not a url", "https://repo.example/a@b", "fdroidrepos:///a@b", "fdroidrepos:/a@b", "https://repo.example/r?x=a@b#c@d"]) {
+  f = form(url);
+  moveLogin(f);
+  if (f.elements.url.value !== url || f.elements.username.value) fail("changed an address without a login: " + url);
+}
+
+const html = sourceToken({ id: "github", name: "GitHub releases", token: { set: true, from: "saved", rejected: true } });
+if (!html.includes("rejected") || !html.includes('data-clear-token="github"')) fail("token row: " + html);
+if (sourceToken({ id: "fdroid", name: "F-Droid" }) !== "") fail("token field on a source without one");
+const slow = "https://a:" + "%FF".repeat(4000) + "@repo.example/r", t0 = Date.now();
+f = form(slow); moveLogin(f);
+if (Date.now() - t0 > 200 || f.elements.password.value !== "%FF".repeat(4000)) fail("decoding malformed escapes is slow or wrong");
+console.log("ok");

tests/sandbox.py

@@ -22,7 +22,6 @@ os.environ["FRAME_CONTROL_DATA_DIR"] = _dir
 os.environ["FRAME_CONTROL_TELEMETRY"] = "0"
 # A maintainer's machine holds the database key; send anything that slips through nowhere.
 os.environ["FRAME_COMPAT_DB_URL"] = "http://127.0.0.1:9"
-os.environ["FRAME_CONTROL_MAILING_LIST_URL"] = "http://127.0.0.1:9"
 
 # A throwaway home, so Path.home() / "~" land here: ~/.ssh (config, host-key pins,
 # its lock file) and the other per-user folders some modules use. A server started

tests/test_apk_search.py

@@ -325,7 +325,7 @@ class EndpointTests(SettingsTest):
             mod.add_repo.side_effect = SourceError('repository fingerprint mismatch')
             job = self.wait(self.request('POST', '/api/sources', {'action': 'add', 'url': link})[1]['job'])
             self.assertEqual(job['error'], 'repository fingerprint mismatch')  # no "SourceError:" prefix
-            for url in ('http://repo.example/repo', 'fdroidrepo://repo.example/repo', 'https://u@repo.example/'):
+            for url in ('http://repo.example/repo', 'fdroidrepo://repo.example/repo', 'https://repo.example:x/'):
                 self.assertEqual(self.request('POST', '/api/sources', {'action': 'add', 'url': url})[0], 400)
             self.assertEqual(self.request('POST', '/api/sources', {'action': 'remove', 'source': 'fdroid'})[0], 200)
             mod.remove_repo.assert_called_once_with(source_id='fdroid')

tests/test_contact.py

@@ -17,7 +17,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
 
 import frame_compat_db as db  # noqa: E402
 import frame_contact as fc  # noqa: E402
-import frame_mailing as fm  # noqa: E402
 import frame_report as fr  # noqa: E402
 import frame_telemetry as tm  # noqa: E402
 from test_telemetry import Base, ReportProblem  # noqa: E402
@@ -37,16 +36,6 @@ class Contact(Base):
             p.start()
             self.addCleanup(p.stop)
         self.got = self.serve()
-        mailing = mock.patch.object(fm, 'submit')
-        self.mailing = mailing.start()
-        self.addCleanup(mailing.stop)
-        # Most consent tests drain the background queue synchronously, so their
-        # assertions cover delivered events without sleeps or real services.
-        self.production_deliver = fc._deliver
-        deliver = mock.patch.object(fc, '_deliver', side_effect=lambda changed:
-                                    fc._send_pending(block=False) if changed else None)
-        deliver.start()
-        self.addCleanup(deliver.stop)
 
     def events(self):
         return [body["batch"][0] for _, body in self.got]
@@ -104,52 +93,6 @@ class Contact(Base):
         fc.save({"email": "me@example.com", "updates": True})
         self.assertEqual(len(self.got), 1)
 
-    def test_mailing_only_receives_explicit_updates_and_no_machine_details(self):
-        fc.prompt({'prompt': 'dismissed'})
-        fc.save({'email': 'reply@example.com', 'followup': True})
-        self.mailing.assert_not_called()
-        fc.save({'email': 'news@example.com', 'updates': True})
-        token, rev, email = self.mailing.call_args.args
-        self.assertEqual((len(token), rev, email), (64, 2, 'news@example.com'))
-        self.assertNotEqual(token, fc.load()['id'])
-        fc.save({'email': 'reply@example.com', 'followup': True})
-        self.assertEqual(self.mailing.call_args.args, (token, 3, ''))
-
-    def test_mailing_failure_waits_independently_of_posthog(self):
-        with mock.patch.object(fm, 'submit', side_effect=fm.SendError('offline')):
-            s = fc.save({'email': 'news@example.com', 'updates': True})
-        self.assertTrue(s['waiting'])
-        self.assertIsNone(fc.load()['pending'])
-        self.assertEqual(fc.load()['mailingPending']['email'], 'news@example.com')
-        with self.offline():
-            self.assertTrue(fc._send_pending())  # PostHog need not be available
-        self.assertFalse(fc.state()['waiting'])
-
-    def test_posthog_failure_does_not_hold_up_mailing_delivery(self):
-        with self.offline():
-            fc.save({'email': 'news@example.com', 'updates': True})
-        self.mailing.assert_called_once()
-        self.assertIsNone(fc.load()['mailingPending'])
-        self.assertIsNotNone(fc.load()['pending'])
-
-    def test_offline_removal_erases_the_queued_address_and_survives_reload(self):
-        with mock.patch.object(fm, 'submit', side_effect=fm.SendError('offline')):
-            fc.save({'email': 'news@example.com', 'updates': True})
-            fc.save({'email': ''})
-        s = fc.load()
-        self.assertEqual(s['mailingPending'], {'rev': 2, 'email': ''})
-        self.assertNotIn('news@example.com', fc.FILE.read_text())
-        self.assertTrue(fc._send_pending())
-        self.assertEqual(self.mailing.call_args.args, (s['mailingToken'], 2, ''))
-
-    def test_production_save_only_wakes_worker_without_network_io(self):
-        # Restore the production notifier while leaving the network mocked.
-        with mock.patch.object(fc, '_deliver', self.production_deliver):
-            s = fc.save({'email': 'news@example.com', 'updates': True})
-        self.assertTrue(s['waiting'])
-        self.mailing.assert_not_called()
-        self.assertEqual(self.got, [])
-
     # ---- withdrawal
 
     def test_removing_the_address_sends_a_withdrawal_without_it(self):
@@ -285,9 +228,10 @@ class Contact(Base):
 
     # ---- the one-time prompt
 
-    def test_the_prompt_is_offered_on_first_launch_then_stays_dismissed(self):
-        self.assertTrue(fc.state()["showPrompt"])  # a new install, before the Frame has connected
-        self.assertEqual(self.got, [])  # offering it sends nothing
+    def test_the_prompt_waits_for_a_working_setup_then_stays_dismissed(self):
+        self.assertFalse(fc.state()["showPrompt"])  # a new install: the Frame hasn't connected yet
+        tm.frame_seen("20260901.1", "3.8")
+        self.assertTrue(fc.state()["showPrompt"])
         fc.prompt({"prompt": "dismissed"})
         fc.prompt({"prompt": "shown"})  # a later session can't bring it back
         self.assertEqual(fc.load()["prompt"], "dismissed")
@@ -297,24 +241,12 @@ class Contact(Base):
             fc.prompt({"prompt": "reset"})
 
     def test_the_prompt_is_shown_once_and_saving_answers_it(self):
+        tm.frame_seen("20260901.1", "3.8")
         fc.prompt({"prompt": "shown"})
         self.assertFalse(fc.state()["showPrompt"])
         fc.save({"email": "me@example.com", "followup": True, "fromPrompt": True})
         self.assertEqual(fc.load()["prompt"], "answered")
 
-    def test_joining_from_the_prompt_is_the_mailing_list_opt_in_only(self):
-        fc.save({"email": " me@example.com ", "updates": True, "fromPrompt": True})
-        self.assertEqual((fc.load()["prompt"], fc.state()["showPrompt"]), ("answered", False))
-        props = self.events()[-1]["properties"]
-        self.assertEqual((props["email"], props["updates"], props["followup"], props["action"]),
-                         ("me@example.com", True, False, "set"))
-
-    def test_a_failed_send_never_blocks_joining(self):
-        with self.offline():
-            s = fc.save({"email": "me@example.com", "updates": True, "fromPrompt": True})
-        self.assertTrue(s["updates"] and s["waiting"])  # kept here, sent later by the retrier
-        self.assertFalse(s["showPrompt"])
-
     # ---- reports and the maintainer's list
 
     def reports(self):

tests/test_devices.py

@@ -323,6 +323,20 @@ class Pins(Base):
 
 
 class Registry(Base):
+    def test_an_address_added_first_wins_on_its_own_network(self):
+        # The page's "Add 192.168.x.x" offer: the headset is reached over Tailscale, which has
+        # worked here before. The LAN address has to go ahead of it to be used at home.
+        d = self.reg.add_device("frame-4", hosts=["frame.tail1234.ts.net"])
+        self.reg.record_success(d["id"], "frame.tail1234.ts.net", "n-home", 6.0)
+        self.reg.add_address(d["id"], "192.168.1.40", kind="lan", first=True)
+        self.reg.record_success(d["id"], "192.168.1.40", "n-home", 1.0)  # Test now found it
+        addrs = self.reg.get(d["id"])["addresses"]
+        self.assertEqual([a["host"] for a in addrs], ["192.168.1.40", "frame.tail1234.ts.net"])
+        at_home = [a["host"] for a, _ in fd.order_addresses(addrs, "n-home", True)]
+        self.assertEqual(at_home[0], "192.168.1.40")
+        away = [a["host"] for a, _ in fd.order_addresses(addrs, "n-cafe", True)]
+        self.assertEqual(away[0], "frame.tail1234.ts.net")  # elsewhere Tailscale still leads
+
     def test_address_editing(self):
         d = self.reg.add_device("frame-3", hosts=["192.168.1.40"])
         a = self.reg.add_address(d["id"], "frame-3.local", label="mDNS")

tests/test_fdroid_sources.py

@@ -81,7 +81,7 @@ class Repositories(unittest.TestCase):
             state.clear()
             self.addCleanup(state.clear)
 
-    def fetch(self, url, path, maximum):
+    def fetch(self, url, path, maximum, auth=None):
         name = url.rsplit('/', 1)[-1]
         if self.v1 and name == 'entry.jar':
             raise urllib.error.HTTPError(url, 404, 'missing', None, None)
@@ -311,7 +311,7 @@ class Repositories(unittest.TestCase):
         source = self.add()
         other = dict(source, id='other-repo')
         started, release = threading.Event(), threading.Event()
-        def fetch(url, path, maximum):
+        def fetch(url, path, maximum, auth=None):
             if threading.current_thread().name == 'slow':
                 started.set()
                 release.wait(5)
@@ -355,7 +355,7 @@ class Repositories(unittest.TestCase):
         self.files['entry.jar'], _ = entry_jar(50)
         source = fdroid.add_repo(URL)
         jars = {'older': entry_jar(100)[0], 'newer': entry_jar(200)[0]}
-        def fetch(url, path, maximum):
+        def fetch(url, path, maximum, auth=None):
             name = threading.current_thread().name
             if name in jars and url.endswith('entry.jar'):
                 Path(path).write_bytes(jars[name])
@@ -401,7 +401,7 @@ class Repositories(unittest.TestCase):
         source = dict(id='overlap', name='Overlap', url=URL, fingerprint=None)
         self.v1 = True
         inner = []
-        def fetch(url, path, maximum):
+        def fetch(url, path, maximum, auth=None):
             if url.endswith('index-v1.jar') and not inner:
                 inner.append(1)  # the v1 load passed its fallback check; a v2 load finishes now
                 self.v1 = False
@@ -471,7 +471,7 @@ class Repositories(unittest.TestCase):
         source = self.add()
         self.expire(source)
         release = threading.Event()
-        def slow(url, path, maximum):
+        def slow(url, path, maximum, auth=None):
             release.wait(5)
             self.fetch(url, path, maximum)
         self.fetch_mock.side_effect = slow
@@ -528,7 +528,7 @@ class Repositories(unittest.TestCase):
         self.expire(source)
         before = self.fetch_mock.call_count
         release = __import__('threading').Event()
-        def slow(url, path, maximum):
+        def slow(url, path, maximum, auth=None):
             release.wait(5)
             self.fetch(url, path, maximum)
         self.fetch_mock.side_effect = slow

tests/test_frame_android_library.py

@@ -121,7 +121,27 @@ class LauncherTests(unittest.TestCase):
 FIXTURES = ROOT / 'tests/fixtures/library'
 
 
+def wait_for_idle_resolvers(timeout=10):
+    # Name lookups that outlast their deadline give their slot back from their own thread,
+    # which a busy runner may not schedule before the next test; wait until all are back.
+    from apk_sources import _images
+    held = []
+    try:
+        end = time.monotonic() + timeout
+        while _images._resolvers.acquire(timeout=max(0, end - time.monotonic())):
+            held.append(1)
+            if len(held) == 4:
+                return
+        raise AssertionError('artwork name lookups from an earlier test are still running')
+    finally:
+        for _ in held:
+            _images._resolvers.release()
+
+
 class ArtworkTests(unittest.TestCase):
+    def setUp(self):
+        wait_for_idle_resolvers()
+
     def test_source_inputs_and_url(self):
         from apk_sources import _images
         data = (FIXTURES / 'icon.png').read_bytes()
@@ -232,10 +252,7 @@ class ArtworkTests(unittest.TestCase):
             self.assertEqual(sum('Too many' in e for e in errors), 2)
         finally:
             gate.set()
-        deadline = time.monotonic() + 5
-        while time.monotonic() < deadline and not _images._resolvers.acquire(blocking=False):
-            time.sleep(0.01)
-        _images._resolvers.release()  # the stuck lookups finished and gave their slots back
+        wait_for_idle_resolvers()  # the stuck lookups finished and gave their slots back
 
     def test_resolver_slot_released_when_thread_cannot_start(self):
         from apk_sources import _images
@@ -243,10 +260,14 @@ class ArtworkTests(unittest.TestCase):
             for _ in range(6):
                 with self.assertRaises(RuntimeError):
                     _images.get('https://example.org/a.png', deadline=time.monotonic() + 1)
-        for _ in range(4):  # every slot came back
-            self.assertTrue(_images._resolvers.acquire(blocking=False))
-        for _ in range(4):
-            _images._resolvers.release()
+        held = 0
+        try:
+            while held < 4 and _images._resolvers.acquire(blocking=False):
+                held += 1
+            self.assertEqual(held, 4)  # every slot came back
+        finally:
+            for _ in range(held):  # even on failure, so later tests don't inherit the leak
+                _images._resolvers.release()
 
     def test_deadline_covers_name_resolution(self):
         import threading

tests/test_link.py

@@ -897,6 +897,47 @@ class Connecting(unittest.TestCase):
         self.assertEqual(self.routes, [])
         self.assertTrue(all("ControlPath=none" in c for c in self.calls() if "-G" not in c))
 
+    def test_a_test_started_earlier_cant_overwrite_a_newer_one(self):
+        d = self.device("nothing.invalid")
+        entered = {1: threading.Event(), 2: threading.Event()}
+        release = {1: threading.Event(), 2: threading.Event()}
+        calls = []
+
+        def probe(host, port, update=None):
+            calls.append(host)
+            n = len(calls)
+            entered[n].set()
+            release[n].wait(10)
+            return {"state": "refused", "detail": f"test {n}", "ip": None, "rtt_ms": None}
+
+        tests = [threading.Thread(target=self.link.test, args=(d["id"],), daemon=True) for _ in range(2)]
+        with mock.patch.object(fl, "probe", probe):
+            try:
+                tests[0].start()
+                self.assertTrue(entered[1].wait(5), "the first test never probed")
+                tests[1].start()
+                self.assertTrue(entered[2].wait(5), "the second test never probed")
+                # The first finishes while the second is still probing: it mustn't show its
+                # rows or mark the second done.
+                release[1].set()
+                tests[0].join(10)
+                self.assertFalse(tests[0].is_alive())
+                running = self.link.snapshot()["tests"][d["id"]]
+                self.assertFalse(running["done"])
+                self.assertEqual(running["rows"][0]["detail"], "Waiting")
+                release[2].set()
+                tests[1].join(10)
+                self.assertFalse(tests[1].is_alive())
+            finally:
+                for e in release.values():
+                    e.set()
+                for t in tests:
+                    if t.ident:  # started
+                        t.join(10)
+        result = self.link.snapshot()["tests"][d["id"]]
+        self.assertTrue(result["done"])
+        self.assertEqual(result["rows"][0]["detail"], "test 2")
+
     def test_switching_to_a_headset_that_never_answers_stops_using_the_last_one(self):
         self.device("localhost")
         self.hosts({"localhost": "ok"})

tests/test_link_ui.py

@@ -56,18 +56,19 @@ class LinkUI(unittest.TestCase):
         r = subprocess.run(["node", "-e", script], capture_output=True, text=True)
         self.assertEqual(r.returncode, 0, r.stderr)
 
-    def test_the_pill_leads_with_the_reason(self):
-        # The header pill cut off "Offline: checking the headset's identity failed · retry in…":
-        # the reason comes first, in a few words, and only the countdown gets cut.
+    def test_the_pill_keeps_the_failure_detail_in_its_tooltip(self):
+        # The connection redesign keeps the state and retry countdown in the pill,
+        # with the full failure explanation in its tooltip and details dialog.
         self.run_page(r'''
   link.s = changed;
   renderPill();
-  assert.match($('conn').innerHTML, /<b>Office Frame<\/b><span>SSH identity changed · retry in 12 s<\/span>/);
-  assert.equal($('conn').title, 'Office Frame is offline: SSH identity changed · retry in 12 s. Click for details.');
+  assert.match($('conn').innerHTML, /<b>Office Frame<\/b><span>Offline<span class="via"> · retrying in 12 s<\/span>/);
+  assert.ok($('conn').title.includes(changed.error.message));
   // Without a short reason: the stage that failed.
   link.s = {...changed, retry_at: null, error: {stage: 'identity', message: 'Something.'}};
   renderPill();
-  assert.match($('conn').innerHTML, /<span>Checking the headset&#39;s identity failed<\/span>/);
+  assert.match($('conn').innerHTML, /<span>Offline<\/span>/);
+  assert.ok($('conn').title.includes('Something.'));
 ''')
 
     def test_the_banner_never_doubles_a_full_stop(self):

tests/test_mailing.py

@@ -1,85 +0,0 @@
-"""Mailing-list HTTP contract; all requests go to a disposable local server."""
-import sandbox  # noqa: F401
-import json
-import os
-import subprocess
-import sys
-import tempfile
-import threading
-import unittest
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from pathlib import Path
-from unittest import mock
-
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'ui'))
-import frame_mailing as fm  # noqa: E402
-
-
-class MailingTransport(unittest.TestCase):
-    def setUp(self):
-        self.got = []
-        self.status, self.answer = 200, b'{"ok":true}'
-        owner = self
-
-        class Handler(BaseHTTPRequestHandler):
-            def do_POST(self):
-                owner.got.append((self.path, json.loads(self.rfile.read(int(self.headers['Content-Length'])))))
-                self.send_response(owner.status)
-                self.end_headers()
-                self.wfile.write(owner.answer)
-
-            def log_message(self, *args):
-                pass
-
-        server = HTTPServer(('127.0.0.1', 0), Handler)
-        thread = threading.Thread(target=server.serve_forever, daemon=True)
-        thread.start()
-        self.addCleanup(server.server_close)
-        self.addCleanup(server.shutdown)
-        env = mock.patch.dict('os.environ', {'FRAME_CONTROL_MAILING_LIST_URL':
-                              f'http://127.0.0.1:{server.server_port}/api/mailing-list'})
-        env.start()
-        self.addCleanup(env.stop)
-
-    def test_only_email_consent_and_retry_credentials_leave_the_app(self):
-        fm.submit('a' * 64, 7, 'me@example.com')
-        fm.submit('a' * 64, 8, '')
-        self.assertEqual(self.got, [
-            ('/api/mailing-list', {'token': 'a' * 64, 'rev': 7, 'email': 'me@example.com', 'updates': True}),
-            ('/api/mailing-list', {'token': 'a' * 64, 'rev': 8, 'email': '', 'updates': False}),
-        ])
-
-    def test_unavailable_unconfigured_and_invalid_responses_are_retryable(self):
-        for status, answer in ((503, b'{}'), (429, b'{}'), (200, b'not json'),
-                               (200, b'[]'), (200, b'{"ok":false}')):
-            self.status, self.answer = status, answer
-            with self.subTest(status=status, answer=answer), self.assertRaises(fm.SendError):
-                fm.submit('a' * 64, 1, 'me@example.com')
-
-    def test_timeout_is_bounded_and_retryable(self):
-        with mock.patch.object(fm.urllib.request, 'urlopen', side_effect=TimeoutError) as send:
-            with self.assertRaises(fm.SendError):
-                fm.submit('a' * 64, 1, 'me@example.com')
-        self.assertEqual(send.call_args.kwargs['timeout'], 10)
-
-    def test_real_contact_worker_delivers_join_and_removal(self):
-        # Isolate the daemon worker in a short-lived process so it cannot outlive
-        # a test's temporary state or mocks. Only the local mailing server is real.
-        with tempfile.TemporaryDirectory() as state:
-            run = subprocess.run([sys.executable, '-c', '''
-import time
-import frame_contact as fc
-fc.frame_telemetry.post = lambda *args, **kwargs: None
-fc.start()
-for choice in ({'email': 'me@example.com', 'updates': True}, {'email': ''}):
-    fc.save(choice)
-    deadline = time.monotonic() + 5
-    while fc.state()['waiting'] and time.monotonic() < deadline:
-        time.sleep(.01)
-    assert not fc.state()['waiting'], 'worker did not drain queue'
-'''], cwd=Path(__file__).resolve().parent.parent / 'ui',
-                env={**os.environ, 'FRAME_CONTROL_DATA_DIR': state},
-                capture_output=True, text=True, timeout=15)
-        self.assertEqual(run.returncode, 0, run.stderr)
-        self.assertEqual([body['email'] for _, body in self.got], ['me@example.com', ''])
-        self.assertEqual([body['rev'] for _, body in self.got], [1, 2])

tests/test_source_logins.py

@@ -0,0 +1,566 @@
+"""Repositories and sources behind a login: F-Droid basic auth and GitHub tokens.
+
+A fake HTTPS layer stands in for the network (urllib's real redirect, error and
+request processing still run), so these tests see every request and its headers.
+"""
+import sandbox  # noqa: F401  (first: keeps tests off real data and services)
+import base64
+import contextlib
+import http.client
+import io
+import json
+import os
+from pathlib import Path
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from unittest.mock import Mock, patch
+import urllib.request
+import urllib.response
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui'))
+from apk_sources import SourceAuthError, SourceError, SourceLimited, _credentials, _images, _web, fdroid, github, search
+import server
+
+FIXTURES = Path(__file__).parent / 'fixtures' / 'fdroid'
+PIN = (FIXTURES / 'fingerprint.txt').read_text().strip()
+REPO = 'https://repo.example/fdroid/repo/'
+USER, PASSWORD = 'alice', 's3cr@t-pässword'
+TOKEN = 'ghp_frameControlTestToken0123456789'
+SECRETS = (PASSWORD, TOKEN, base64.b64encode((USER + ':' + PASSWORD).encode()).decode())
+REAL_BUILD_OPENER = urllib.request.build_opener
+
+
+def basic(user, password):
+    return 'Basic ' + base64.b64encode((user + ':' + password).encode()).decode()
+
+
+class FakeHTTPS(urllib.request.HTTPSHandler):
+    """Answers https requests from route(url, authorization) -> (status, headers, body)."""
+
+    def __init__(self, route):
+        super().__init__()
+        self.route, self.seen = route, []
+
+    def https_open(self, req):
+        auth = req.get_header('Authorization')
+        self.seen.append((req.full_url, auth))
+        status, headers, body = self.route(req.full_url, auth)
+        message = http.client.HTTPMessage()
+        for key, value in headers.items():
+            message[key] = value
+        response = urllib.response.addinfourl(io.BytesIO(body), message, req.full_url, status)
+        response.msg = 'fake'
+        return response
+
+
+class Base(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.root = Path(tmp.name)
+        for name, value in [('data_dir', lambda *p: self.root.joinpath('data', *p)),
+                            ('cache_dir', lambda *p: self.root.joinpath('cache', *p))]:
+            mock = patch.object(fdroid.frame_host, name, value)
+            mock.start()
+            self.addCleanup(mock.stop)
+        self.web = FakeHTTPS(self.route)
+        mock = patch('urllib.request.build_opener', lambda *handlers: REAL_BUILD_OPENER(*handlers, self.web))
+        mock.start()
+        self.addCleanup(mock.stop)
+        mock = patch.dict(os.environ, {github.ENV: ''})
+        mock.start()
+        self.addCleanup(mock.stop)
+        for state in (_web._limited, fdroid._stale, fdroid._retry_at, fdroid._refreshing, github._rejected):
+            state.clear()
+            self.addCleanup(state.clear)
+        self.stderr = io.StringIO()
+        mock = patch('sys.stderr', self.stderr)
+        mock.start()
+        self.addCleanup(mock.stop)
+
+    def route(self, url, auth):
+        raise AssertionError('unexpected request ' + url)
+
+    def assertNoSecrets(self, *texts):
+        for text in texts:
+            for secret in SECRETS + (USER + ':',):
+                self.assertNotIn(secret, text)
+
+    def disk(self):
+        """Everything saved under the data and cache folders except the credentials file."""
+        out = []
+        for path in self.root.rglob('*'):
+            if path.is_file() and path.name != 'source-credentials.json':
+                out.append(path.read_bytes().decode('utf-8', 'replace'))
+        return '\n'.join(out)
+
+    def credentials_file(self):
+        return self.root / 'data' / 'source-credentials.json'
+
+
+class Repository(Base):
+    """A repository at REPO that wants USER and PASSWORD."""
+    def setUp(self):
+        super().setUp()
+        self.login = basic(USER, PASSWORD)
+        self.redirect = None  # file name -> absolute URL it redirects to
+        self.forbidden = 401
+
+    def serve(self, name):
+        return (FIXTURES / ('example.apk' if name.endswith('.apk') else name)).read_bytes()
+
+    def route(self, url, auth):
+        for mirror in ('https://mirror.example/fdroid/repo/', 'https://repo.example/elsewhere/'):
+            if url.startswith(mirror):  # serves anyone; a login must never get here unless same-origin
+                return 200, {}, self.serve(url[len(mirror):])
+        self.assertTrue(url.startswith(REPO), url)
+        if auth != self.login:
+            return self.forbidden, {'WWW-Authenticate': 'Basic realm="repo"'}, b'no'
+        name = url[len(REPO):]
+        if self.redirect and name in self.redirect:
+            return 302, {'Location': self.redirect[name]}, b''
+        return 200, {}, self.serve(name)
+
+    def add(self, **kw):
+        return fdroid.add_repo('https://alice:s3cr%40t-p%C3%A4ssword@repo.example/fdroid/repo', PIN, **kw)
+
+
+class RepositoryLogin(Repository):
+    def test_add_search_download_with_login_in_the_url(self):
+        source = self.add()
+        self.assertEqual(source['url'], REPO)
+        self.assertEqual(source['fingerprint'], PIN)
+        saved = fdroid.user_repos()[0]
+        self.assertEqual(saved['url'], REPO)
+        self.assertNotIn('login', saved)
+        self.assertEqual(len(fdroid.search(source, 'example offline')), 1)
+        downloaded = fdroid.download(source, 'org.example.app', 1)
+        self.assertTrue(downloaded['verified'])
+        self.assertTrue(self.web.seen)
+        self.assertTrue(all(auth == self.login for url, auth in self.web.seen))
+        self.assertTrue(all('@' not in url for url, auth in self.web.seen))  # never in the URL
+        self.assertTrue(next(s for s in fdroid.sources() if s['id'] == source['id'])['login'])
+        self.assertNoSecrets(json.dumps(source), json.dumps(fdroid.sources()), self.disk(), self.stderr.getvalue())
+        # The settings file holds no part of the login; the credentials file holds it, privately.
+        settings = (self.root / 'data' / 'apk-repos.json').read_text()
+        self.assertNotIn(USER, settings)
+        self.assertTrue(self.credentials_file().is_file())
+        if os.name != 'nt':
+            self.assertEqual(stat.S_IMODE(self.credentials_file().stat().st_mode), 0o600)
+
+    def test_login_given_apart_and_saved_login_reused_on_refresh(self):
+        source = fdroid.add_repo(REPO, PIN, username=USER, password=PASSWORD)
+        self.assertEqual(source['url'], REPO)
+        self.web.seen.clear()
+        fdroid._load(fdroid.user_repos()[0], force=True)  # as a background refresh would
+        self.assertTrue(self.web.seen)
+        self.assertTrue(all(auth == self.login for url, auth in self.web.seen))
+        fdroid.add_repo(REPO)  # re-adding without a login keeps the saved one
+        self.assertTrue(_credentials.has(source['id']))
+
+    def test_wrong_or_missing_login_saves_nothing(self):
+        for kw, message in [({'username': USER, 'password': 'wrong'}, 'check the username and password'),
+                            ({}, 'needs a username and password')]:
+            with self.subTest(kw=kw):
+                with self.assertRaisesRegex(SourceAuthError, message) as caught:
+                    fdroid.add_repo(REPO, PIN, **kw)
+                self.assertNoSecrets(str(caught.exception), repr(caught.exception))
+                self.assertNotIn('wrong', str(caught.exception))
+                self.assertEqual(fdroid.user_repos(), [])
+                self.assertFalse(self.credentials_file().exists())
+                self.assertFalse((self.root / 'data' / 'apk-repo-state.json').exists())
+                self.assertEqual(_web._limited, {})  # an auth failure isn't a rate limit
+
+    def test_403_with_a_login_is_a_login_error_not_a_rate_limit(self):
+        self.forbidden = 403
+        with self.assertRaisesRegex(SourceAuthError, 'check the username and password'):
+            fdroid.add_repo(REPO, PIN, username=USER, password='wrong')
+        self.assertEqual(_web._limited, {})
+        self.assertEqual(fdroid.user_repos(), [])
+
+    def test_signature_checks_still_apply(self):
+        with self.assertRaisesRegex(SourceError, 'fingerprint mismatch'):
+            fdroid.add_repo(REPO, '0' * 64, username=USER, password=PASSWORD)
+        self.assertEqual(fdroid.user_repos(), [])
+        self.assertFalse(self.credentials_file().exists())
+
+    def test_cross_origin_redirect_never_gets_the_login(self):
+        self.redirect = {'entry.jar': 'https://mirror.example/fdroid/repo/entry.jar',
+                         'index-v2.json': 'https://repo.example/elsewhere/index-v2.json'}
+        source = self.add()
+        self.assertEqual(source['fingerprint'], PIN)  # the mirror's copy still had to verify
+        mirror = [auth for url, auth in self.web.seen if url.startswith('https://mirror.example/')]
+        same_origin = [auth for url, auth in self.web.seen if url.startswith('https://repo.example/elsewhere/')]
+        self.assertEqual(mirror, [None])
+        self.assertEqual(same_origin, [None])  # same origin but outside the repository: no login either
+
+    def test_traversal_names_are_refused_and_never_get_the_login(self):
+        for name in ('%252e%252e/%252e%252e/evil/collect.apk', '..;/..;/evil/collect.apk', 'a%2f..%2f..%2fevil.apk',
+                     'a%5c..%5cevil.apk', '%2e%2e/evil.apk', 'x;y.apk', '100%25.apk'):
+            with self.subTest(name=name), self.assertRaisesRegex(SourceError, 'unsafe|outside'):
+                fdroid._child(REPO, name)
+        self.assertEqual(fdroid._child(REPO, '/app/en-US/phoneScreenshots/#0 a.png'),
+                         REPO + 'app/en-US/phoneScreenshots/%230%20a.png')
+        # Even if such a URL reached the downloader directly, the login stays home.
+        login = _credentials.basic(REPO, USER, PASSWORD)
+        for url in (REPO + '%252e%252e/%252e%252e/evil/collect.apk', REPO + '..;/..;/evil/collect.apk'):
+            with self.subTest(url=url), tempfile.TemporaryDirectory() as tmp:
+                self.web.seen.clear()
+                with contextlib.suppress(SourceError):  # the repository answers 401 without the login
+                    fdroid._fetch(url, Path(tmp) / 'out', 1 << 20, auth=login)
+                self.assertEqual(self.web.seen, [(url, None)])
+
+    def test_same_origin_redirect_outside_the_repository_gets_no_login(self):
+        self.redirect = {'example1.apk': 'https://repo.example/elsewhere/example1.apk'}
+        source = self.add()
+        self.web.seen.clear()
+        self.assertTrue(fdroid.download(source, 'org.example.app', 1)['verified'])
+        self.assertEqual(self.web.seen, [(REPO + 'example1.apk', self.login),
+                                         ('https://repo.example/elsewhere/example1.apk', None)])
+
+    def test_other_ports_and_http_never_get_the_login(self):
+        login = _credentials.basic(REPO, USER, PASSWORD)
+        for url in ('https://repo.example:8443/fdroid/repo/x', 'http://repo.example/fdroid/repo/x',
+                    'https://other.example/fdroid/repo/x', 'https://repo.example.evil/x'):
+            self.assertFalse(login.matches(url), url)
+        self.assertTrue(login.matches('https://REPO.example:443/anything'))
+        self.assertNoSecrets(repr(login))
+        handler = _credentials.Handler(login)
+        req = urllib.request.Request('https://other.example/x', headers={'Authorization': 'Basic stale'})
+        self.assertIsNone(handler.https_request(req).get_header('Authorization'))
+
+    def test_http_and_bad_logins_are_refused_before_any_request(self):
+        for url, kw in [('http://alice:pw@repo.example/fdroid/repo', {}),
+                        ('http://repo.example/fdroid/repo', {'username': USER, 'password': PASSWORD}),
+                        (REPO, {'username': 'a:b', 'password': PASSWORD}),
+                        (REPO, {'username': USER, 'password': 'x\r\nX-Evil: 1'}),
+                        (REPO, {'password': PASSWORD}),
+                        ('https://alice:pw@repo.example/fdroid/repo', {'username': 'bob'})]:
+            with self.subTest(url=url, kw=kw), self.assertRaises(SourceError) as caught:
+                fdroid.add_repo(url, PIN, **kw)
+            self.assertNotIn('pw', str(caught.exception).replace('password', ''))
+            self.assertNoSecrets(str(caught.exception))
+        self.assertEqual(self.web.seen, [])
+
+    def test_remove_deletes_the_login(self):
+        source = self.add()
+        other = _credentials.bearer('https://api.github.com/', TOKEN)
+        _credentials.put('github', other)
+        fdroid.remove_repo(source['id'])
+        self.assertFalse(_credentials.has(source['id']))
+        self.assertNotIn(PASSWORD, self.credentials_file().read_text())
+        _credentials.remove('github')
+        self.assertFalse(self.credentials_file().exists())  # nothing left: the file goes
+
+    def test_cli_reads_the_password_from_stdin(self):
+        out = io.StringIO()
+        argv = ['fdroid.py', 'add', REPO, '--fingerprint', PIN, '--username', USER, '--password-stdin']
+        with patch.object(sys, 'argv', argv), patch.object(sys, 'stdin', io.StringIO(PASSWORD + '\n')), \
+                contextlib.redirect_stdout(out):
+            fdroid.main()
+        self.assertEqual(json.loads(out.getvalue())['url'], REPO)
+        self.assertNoSecrets(out.getvalue())
+        self.assertTrue(_credentials.has(fdroid.user_repos()[0]['id']))
+
+    def test_artwork_from_the_repository_gets_the_login_and_only_there(self):
+        source = self.add()
+        self.assertEqual(_credentials.for_url(REPO + 'icons/a.png').header(), self.login)
+        self.assertTrue(_credentials.has(source['id']))
+        self.assertIsNone(_credentials.for_url('https://repo.example/other/a.png'))  # outside the repository
+        self.assertIsNone(_credentials.for_url('https://cdn.example/fdroid/repo/a.png'))
+
+
+class GitHubToken(Base):
+    def setUp(self):
+        super().setUp()
+        self.accept = {TOKEN}
+        self.releases = json.dumps([]).encode()
+        self.anonymous_limited = False
+
+    def route(self, url, auth):
+        if url.startswith('https://api.github.com/'):
+            if auth is not None and auth not in {'Bearer ' + t for t in self.accept}:
+                return 401, {}, b'{"message": "Bad credentials"}'
+            if auth is None and self.anonymous_limited:
+                return 403, {'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': str(int(time.time()) + 1800)}, b'{}'
+            if url.endswith('/redirected'):
+                return 302, {'Location': 'https://github.com/landed'}, b''
+            return 200, {}, self.releases
+        if url == 'https://github.com/landed':
+            return 200, {}, b'{"ok": true}'
+        raise AssertionError('unexpected request ' + url)
+
+    def test_saved_token_goes_only_to_the_api(self):
+        github.set_token(TOKEN)
+        self.assertEqual(github._api('/repos/a/b/releases'), [])
+        self.assertEqual(self.web.seen, [('https://api.github.com/repos/a/b/releases', 'Bearer ' + TOKEN)])
+        auth = github._credential(github._token()[0])
+        self.assertEqual(_web.read('https://api.github.com/redirected', ('api.github.com', 'github.com'), auth=auth),
+                         b'{"ok": true}')
+        self.assertEqual(self.web.seen[-2:], [('https://api.github.com/redirected', 'Bearer ' + TOKEN),
+                                              ('https://github.com/landed', None)])  # redirect: no token
+        status = github.sources()[0]['token']
+        self.assertEqual(status, {'set': True, 'from': 'saved', 'rejected': False})
+        self.assertNoSecrets(json.dumps(github.sources()), self.disk(), self.stderr.getvalue())
+        if os.name != 'nt':
+            self.assertEqual(stat.S_IMODE(self.credentials_file().stat().st_mode), 0o600)
+
+    def test_rejected_token_retries_once_without_it(self):
+        github.set_token(TOKEN)
+        self.accept = set()
+        self.assertEqual(github._api('/repos/a/b/releases'), [])
+        self.assertEqual([auth for _, auth in self.web.seen], ['Bearer ' + TOKEN, None])
+        self.assertTrue(github.token_status()['rejected'])
+        self.assertIn('rejected', self.stderr.getvalue())
+        self.assertNoSecrets(self.stderr.getvalue())
+        self.web.seen.clear()
+        github._api('/repos/a/b/other')
+        self.assertEqual([auth for _, auth in self.web.seen], [None])  # not tried again this run
+        github.set_token(TOKEN)  # saving it again gives it a fresh chance
+        self.assertFalse(github.token_status()['rejected'])
+
+    def test_environment_variable_wins(self):
+        github.set_token(TOKEN)
+        env = 'ghp_fromTheEnvironment'
+        self.accept = {env}
+        with patch.dict(os.environ, {github.ENV: env}):
+            github._api('/repos/a/b/releases')
+            self.assertEqual(self.web.seen[-1][1], 'Bearer ' + env)
+            self.assertEqual(github.token_status(), {'set': True, 'from': 'env', 'rejected': False})
+        self.assertEqual(github.token_status()['from'], 'saved')
+
+    def test_clear_removes_it_from_disk(self):
+        github.set_token(TOKEN)
+        github.clear_token()
+        self.assertFalse(self.credentials_file().exists())
+        self.assertEqual(github.token_status(), {'set': False, 'from': None, 'rejected': False})
+        github._api('/repos/a/b/releases')
+        self.assertEqual(self.web.seen[-1][1], None)
+
+    def test_saving_a_token_escapes_the_anonymous_rate_limit(self):
+        self.anonymous_limited = True
+        with self.assertRaisesRegex(SourceLimited, 'GitHub is limiting requests'):
+            github._api('/repos/a/b/releases')
+        self.assertEqual(self.web.seen[-1][1], None)
+        github.set_token(TOKEN)
+        self.assertEqual(github._api('/repos/a/b/other'), [])  # tried, with the token's own quota
+        self.assertEqual(self.web.seen[-1], ('https://api.github.com/repos/a/b/other', 'Bearer ' + TOKEN))
+        github.clear_token()
+        count = len(self.web.seen)
+        with self.assertRaisesRegex(SourceLimited, 'GitHub is limiting requests'):
+            github._api('/repos/a/b/third')
+        self.assertEqual(len(self.web.seen), count)  # anonymous is still backing off: no request
+
+    def test_bad_tokens_are_refused(self):
+        for token in ('', '   ', 'two words', 'x\nAuthorization: y'):
+            with self.subTest(token=token), self.assertRaises(SourceError):
+                github.set_token(token)
+        self.assertFalse(self.credentials_file().exists())
+
+
+PNG = b'\x89PNG\r\n\x1a\nfixture'
+PRIVATE = 'https://repo.example/private/repo/'
+TRAVERSALS = [PRIVATE + '%2e%2e/%2e%2e/evil/collect.png', PRIVATE + '%2E%2E/x.png', PRIVATE + '../../evil/x.png',
+              PRIVATE + './icons/a.png', PRIVATE + '..%2f..%2fevil/x.png', PRIVATE + 'a%5c..%5c..%5cevil.png',
+              PRIVATE + 'a;b/../../evil.png', PRIVATE + '%252e%252e/%252e%252e/evil.png',
+              'https://repo.example/private/repository/x.png', 'https://repo.example/private/repo%2fx.png']
+
+
+class ArtworkScope(Base):
+    """The artwork proxy: actual outgoing headers, through _images.get's own HTTP code."""
+
+    def setUp(self):
+        super().setUp()
+        self.sent, self.redirects = [], {}
+        _credentials.put('fdroid-user-private', _credentials.basic(PRIVATE, USER, PASSWORD))
+        test = self
+
+        class Response:
+            def __init__(self, status, location=''):
+                self.status, self.location, self.body = status, location, PNG if status == 200 else b''
+
+            def getheader(self, name, default=''):
+                return self.location if name == 'Location' else default
+
+            def read1(self, n):
+                chunk, self.body = self.body[:n], self.body[n:]
+                return chunk
+
+        class Connection:
+            def __init__(self, host, port, timeout=None):
+                self.host, self.port = host, port
+
+            def request(self, method, path, headers):
+                url = 'https://%s%s' % (self.host, path)
+                test.sent.append((url, headers.get('Authorization')))
+                self.url = url
+
+            def getresponse(self):
+                if self.url in test.redirects:
+                    return Response(302, test.redirects[self.url])
+                return Response(200)
+
+            def close(self):
+                pass
+
+        # A public address, no real socket or TLS, and Connection in place of http.client's.
+        for target, name, value in [(_images, '_resolve', lambda host, port, timeout: [(2, 1, 6, '', ('93.184.216.34', port))]),
+                                    (_images.socket, 'create_connection', lambda *a, **k: Mock()),
+                                    (_images.ssl, 'create_default_context', lambda: Mock()),
+                                    (_images.http.client, 'HTTPConnection', Connection)]:
+            mock = patch.object(target, name, value)
+            mock.start()
+            self.addCleanup(mock.stop)
+
+    def test_artwork_in_the_repository_gets_the_login(self):
+        self.assertEqual(_images.get(PRIVATE + 'icons/a%20b.png'), PNG)
+        self.assertEqual(self.sent, [(PRIVATE + 'icons/a%20b.png', basic(USER, PASSWORD))])
+
+    def test_paths_a_server_could_normalise_outside_the_repository_get_nothing(self):
+        for url in TRAVERSALS:
+            with self.subTest(url=url):
+                self.sent.clear()
+                self.assertEqual(_images.get(url), PNG)
+                self.assertEqual(self.sent, [(url, None)])
+                self.assertIsNone(_credentials.for_url(url))
+
+    def test_redirect_to_a_traversal_gets_nothing(self):
+        start = PRIVATE + 'icons/a.png'
+        self.redirects[start] = PRIVATE + '%2e%2e/%2e%2e/evil/collect.png'
+        self.assertEqual(_images.get(start), PNG)
+        self.assertEqual(self.sent, [(start, basic(USER, PASSWORD)), (self.redirects[start], None)])
+
+
+class ScopedHandler(Base):
+    """Handler(scoped=True), for callers (artwork-like) that must stay inside a scope, redirects included."""
+
+    def route(self, url, auth):
+        if url == PRIVATE + 'icons/a.png':
+            return 302, {'Location': PRIVATE + '%2e%2e/%2e%2e/evil/collect.png'}, b''
+        return 200, {}, PNG
+
+    def test_redirect_into_a_traversal_is_stripped(self):
+        login = _credentials.basic(PRIVATE, USER, PASSWORD)
+        opener = urllib.request.build_opener(_credentials.Handler(login, scoped=True))
+        with opener.open(PRIVATE + 'icons/a.png') as r:
+            self.assertEqual(r.read(), PNG)
+        self.assertEqual(self.web.seen, [(PRIVATE + 'icons/a.png', basic(USER, PASSWORD)),
+                                         (PRIVATE + '%2e%2e/%2e%2e/evil/collect.png', None)])
+        for url in TRAVERSALS:
+            self.assertFalse(_credentials.in_scope(login, url), url)
+        self.assertTrue(_credentials.in_scope(login, PRIVATE + 'a/b%20c.png?x=1'))
+        self.assertTrue(_credentials.in_scope(login, PRIVATE.rstrip('/')))
+
+
+CHURN = '''
+import sys
+sys.path.insert(0, sys.argv[1])
+from apk_sources import _credentials
+for i in range(int(sys.argv[3])):
+    key = sys.argv[2] + str(i)
+    _credentials.put(key, _credentials.basic('https://h.example/', 'u', 'p'))
+    if i % 2:
+        _credentials.remove(key)
+'''
+
+
+class TwoProcesses(Base):
+    def test_interleaved_saves_and_removals_are_not_lost(self):
+        ui = str(Path(__file__).resolve().parents[1] / 'ui')
+        env = dict(os.environ, FRAME_CONTROL_DATA_DIR=str(self.root / 'data'))
+        count = 60
+        procs = [subprocess.Popen([sys.executable, '-c', CHURN, ui, prefix, str(count)], env=env,
+                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE) for prefix in ('a', 'b')]
+        for p in procs:
+            out, err = p.communicate(timeout=120)
+            self.assertEqual(p.returncode, 0, err.decode())
+        saved = set(json.loads(self.credentials_file().read_text()))
+        self.assertEqual(saved, {p + str(i) for p in 'ab' for i in range(0, count, 2)})
+        if os.name != 'nt':
+            self.assertEqual(stat.S_IMODE(self.credentials_file().stat().st_mode), 0o600)
+
+
+@unittest.skipUnless(shutil.which('node'), 'needs node')
+class PageLogin(unittest.TestCase):
+    def test_login_leaves_the_address_field_before_sending(self):
+        """tests/page/source_login.mjs runs the real page functions from index.html."""
+        r = subprocess.run(['node', str(Path(__file__).parent / 'page' / 'source_login.mjs')],
+                           capture_output=True, text=True)
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        self.assertIn('ok', r.stdout)
+
+
+class Endpoints(Repository):
+    """The server API: logins in, never out, in responses or the server log."""
+
+    def setUp(self):
+        super().setUp()
+        mock = patch.object(search, 'modules', return_value=([fdroid, github], []))
+        mock.start()
+        self.addCleanup(mock.stop)
+        self.httpd = server.ThreadingHTTPServer(('127.0.0.1', 0), server.Handler)
+        threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
+        self.addCleanup(self.httpd.server_close)
+        self.addCleanup(self.httpd.shutdown)
+
+    def request(self, method, path, body=None):
+        c = http.client.HTTPConnection('127.0.0.1', self.httpd.server_port)
+        c.request(method, path, json.dumps(body) if body is not None else None,
+                  {'X-Frame-UI': '1', 'Content-Type': 'application/json'})
+        r = c.getresponse()
+        raw = r.read().decode()
+        c.close()
+        self.replies.append(raw)
+        return r.status, json.loads(raw)
+
+    def job(self, reply):
+        for _ in range(500):
+            job = self.request('GET', '/api/job?id=' + reply['job'])[1]
+            if job['done']:
+                return job
+            time.sleep(.01)
+        self.fail('job did not finish')
+
+    def test_add_with_login_and_token_settings(self):
+        self.replies = []
+        bad = self.job(self.request('POST', '/api/sources', {'action': 'add', 'url': REPO, 'fingerprint': PIN,
+                                                             'username': USER, 'password': 'wrong'})[1])
+        self.assertIn('check the username and password', bad['error'])
+        self.assertEqual(fdroid.user_repos(), [])
+        code, reply = self.request('POST', '/api/sources', {'action': 'add', 'url': REPO, 'fingerprint': PIN,
+                                                            'username': USER, 'password': PASSWORD})
+        self.assertEqual(code, 200)
+        job = self.job(reply)
+        self.assertIsNone(job['error'])
+        listed = self.request('GET', '/api/sources')[1]['sources']
+        repo = next(s for s in listed if s['id'] == job['result']['source']['id'])
+        self.assertEqual((repo['url'], repo['login']), (REPO, True))
+        self.assertEqual(self.request('POST', '/api/sources', {'action': 'set-token', 'source': 'github',
+                                                               'token': TOKEN})[0], 200)
+        github_row = next(s for s in self.request('GET', '/api/sources')[1]['sources'] if s['id'] == 'github')
+        self.assertEqual(github_row['token'], {'set': True, 'from': 'saved', 'rejected': False})
+        self.assertEqual(self.request('POST', '/api/sources', {'action': 'set-token', 'source': 'fdroid',
+                                                               'token': TOKEN})[0], 400)
+        self.assertEqual(self.request('POST', '/api/sources', {'action': 'clear-token', 'source': 'github'})[0], 200)
+        self.assertFalse(_credentials.has('github'))
+        self.assertEqual(self.request('POST', '/api/sources', {'action': 'remove', 'source': repo['id']})[0], 200)
+        self.assertFalse(self.credentials_file().exists())
+        self.assertNoSecrets(*self.replies, self.stderr.getvalue(), self.disk())
+
+    def test_login_in_the_url_is_accepted(self):
+        self.replies = []
+        url = 'https://alice:s3cr%40t-p%C3%A4ssword@repo.example/fdroid/repo'
+        job = self.job(self.request('POST', '/api/sources', {'action': 'add', 'url': url, 'fingerprint': PIN})[1])
+        self.assertIsNone(job['error'])
+        self.assertEqual(fdroid.user_repos()[0]['url'], REPO)
+        self.assertNoSecrets(*self.replies, self.stderr.getvalue())
+
+
+if __name__ == '__main__':
+    unittest.main()

ui/apk_sources/__init__.py

@@ -45,6 +45,10 @@ class SourceError(Exception):
     """User-readable failure from a source (network, format, verification)."""
 
 
+class SourceAuthError(SourceError):
+    """The server refused our login (HTTP 401, or 403 when a login was sent)."""
+
+
 class SourceLimited(SourceError):
     """The source's host asked us to slow down; retry_after is in seconds."""
 

ui/apk_sources/_credentials.py

@@ -0,0 +1,270 @@
+"""Logins for sources behind one: HTTP basic auth and access tokens.
+
+Saved apart from the source settings, in frame_host.data_dir('source-credentials.json')
+(mode 0600), keyed by a caller-chosen name: an F-Droid source id, 'github'. Each
+credential belongs to one exact origin (https, host, port) and is only ever sent
+to a request for that origin: Handler adds it to each request, redirects
+included, and removes any Authorization header from every other request.
+
+Scoped use (for_url, and Handler(..., scoped=True), which F-Droid repositories and
+artwork use) is stricter: the URL must also be under the credential's scope by whole
+path segments, with no dot segments, encoded slashes or ';' a server could normalise
+into a path outside it. Origin-only matching suits a host-wide token (GitHub's API).
+
+Secrets leave this module only as an Authorization header value. Credential's
+repr hides them, and nothing here puts them in an exception message.
+
+Each change is a read-modify-write under an inter-process lock (the CLI and the
+app may both save), so one process can't undo another's save or removal.
+"""
+import base64
+import contextlib
+import errno
+import json
+import os
+import tempfile
+import threading
+import urllib.parse
+import urllib.request
+
+import frame_host
+
+_lock = threading.RLock()
+MAX = 4096  # longest username, password or token accepted
+
+
+def _path():
+    return frame_host.data_dir('source-credentials.json')
+
+
+@contextlib.contextmanager
+def _changing():
+    """This thread, then this process, holds the credentials file for a read-modify-write."""
+    path = _path().with_name('source-credentials.lock')
+    with _lock:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(str(path), 'a+b') as f:
+            if os.name == 'nt':
+                import msvcrt
+                while True:
+                    try:
+                        f.seek(0)
+                        msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1)
+                        break
+                    except OSError as e:  # LK_LOCK gives up after ~10 s of contention; keep waiting
+                        if e.errno not in (errno.EACCES, errno.EDEADLK):
+                            raise
+                try:
+                    yield
+                finally:
+                    f.seek(0)
+                    msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                import fcntl
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                try:
+                    yield
+                finally:
+                    fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+
+
+def path_segments(path):
+    """A URL path's decoded segments, or None if a server might read it as another path:
+    '.' or '..' (plain or percent-encoded), encoded '/' or '\\', '%' left after decoding
+    (double encoding), or ';' path parameters."""
+    if ';' in path or '\\' in path:
+        return None
+    parts = [urllib.parse.unquote(part) for part in path.split('/')]
+    if any(part in ('.', '..') or '/' in part or '\\' in part or '%' in part or ';' in part for part in parts):
+        return None
+    return parts
+
+
+def in_scope(credential, url):
+    """True if url is on credential's origin and under its scope by whole, plain path segments."""
+    if credential is None or not credential.matches(url):
+        return False
+    try:
+        path, scope = urllib.parse.urlsplit(url).path, urllib.parse.urlsplit(credential.scope).path
+    except ValueError:
+        return False
+    parts, base = path_segments(path), path_segments(scope)
+    if parts is None or base is None:
+        return False
+    if base and base[-1] == '':
+        base = base[:-1]  # 'https://h/repo/' covers '/repo/x', not '/repository'
+    return parts[:len(base)] == base
+
+
+def origin(url):
+    """'https://host:port' for an https URL; ValueError for anything else."""
+    p = urllib.parse.urlsplit(url)
+    port = p.port  # raises ValueError on a bad port
+    if p.scheme.lower() != 'https' or not p.hostname:
+        raise ValueError('credentials are only sent over HTTPS')
+    host = p.hostname.lower()
+    return 'https://%s:%d' % ('[%s]' % host if ':' in host else host, port or 443)
+
+
+def _check(value, what, allow_empty=False):
+    if not isinstance(value, str) or (not value and not allow_empty) or len(value) > MAX:
+        raise ValueError('invalid ' + what)
+    if any(ord(c) < 32 or ord(c) == 127 for c in value):
+        raise ValueError(what + ' must not contain control characters')
+    return value
+
+
+class Credential:
+    """A username and password (basic) or a token (bearer) for one origin.
+
+    scope is the URL prefix it belongs under: in_scope checks it (by path segments),
+    matches() checks only the origin."""
+    __slots__ = ('origin', 'scope', 'username', '_secret')
+
+    def __init__(self, scope, secret, username=None):
+        self.origin = origin(scope)
+        self.scope = scope
+        if username is not None:
+            _check(username, 'username')
+            if ':' in username:
+                raise ValueError('username must not contain ":"')
+            _check(secret, 'password', allow_empty=True)
+        else:
+            _check(secret, 'token')
+            if any(c.isspace() for c in secret):
+                raise ValueError('token must not contain spaces')
+        self.username, self._secret = username, secret
+
+    @property
+    def kind(self):
+        return 'bearer' if self.username is None else 'basic'
+
+    def header(self):
+        if self.username is None:
+            return 'Bearer ' + self._secret
+        return 'Basic ' + base64.b64encode((self.username + ':' + self._secret).encode('utf-8')).decode('ascii')
+
+    def matches(self, url):
+        """True only for a URL on exactly this credential's origin."""
+        try:
+            return origin(url) == self.origin
+        except ValueError:
+            return False
+
+    def same(self, other):
+        return isinstance(other, Credential) and (self.origin, self.username, self._secret) == (
+            other.origin, other.username, other._secret)
+
+    def __repr__(self):
+        return '<Credential %s for %s>' % (self.kind, self.origin)
+
+    def _json(self):
+        return {'scope': self.scope, 'username': self.username, 'secret': self._secret}
+
+
+def basic(scope, username, password):
+    """Basic auth for scope's origin, e.g. basic('https://example.org/fdroid/repo/', 'me', 'pw')."""
+    return Credential(scope, password, username)
+
+
+def bearer(scope, token):
+    """An access token for scope's origin, sent as 'Authorization: Bearer <token>'."""
+    return Credential(scope, token)
+
+
+def _read():
+    try:
+        with open(str(_path()), encoding='utf-8') as f:
+            data = json.load(f)
+        if not isinstance(data, dict):
+            raise ValueError
+        return data
+    except FileNotFoundError:
+        return {}
+    except (OSError, ValueError):
+        return {}  # unreadable: as if none were saved; the next save rewrites it
+
+
+def _save(data):
+    path = _path()
+    if not data:
+        try:
+            os.remove(str(path))
+        except FileNotFoundError:
+            pass
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix='.credentials-', suffix='.part')  # created 0600
+    try:
+        with os.fdopen(fd, 'w', encoding='utf-8') as f:
+            json.dump(data, f, separators=(',', ':'))
+        os.chmod(tmp, 0o600)
+        os.replace(tmp, str(path))
+    finally:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+
+
+def get(key):
+    """The saved Credential for key, or None."""
+    with _lock:
+        item = _read().get(key)
+    try:
+        return Credential(item['scope'], item['secret'], item.get('username'))
+    except (TypeError, KeyError, ValueError):
+        return None
+
+
+def has(key):
+    return get(key) is not None
+
+
+def put(key, credential):
+    if not isinstance(credential, Credential):
+        raise TypeError('put() needs a Credential')
+    with _changing():
+        data = _read()
+        data[key] = credential._json()
+        _save(data)
+
+
+def remove(key):
+    """Delete key's credential from disk; True if there was one."""
+    with _changing():
+        data = _read()
+        if key not in data:
+            return False
+        del data[key]
+        _save(data)
+        return True
+
+
+def for_url(url):
+    """A saved credential url is in the scope of (see in_scope), or None. Call it again for
+    each redirect hop: the answer depends only on the URL being requested."""
+    with _lock:
+        keys = list(_read())
+    for key in keys:
+        credential = get(key)
+        if in_scope(credential, url):
+            return credential
+    return None
+
+
+class Handler(urllib.request.BaseHandler):
+    """Adds the credential to requests for its origin (redirects included) and strips
+    Authorization from every other request, so a redirect elsewhere never gets it.
+    scoped=True also requires each request, redirects included, to be in_scope."""
+    handler_order = 900  # after the default processors
+
+    def __init__(self, credential, scoped=False):
+        self.credential, self.scoped = credential, scoped
+
+    def _request(self, req):
+        req.remove_header('Authorization')
+        allowed = in_scope if self.scoped else lambda c, url: c is not None and c.matches(url)
+        if allowed(self.credential, req.full_url):
+            req.add_unredirected_header('Authorization', self.credential.header())
+        return req
+
+    http_request = https_request = _request

ui/apk_sources/_images.py

@@ -9,7 +9,7 @@ import threading
 import time
 from urllib.parse import urljoin, urlsplit
 
-from apk_sources import SourceError
+from apk_sources import SourceError, _credentials
 
 _lock = threading.Lock()
 _urls = OrderedDict()
@@ -141,7 +141,11 @@ def get(url, headers=None, redirects=3, deadline=None, limit=MAX_IMAGE):
         path = p.path or '/'
         if p.query:
             path += '?' + p.query
-        conn.request('GET', path, headers={'User-Agent': 'FrameControl/0.3.1', **(headers or {})})
+        sent = {'User-Agent': 'FrameControl/0.3.1', **(headers or {})}
+        login = _credentials.for_url(url)  # a repository behind a login: its own artwork only, per hop
+        if login:
+            sent['Authorization'] = login.header()
+        conn.request('GET', path, headers=sent)
         live[0].settimeout(min(10, left()))
         response = conn.getresponse()
         if response.status in (301, 302, 303, 307, 308) and redirects:

ui/apk_sources/_web.py

@@ -3,7 +3,7 @@ import hashlib, os, shutil, tempfile, threading, time, urllib.error, urllib.pars
 from email.utils import parsedate_to_datetime
 
 import frame_host
-from . import SourceError, SourceLimited
+from . import SourceAuthError, SourceError, SourceLimited, _credentials
 
 UA = 'FrameControl/0.1'
 APK_CAP = 2 * 1024 ** 3  # cached APKs across all sources, least recently used go first
@@ -38,7 +38,7 @@ def release(path):
         else:
             _in_use.pop(path, None)
 BACKOFF = 600  # seconds to leave a host alone after 403/429 without Retry-After
-_limited = {}  # host -> time.time() before which we don't contact it
+_limited = {}  # host (+ login identity) -> time.time() before which we don't ask it with that login
 _limited_lock = threading.Lock()
 
 
@@ -46,8 +46,16 @@ def _host(url):
     return (urllib.parse.urlsplit(url).hostname or '').lower()
 
 
-def throttle(url, headers=None):
-    """Remember that url's host asked us to back off; return the delay in seconds."""
+def _who(url, auth=None):
+    """Backoff key: the host, per login. A token has its own quota, so anonymous
+    rate limiting mustn't hold back requests made with one (or the reverse)."""
+    if auth is None:
+        return _host(url)
+    return _host(url) + ' ' + hashlib.sha256(auth.header().encode()).hexdigest()[:16]
+
+
+def throttle(url, headers=None, auth=None):
+    """Remember that url's host asked us (with this login) to back off; return the delay in seconds."""
     headers = headers or {}
     now, delay = time.time(), None
     value = (headers.get('Retry-After') or '').strip()
@@ -62,14 +70,15 @@ def throttle(url, headers=None):
     if delay is None and headers.get('X-RateLimit-Remaining') == '0' and reset.isdigit():
         delay = int(reset) - now  # GitHub
     delay = min(max(delay if delay is not None else BACKOFF, 1), 6 * 3600)
+    key = _who(url, auth)
     with _limited_lock:
-        _limited[_host(url)] = max(_limited.get(_host(url), 0), now + delay)
+        _limited[key] = max(_limited.get(key, 0), now + delay)
     return delay
 
 
-def wait_time(url):
+def wait_time(url, auth=None):
     with _limited_lock:
-        return max(0, _limited.get(_host(url), 0) - time.time())
+        return max(0, _limited.get(_who(url, auth), 0) - time.time())
 
 
 def limited_error(name, seconds, hint=''):
@@ -136,6 +145,8 @@ def checked_url(url, hosts):
 
 
 class Redirect(urllib.request.HTTPRedirectHandler):
+    """Only to the allowed hosts, and never carrying an Authorization header along:
+    a credential passed to open_url() is added again, by its Handler, only on its origin."""
     def __init__(self, hosts):
         self.hosts = hosts
 
@@ -147,13 +158,15 @@ class Redirect(urllib.request.HTTPRedirectHandler):
         return result
 
 
-def open_url(url, hosts, headers=None):
+def open_url(url, hosts, headers=None, auth=None):
+    """auth: an _credentials.Credential, sent only to requests on its own origin."""
     checked_url(url, hosts)
-    return urllib.request.build_opener(Redirect(hosts)).open(
+    handlers = [Redirect(hosts)] + ([_credentials.Handler(auth)] if auth else [])
+    return urllib.request.build_opener(*handlers).open(
         urllib.request.Request(url, headers={'User-Agent': UA, **(headers or {})}), timeout=60)
 
 
-def read(url, hosts, headers=None, ttl=3600, name=None, hint=''):
+def read(url, hosts, headers=None, ttl=3600, name=None, hint='', auth=None):
     path = os.path.join(cache(), hashlib.sha256(url.encode()).hexdigest() + '.data')
     name = name or _host(url) or 'The source'
 
@@ -165,13 +178,13 @@ def read(url, hosts, headers=None, ttl=3600, name=None, hint=''):
         if os.path.isfile(path) and time.time() - os.path.getmtime(path) < ttl:
             with open(path, 'rb') as f:
                 return f.read()
-        wait = wait_time(url)
+        wait = wait_time(url, auth)
         if wait:
             data = cached()
             if data is None:
                 raise limited_error(name, wait, hint)
             return data
-        with open_url(url, hosts, headers) as r:
+        with open_url(url, hosts, headers, auth) as r:
             data = r.read(8 * 1024 * 1024 + 1)
         if len(data) > 8 * 1024 * 1024:
             raise SourceError('Source index is too large')
@@ -185,8 +198,10 @@ def read(url, hosts, headers=None, ttl=3600, name=None, hint=''):
                 os.remove(tmp)
         return data
     except urllib.error.HTTPError as e:
+        if e.code == 401:  # not 403: GitHub also uses that for rate limits, token or not
+            raise SourceAuthError('%s refused the login (HTTP 401)' % name) from e
         if e.code in (403, 429):
-            delay = throttle(url, e.headers)
+            delay = throttle(url, e.headers, auth)
             data = cached()
             if data is None:
                 raise limited_error(name, delay, hint) from e

ui/apk_sources/fdroid.py

@@ -20,7 +20,7 @@ import zipfile
 
 if __package__ in (None, ''):
     sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-from apk_sources import SourceError, SourceLimited, _web
+from apk_sources import SourceAuthError, SourceError, SourceLimited, _credentials, _web
 import frame_host
 from frame_apk_sign import _der_parts, _cert_key, der
 from frame_catalog import _IndexReader, _reduce_index, _sha256
@@ -71,6 +71,39 @@ def _url(url, fingerprint=None):
     return urllib.parse.urlunsplit(('https', p.netloc.lower(), p.path.rstrip('/') + '/', '', '')), pin
 
 
+def _split_login(url, username=None, password=None):
+    """(url without user:pass, credential or None). The address may carry the login, as
+    F-Droid's client accepts (https://user:pass@host/fdroid/repo), or it may be given apart."""
+    try:
+        p = urllib.parse.urlsplit(url)
+        p.port
+    except ValueError as e:
+        raise SourceError('invalid repository URL') from e
+    if '@' in p.netloc:
+        if p.username is None:
+            raise SourceError('invalid repository URL')
+        linked = urllib.parse.unquote(p.username), None if p.password is None else urllib.parse.unquote(p.password)
+        if username not in (None, '', linked[0]) or (linked[1] is not None and password not in (None, '', linked[1])):
+            raise SourceError('the address and the username/password fields disagree')
+        username = linked[0]
+        password = linked[1] if linked[1] is not None else password
+        url = urllib.parse.urlunsplit((p.scheme, p.netloc.rpartition('@')[2], p.path, p.query, p.fragment))
+    if username in (None, ''):
+        if password:
+            raise SourceError('a password needs a username')
+        return url, None
+    clean, _ = _url(url)
+    try:
+        return url, _credentials.basic(clean, username, password or '')
+    except ValueError as e:
+        raise SourceError('cannot use that login: ' + str(e)) from None
+
+
+def _login(source):
+    """The saved login for a user repository, or None."""
+    return None if source.get('builtin') else _credentials.get(source['id'])
+
+
 def _child(base, name):
     name = str(name).lstrip('/')
     decoded = urllib.parse.unquote(name)
@@ -81,6 +114,9 @@ def _child(base, name):
     url = urllib.parse.urljoin(base, urllib.parse.quote(decoded, safe='/'))  # names may contain '#' or spaces
     if not url.startswith(base) or urllib.parse.urlsplit(url).query or urllib.parse.urlsplit(url).fragment:
         raise SourceError('repository file is outside its repository')
+    # Nothing a server could normalise elsewhere: double-encoded dots, ';' parameters, encoded slashes.
+    if _credentials.path_segments(urllib.parse.urlsplit(url).path) is None:
+        raise SourceError('unsafe repository file name')
     return url
 
 
@@ -91,14 +127,17 @@ class _HTTPSRedirect(urllib.request.HTTPRedirectHandler):
         return super().redirect_request(req, fp, code, msg, headers, newurl)
 
 
-def _fetch(url, path, maximum):
+def _fetch(url, path, maximum, auth=None):
+    """auth: the repository's login, sent only inside the repository (its origin and directory,
+    by whole plain path segments), on every request including redirects."""
     host = urllib.parse.urlsplit(url).hostname
-    wait = _web.wait_time(url)
+    wait = _web.wait_time(url, auth)
     if wait:
         raise _web.limited_error(host, wait)
     request = urllib.request.Request(url, headers={'User-Agent': 'FrameControl/1.0'})
+    handlers = [_HTTPSRedirect()] + ([_credentials.Handler(auth, scoped=True)] if auth else [])
     try:
-        with urllib.request.build_opener(_HTTPSRedirect()).open(request, timeout=60) as r, open(path, 'wb') as f:
+        with urllib.request.build_opener(*handlers).open(request, timeout=60) as r, open(path, 'wb') as f:
             total = 0
             while True:
                 chunk = r.read(1 << 20)
@@ -109,8 +148,11 @@ def _fetch(url, path, maximum):
                     raise SourceError('repository file exceeds size limit')
                 f.write(chunk)
     except urllib.error.HTTPError as e:
+        if e.code == 401 or (e.code == 403 and auth):
+            raise SourceAuthError('The repository refused the login (HTTP %d); check the username and password'
+                                  % e.code if auth else 'This repository needs a username and password') from None
         if e.code in (403, 429):
-            raise _web.limited_error(host, _web.throttle(url, e.headers)) from e
+            raise _web.limited_error(host, _web.throttle(url, e.headers, auth)) from e
         raise
 
 
@@ -342,7 +384,7 @@ def sources():
     return [dict(id=i, kind=KIND, name=n, url=u, fingerprint=p, builtin=True,
                  # The archive only holds superseded versions; it clutters search unless asked for.
                  enabled=settings['enabled'].get(i, i != 'fdroid-archive'), trust='community')
-            for i, n, u, p in builtins] + settings['repos']
+            for i, n, u, p in builtins] + [dict(s, login=_credentials.has(s['id'])) for s in settings['repos']]
 
 
 def _text(value):
@@ -514,10 +556,12 @@ def _source_lock(source_id):
         return _load_locks.setdefault(source_id, threading.Lock())
 
 
-def _load(source, force=False):
+def _load(source, force=False, auth=None):
+    """auth: a login to use instead of the saved one (while adding, before anything is saved)."""
     if not re.fullmatch(r'[a-z0-9-]+', source['id']):
         raise SourceError('invalid source id')
     _url(source['url'], source.get('fingerprint'))
+    auth = auth or _login(source)
     cache = frame_host.cache_dir('apk-sources', source['id'] + '.json')
     found = None if force else _cached(source, cache)
     if found:
@@ -535,7 +579,7 @@ def _load(source, force=False):
                 jar, raw = Path(tmp) / 'index.jar', Path(tmp) / 'index.json'
                 v2 = True
                 try:
-                    _fetch(source['url'] + 'entry.jar', jar, 8 * 1024 * 1024)
+                    _fetch(source['url'] + 'entry.jar', jar, 8 * 1024 * 1024, auth=auth)
                 except urllib.error.HTTPError as e:
                     if e.code not in (404, 410):
                         raise
@@ -543,7 +587,7 @@ def _load(source, force=False):
                         raise SourceError('repository no longer serves its signed v2 index; '
                                           'refusing to fall back to the older v1 index') from e
                     v2 = False
-                    _fetch(source['url'] + 'index-v1.jar', jar, 256 * 1024 * 1024)
+                    _fetch(source['url'] + 'index-v1.jar', jar, 256 * 1024 * 1024, auth=auth)
                     content, pin = _jar(jar, 'index-v1.json', source.get('fingerprint'))
                     timestamp = _v1(content, raw)
                     _check_timestamp(source, timestamp)
@@ -552,7 +596,7 @@ def _load(source, force=False):
                     signed = json.loads(content)
                     timestamp, entry = signed.get('timestamp'), signed['index']
                     _check_timestamp(source, timestamp)
-                    _fetch(_child(source['url'], entry['name']), raw, 512 * 1024 * 1024)
+                    _fetch(_child(source['url'], entry['name']), raw, 512 * 1024 * 1024, auth=auth)
                     if _sha256(raw) != entry['sha256'] or (entry.get('size') is not None and raw.stat().st_size != entry['size']):
                         raise SourceError('index SHA-256 or size mismatch')
                 apps = _reduce(raw, source)
@@ -572,7 +616,10 @@ def _load(source, force=False):
             raise SourceError('cannot load repository: ' + str(e)) from e
 
 
-def add_repo(url, fingerprint=None, name=None):
+def add_repo(url, fingerprint=None, name=None, username=None, password=None):
+    """Check and save a repository. A login (in the URL or apart) is kept out of the
+    saved URL, in _credentials, and saved only once the repository's index verified."""
+    url, login = _split_login(url, username, password)
     url, pin = _url(url, fingerprint)
     with _LOCK:
         existing = next((s for s in _read()['repos'] if s['url'] == url), None)
@@ -583,15 +630,26 @@ def add_repo(url, fingerprint=None, name=None):
     source = dict(id='fdroid-user-' + hashlib.sha256(url.encode()).hexdigest()[:20], kind=KIND,
                   name=name or (existing or {}).get('name') or urllib.parse.urlsplit(url).hostname,
                   url=url, builtin=False, enabled=True, trust='user', fingerprint=pin)
-    _, source['fingerprint'] = _load(source, force=True)  # network work outside the settings lock
+    # Network work outside the settings lock; without a new login, a saved one (re-adding) is used.
+    _, source['fingerprint'] = _load(source, force=True, auth=login)
     source['trust_on_first_use'] = existing.get('trust_on_first_use', False) if existing else pin is None
     with _LOCK:
         settings = _read()
         current = next((s for s in settings['repos'] if s['id'] == source['id']), None)
         if current and current['fingerprint'] != source['fingerprint']:
             raise SourceError('repository already has a different pinned fingerprint; remove it first')
-        settings['repos'] = [s for s in settings['repos'] if s['id'] != source['id']] + [source]
-        _write(_storage(), settings)
+        previous = _credentials.get(source['id'])
+        if login:
+            _credentials.put(source['id'], login)
+        try:
+            settings['repos'] = [s for s in settings['repos'] if s['id'] != source['id']] + [source]
+            _write(_storage(), settings)
+        except BaseException:
+            if login and previous:  # not saved after all: leave the login as it was
+                _credentials.put(source['id'], previous)
+            elif login:
+                _credentials.remove(source['id'])
+            raise
     return source
 
 
@@ -602,6 +660,7 @@ def remove_repo(source_id):
             raise SourceError('unknown user repository')
         settings['repos'] = [s for s in settings['repos'] if s['id'] != source_id]
         _write(_storage(), settings)
+        _credentials.remove(source_id)
     with _state_file_lock(), _LOCK:  # lock order: state file, then _LOCK
         states = _states()
         if states.pop(source_id, None) is not None:  # re-adding is a deliberate new trust decision
@@ -663,7 +722,7 @@ def download(source, entry_id, version_code=None):
             fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix='.part')
             os.close(fd)
             try:
-                _fetch(_child(source['url'], version['name']), tmp, 4 * 1024 ** 3)
+                _fetch(_child(source['url'], version['name']), tmp, 4 * 1024 ** 3, auth=_login(source))
                 if _sha256(tmp) != sha:
                     raise SourceError('APK SHA-256 mismatch; download discarded')
                 os.replace(tmp, path)
@@ -684,6 +743,8 @@ def main():
     add.add_argument('url')
     add.add_argument('--fingerprint')
     add.add_argument('--name')
+    add.add_argument('--username', help='for a repository behind a login (HTTP basic auth)')
+    add.add_argument('--password-stdin', action='store_true', help='read the password from the first line of stdin')
     sub.add_parser('list')
     remove = sub.add_parser('remove')
     remove.add_argument('source')
@@ -702,7 +763,12 @@ def main():
 def _cli(parser, args):
     try:
         if args.command == 'add':
-            result = add_repo(args.url, args.fingerprint, args.name)
+            password = None
+            if args.password_stdin:
+                if not args.username:
+                    raise SourceError('--password-stdin needs --username')
+                password = sys.stdin.readline().rstrip('\r\n')
+            result = add_repo(args.url, args.fingerprint, args.name, args.username, password)
         elif args.command == 'list':
             result = sources()
         elif args.command == 'remove':

ui/apk_sources/github.py

@@ -1,17 +1,77 @@
 """Curated publisher releases; optional topic discovery is page-only."""
-import fnmatch, json, os, re, urllib.parse
+import fnmatch, hashlib, json, os, re, sys, threading, urllib.parse
 
-from . import SourceError, _web
+from . import SourceAuthError, SourceError, _credentials, _web
 
 KIND = 'github'
 API = 'https://api.github.com'
+ENV = 'FRAME_GITHUB_TOKEN'  # wins over a token saved in source settings
+CREDENTIAL = 'github'  # _credentials key; only ever sent to api.github.com
+_rejected = set()  # SHA-256 of tokens GitHub answered 401 to (this run only)
+_rejected_lock = threading.Lock()
 TOPICS = ('oculus-quest', 'openxr', 'quest')
 HOSTS = ('github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com')
 
 
 def sources():
     return [{'id': KIND, 'kind': KIND, 'name': 'GitHub releases', 'url': 'https://github.com',
-             'builtin': True, 'enabled': True, 'trust': 'community'}]
+             'builtin': True, 'enabled': True, 'trust': 'community', 'token': token_status()}]
+
+
+def _token():
+    """(token, where it came from) or (None, None). The environment variable wins."""
+    env = os.environ.get(ENV)
+    if env:
+        return env, 'env'
+    saved = _credentials.get(CREDENTIAL)
+    if saved and saved.kind == 'bearer':
+        return saved, 'saved'
+    return None, None
+
+
+def _credential(token):
+    if isinstance(token, _credentials.Credential):
+        return token
+    try:
+        return _credentials.bearer(API + '/', token)
+    except ValueError as e:
+        raise SourceError('%s is not a usable GitHub token: %s' % (ENV, e)) from None
+
+
+def _digest(credential):
+    return hashlib.sha256(credential.header().encode()).hexdigest()
+
+
+def token_status():
+    """Whether a token is set, where from, and whether GitHub rejected it; never the token."""
+    token, where = _token()
+    rejected = False
+    if token is not None:
+        try:
+            with _rejected_lock:
+                rejected = _digest(_credential(token)) in _rejected
+        except SourceError:
+            rejected = True
+    return {'set': token is not None, 'from': where, 'rejected': rejected}
+
+
+def set_token(token):
+    """Save a personal access token for api.github.com (FRAME_GITHUB_TOKEN still wins)."""
+    if not isinstance(token, str) or not token.strip():
+        raise SourceError('Paste a GitHub token')
+    try:
+        credential = _credentials.bearer(API + '/', token.strip())
+    except ValueError as e:
+        raise SourceError('That is not a usable GitHub token: ' + str(e)) from None
+    _credentials.put(CREDENTIAL, credential)
+    with _rejected_lock:
+        _rejected.discard(_digest(credential))  # a fresh chance
+    return {'message': 'GitHub token saved' + (' (%s is set and is used instead)' % ENV if os.environ.get(ENV) else '')}
+
+
+def clear_token():
+    _credentials.remove(CREDENTIAL)
+    return {'message': 'GitHub token removed'}
 
 
 def _curated():
@@ -20,13 +80,33 @@ def _curated():
 
 
 def _api(path):
+    """GET from the GitHub API, with the user's token if there is one. If GitHub rejects
+    the token (401), retry once without it, like Obtainium, and remember the rejection."""
     headers = {'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'}
-    token = os.environ.get('FRAME_GITHUB_TOKEN')
-    if token:
-        headers['Authorization'] = 'Bearer ' + token
+    token, _ = _token()
+    auth = _credential(token) if token is not None else None
+    with _rejected_lock:
+        if auth and _digest(auth) in _rejected:
+            auth = None  # already refused this run: don't spend a request on it again
+    hint = ' (add a GitHub token in the store\'s source settings, or set %s, to raise the limit)' % ENV
+    if token is not None and not auth:
+        hint = ' (GitHub rejected your token; replace it in the store\'s source settings)'
+
+    def get(auth):
+        return _web.read(API + path, ('api.github.com',), headers, name='GitHub',
+                         hint='' if auth else hint, auth=auth)
     try:
-        return json.loads(_web.read(API + path, ('api.github.com',), headers, name='GitHub',
-                                    hint='' if token else ' (set FRAME_GITHUB_TOKEN to raise the limit)'))
+        try:
+            data = get(auth)
+        except SourceAuthError:
+            if not auth:
+                raise
+            with _rejected_lock:
+                _rejected.add(_digest(auth))
+            print('GitHub rejected the access token (HTTP 401); continuing without it', file=sys.stderr)
+            hint = ' (GitHub rejected your token; replace it in the store\'s source settings)'
+            data = get(None)
+        return json.loads(data)
     except (ValueError, TypeError) as e:
         raise SourceError('Invalid GitHub response') from e
 

ui/apk_sources/search.py

@@ -113,6 +113,14 @@ def manage_repo(action, **kwargs):
     return getattr(module, action)(**kwargs)
 
 
+def set_token(source_id, token):
+    """Save (token) or remove (None) a source's access token; status is in sources()."""
+    module, _ = resolve(source_id)
+    if not hasattr(module, 'set_token'):
+        raise SourceError('This source does not use an access token')
+    return module.set_token(token) if token is not None else module.clear_token()
+
+
 def normalise(name):
     return ' '.join(re.findall(r'\w+', unicodedata.normalize('NFKC', name or '').casefold()))
 

ui/frame_contact.py

@@ -2,7 +2,7 @@
 
 Two separate opt-in choices, both off until ticked:
 
-- updates: the mailing list, occasional emails about new features and updates
+- updates: occasional notices about Frame Control releases and updates
 - followup: the maintainer may ask follow-up questions, mainly about problem reports
 
 The address and the choices are kept on this computer (frame_host.data_dir('contact')) and
@@ -12,26 +12,20 @@ sends a new event under this copy's own random contact id (not the analytics id)
 by `rev`, and the highest rev for an id is the one that counts, whatever the clocks say:
 removing the address sends a withdrawal with no address in it, and wipes the address from
 the local log of what was sent. The maintainer lists who agreed to what with
-`python3 ui/frame_report.py contacts`. Mailing-list changes also go to the site's
-private Cloudflare D1 list through frame_mailing.submit, independently of PostHog.
-Neither endpoint sends email. No mailing request is made for a skip or for a
-follow-up-only address.
+`python3 ui/frame_report.py contacts`. Nothing here sends email.
 
 A change that can't be sent (offline) waits in the state file and is retried in the
-background, so a withdrawal is never lost. Saving never waits for a network request.
-The page's one-time prompt is remembered here
+background, so a withdrawal is never lost. The page's one-time prompt is remembered here
 too: once it has been shown or dismissed it never comes back.
 """
 import json
 import os
 import re
-import secrets
 import threading
 import time
 import uuid
 
 import frame_host
-import frame_mailing
 import frame_telemetry
 
 STATE = frame_host.data_dir('contact')
@@ -50,8 +44,7 @@ _retrier = None
 
 def _defaults():
     return {'id': str(uuid.uuid4()), 'email': '', 'updates': False, 'followup': False,
-            'prompt': 'new', 'pending': None, 'rev': 0,
-            'mailingToken': secrets.token_hex(32), 'mailingPending': None}
+            'prompt': 'new', 'pending': None, 'rev': 0}
 
 
 def load():
@@ -68,10 +61,9 @@ def load():
 
 
 def _save(s):
-    STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
+    STATE.mkdir(parents=True, exist_ok=True)
     tmp = FILE.with_suffix('.tmp')
-    with os.fdopen(os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
-        json.dump(s, f, indent=1)
+    tmp.write_text(json.dumps(s, indent=1))
     os.replace(tmp, FILE)
 
 
@@ -105,13 +97,12 @@ def from_report(email):
 
 
 def state():
-    """What the page shows. showPrompt: the one-time mailing list offer hasn't been shown or
-    answered yet. It's offered on first launch (the page shows it after the first-run privacy
-    notice is closed, never both at once), and works whether or not the Frame is set up."""
+    """What the page shows. showPrompt: the one-time prompt hasn't been shown or answered yet,
+    and the Frame has connected at least once (setup worked), so it never greets a new install."""
     s = load()
+    set_up = bool(frame_telemetry.settings().get('frames_seen'))
     return {'email': s['email'], 'updates': s['updates'], 'followup': s['followup'],
-            'waiting': s['pending'] is not None or s['mailingPending'] is not None,
-            'showPrompt': s['prompt'] == 'new'}
+            'waiting': s['pending'] is not None, 'showPrompt': s['prompt'] == 'new' and set_up}
 
 
 def _event(s):
@@ -131,35 +122,19 @@ def _send_pending(block=True):
     try:
         while True:
             with _lock:
-                s = load()
-                event, mailing = s['pending'], s['mailingPending']
-            if event is None and mailing is None:
+                event = load()['pending']
+            if event is None:
                 break
-            failed = False
-            if event is not None:
-                try:
-                    frame_telemetry.post([event], timeout=30)
-                    _sent(event)
-                except frame_telemetry.SendError:
-                    failed = True
-            if mailing is not None:
-                try:
-                    frame_mailing.submit(s['mailingToken'], mailing['rev'], mailing['email'])
-                    with _lock:
-                        latest = load()
-                        if latest['mailingPending'] == mailing:
-                            latest['mailingPending'] = None
-                            _save(latest)
-                except frame_mailing.SendError:
-                    failed = True
-            if failed:
+            try:
+                frame_telemetry.post([event], timeout=30)
+            except frame_telemetry.SendError:
                 return False
+            _sent(event)
     finally:
         _send_lock.release()
     # A change saved just as this finished found the lock still held and left it to us.
     with _lock:
-        s = load()
-        left = s['pending'] is not None or s['mailingPending'] is not None
+        left = load()['pending'] is not None
     return _send_pending(block=False) if left else True
 
 
@@ -222,7 +197,6 @@ def _apply(body):
     with _lock:
         s = load()
         old = s['email']
-        old_updates = s['updates']
         changed = (email, updates, followup) != (s['email'], s['updates'], s['followup'])
         s.update(email=email, updates=updates, followup=followup)
         if body.get('fromPrompt') or email:
@@ -232,10 +206,6 @@ def _apply(body):
             # is sent even for an address still waiting here: its send may already be under way.
             s['rev'] += 1
             s['pending'] = _event(s)
-            # No mailing request for a skip or a follow-up-only address. A
-            # withdrawal keeps no address in the queue, even while offline.
-            if old_updates or updates:
-                s['mailingPending'] = {'rev': s['rev'], 'email': email if updates else ''}
         _save(s)
         if old and old.lower() != email.lower():
             try:
@@ -246,8 +216,8 @@ def _apply(body):
 
 
 def _deliver(changed):
-    if changed:
-        _wake.set()  # all network I/O stays in start()'s background worker
+    if changed and not _send_pending(block=False):
+        _wake.set()  # offline, or a send under way that will take this change with it
 
 
 def prompt(body):

ui/frame_devices.py

@@ -740,7 +740,9 @@ class Registry:
                 return a
         raise DeviceError(f"{host} isn't one of this headset's addresses")
 
-    def add_address(self, device_id, host, kind=None, label=""):
+    def add_address(self, device_id, host, kind=None, label="", first=False):
+        """Add an address at the end of the list, or at the front (first=True), where the
+        user's order makes it win over the others that work on the same network."""
         with self._changing():
             d = self._find(device_id)
             a = new_address(host, kind, label)
@@ -748,7 +750,7 @@ class Registry:
                 raise DeviceError(f"{a['host']} is already on the list")
             if len(d["addresses"]) >= 32:
                 raise DeviceError("That's enough addresses for one headset")
-            d["addresses"].append(a)
+            d["addresses"].insert(0 if first else len(d["addresses"]), a)
             self.save()
             return copy.deepcopy(a)
 

ui/frame_link.py

@@ -356,6 +356,7 @@ class Link:
         self.version = 0
         self.stopped = False
         self.kicks = []                 # reasons someone asked for a (re)connect
+        self.test_gen = {}              # device id -> its newest test of the addresses (see test())
         self.busy = False               # the loop is handling kicks
         self.state = {"phase": "idle", "reason": None, "device": None, "network": None, "stages": [],
                       "probes": [], "via": None, "error": None, "retry_at": None, "attempt": 0,
@@ -1306,9 +1307,13 @@ class Link:
         started = now()
         rows = [{"host": a["host"], "kind": a["kind"], "state": "waiting", "detail": "Waiting", "ip": None,
                  "rtt_ms": None, "ssh": None} for a in device["addresses"]]
+        with self.cond:
+            gen = self.test_gen[device_id] = self.test_gen.get(device_id, 0) + 1
 
         def put(**fields):
             with self.cond:
+                if gen != self.test_gen[device_id]:
+                    return  # a newer test has started: its results are the ones to show
                 self.state["tests"][device_id] = dict({"started": started, "done": False, "rows": rows}, **fields)
                 self.version += 1
                 self.cond.notify_all()
@@ -1617,7 +1622,8 @@ def devices_action(link, body, open_setup, busy=lambda: 0):
                 raise frame_devices.DeviceError(f"Removed, but couldn't edit ~/.ssh/config: {e}")
         msg = f"Removed {d['name']}" + (f" and its '{d['alias']}' entry in ~/.ssh/config" if removed else "")
     elif action == "address-add":
-        a = reg.add_address(did, body.get("host"), body.get("kind") or None, body.get("label") or "")
+        a = reg.add_address(did, body.get("host"), body.get("kind") or None, body.get("label") or "",
+                            first=body.get("first") is True)
         if is_active and link.state["phase"] == "failed":
             link.kick("retry")
         msg = f"Added {a['host']}"

ui/frame_mailing.py

@@ -1,36 +0,0 @@
-"""Mailing-list transport. Consent and the retry queue live in frame_contact.py.
-
-The Cloudflare Pages endpoint stores addresses privately in D1. Override
-FRAME_CONTROL_MAILING_LIST_URL to use a local server or a future provider adapter.
-No analytics identifiers or machine details are sent through this integration.
-"""
-import json
-import os
-import urllib.error
-import urllib.request
-
-DEFAULT_URL = 'https://framecontrol.fun/api/mailing-list'
-
-
-class SendError(Exception):
-    pass
-
-
-def submit(token, rev, email):
-    """Set this copy's latest opt-in, or withdraw it with an empty email.
-
-    token is a random capability, separate from analytics and contact IDs. The
-    revision makes retries idempotent and prevents delayed joins undoing a removal.
-    """
-    url = os.environ.get('FRAME_CONTROL_MAILING_LIST_URL', DEFAULT_URL)
-    body = json.dumps({'token': token, 'rev': rev, 'email': email,
-                       'updates': bool(email)}).encode('utf-8')
-    try:
-        request = urllib.request.Request(url, data=body, headers={
-            'Content-Type': 'application/json', 'User-Agent': 'Frame-Control-mailing-list'})
-        with urllib.request.urlopen(request, timeout=10) as response:
-            answer = json.loads(response.read(1024))
-            if response.status != 200 or not isinstance(answer, dict) or answer.get('ok') is not True:
-                raise SendError('mailing list did not accept the change')
-    except (OSError, ValueError, urllib.error.URLError) as err:
-        raise SendError('could not reach the mailing list') from err

ui/server.py

@@ -2399,6 +2399,16 @@ def source_text(body, key, optional=False):
     return value.strip()
 
 
+def login_text(body, key):
+    """A username, password or token: kept exactly (no stripping) and never echoed back."""
+    value = body.get(key)
+    if value in (None, ''):
+        return None
+    if not isinstance(value, str) or len(value) > 4096:
+        raise Failure('Provide a valid ' + key, 400)
+    return value
+
+
 def source_search(query):
     args = parse_qs(query)
     q = args.get('q', [''])[0]
@@ -2437,19 +2447,31 @@ def source_manage(body):
         if action == 'add':
             url = source_text(body, 'url')
             fingerprint, name = source_text(body, 'fingerprint', True), source_text(body, 'name', True)
-            parts = urlparse(url)
-            if parts.scheme not in ('https', 'fdroidrepos') or not parts.hostname or parts.username:
-                raise Failure('Use an HTTPS or fdroidrepos:// repository URL without credentials', 400)
+            username, password = login_text(body, 'username'), login_text(body, 'password')
+            try:
+                parts = urlparse(url)
+                parts.port
+            except ValueError:
+                raise Failure('Use an HTTPS or fdroidrepos:// repository URL', 400) from None
+            if parts.scheme not in ('https', 'fdroidrepos') or not parts.hostname:
+                raise Failure('Use an HTTPS or fdroidrepos:// repository URL', 400)
             apk_search.repo_module()  # fail now if this build can't manage repositories
 
             def add():  # downloads and verifies the whole index: a job, not a request
-                source = apk_search.manage_repo('add_repo', url=url, fingerprint=fingerprint, name=name)
+                # A login (here or in the URL) is stored apart and never comes back in a response.
+                login = {'username': username, 'password': password} if username or password else {}
+                source = apk_search.manage_repo('add_repo', url=url, fingerprint=fingerprint, name=name, **login)
                 message = 'Added ' + source['name']
                 if source.get('trust_on_first_use'):
                     message += '. Trusted on first use: ' + source['fingerprint'].upper()
                 return {'message': message, 'source': {k: source.get(k) for k in
                                                        ('id', 'name', 'fingerprint', 'trust_on_first_use')}}
             return start_job('Add repository', add)
+        if action in ('set-token', 'clear-token'):
+            token = login_text(body, 'token') if action == 'set-token' else None
+            if action == 'set-token' and not token:
+                raise Failure('Paste a token', 400)
+            return apk_search.set_token(source_text(body, 'source'), token)
         if action == 'game-data':
             package = source_text(body, 'package')
             return start_job('Add game data', lambda: apk_search.add_game_data(package))