diff --git a/tests/test_apk_search.py b/tests/test_apk_search.py index 6b4e3ed..29f8511 100644 --- a/tests/test_apk_search.py +++ b/tests/test_apk_search.py @@ -325,7 +325,7 @@ class EndpointTests(SettingsTest): mod.add_repo.side_effect = SourceError('repository fingerprint mismatch') job = self.wait(self.request('POST', '/api/sources', {'action': 'add', 'url': link})[1]['job']) self.assertEqual(job['error'], 'repository fingerprint mismatch') # no "SourceError:" prefix - for url in ('http://repo.example/repo', 'fdroidrepo://repo.example/repo', 'https://u@repo.example/'): + for url in ('http://repo.example/repo', 'fdroidrepo://repo.example/repo', 'https://repo.example:x/'): self.assertEqual(self.request('POST', '/api/sources', {'action': 'add', 'url': url})[0], 400) self.assertEqual(self.request('POST', '/api/sources', {'action': 'remove', 'source': 'fdroid'})[0], 200) mod.remove_repo.assert_called_once_with(source_id='fdroid') diff --git a/tests/test_fdroid_sources.py b/tests/test_fdroid_sources.py index 6ec0b2b..3f18e63 100644 --- a/tests/test_fdroid_sources.py +++ b/tests/test_fdroid_sources.py @@ -81,7 +81,7 @@ class Repositories(unittest.TestCase): state.clear() self.addCleanup(state.clear) - def fetch(self, url, path, maximum): + def fetch(self, url, path, maximum, auth=None): name = url.rsplit('/', 1)[-1] if self.v1 and name == 'entry.jar': raise urllib.error.HTTPError(url, 404, 'missing', None, None) @@ -311,7 +311,7 @@ class Repositories(unittest.TestCase): source = self.add() other = dict(source, id='other-repo') started, release = threading.Event(), threading.Event() - def fetch(url, path, maximum): + def fetch(url, path, maximum, auth=None): if threading.current_thread().name == 'slow': started.set() release.wait(5) @@ -355,7 +355,7 @@ class Repositories(unittest.TestCase): self.files['entry.jar'], _ = entry_jar(50) source = fdroid.add_repo(URL) jars = {'older': entry_jar(100)[0], 'newer': entry_jar(200)[0]} - def fetch(url, path, maximum): + def fetch(url, path, maximum, auth=None): name = threading.current_thread().name if name in jars and url.endswith('entry.jar'): Path(path).write_bytes(jars[name]) @@ -401,7 +401,7 @@ class Repositories(unittest.TestCase): source = dict(id='overlap', name='Overlap', url=URL, fingerprint=None) self.v1 = True inner = [] - def fetch(url, path, maximum): + def fetch(url, path, maximum, auth=None): if url.endswith('index-v1.jar') and not inner: inner.append(1) # the v1 load passed its fallback check; a v2 load finishes now self.v1 = False @@ -471,7 +471,7 @@ class Repositories(unittest.TestCase): source = self.add() self.expire(source) release = threading.Event() - def slow(url, path, maximum): + def slow(url, path, maximum, auth=None): release.wait(5) self.fetch(url, path, maximum) self.fetch_mock.side_effect = slow @@ -528,7 +528,7 @@ class Repositories(unittest.TestCase): self.expire(source) before = self.fetch_mock.call_count release = __import__('threading').Event() - def slow(url, path, maximum): + def slow(url, path, maximum, auth=None): release.wait(5) self.fetch(url, path, maximum) self.fetch_mock.side_effect = slow diff --git a/tests/test_source_logins.py b/tests/test_source_logins.py new file mode 100644 index 0000000..3187992 --- /dev/null +++ b/tests/test_source_logins.py @@ -0,0 +1,391 @@ +"""Repositories and sources behind a login: F-Droid basic auth and GitHub tokens. + +A fake HTTPS layer stands in for the network (urllib's real redirect, error and +request processing still run), so these tests see every request and its headers. +""" +import sandbox # noqa: F401 (first: keeps tests off real data and services) +import base64 +import contextlib +import http.client +import io +import json +import os +from pathlib import Path +import stat +import sys +import tempfile +import threading +import time +import unittest +from unittest.mock import patch +import urllib.request +import urllib.response + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui')) +from apk_sources import SourceAuthError, SourceError, _credentials, _images, _web, fdroid, github, search +import server + +FIXTURES = Path(__file__).parent / 'fixtures' / 'fdroid' +PIN = (FIXTURES / 'fingerprint.txt').read_text().strip() +REPO = 'https://repo.example/fdroid/repo/' +USER, PASSWORD = 'alice', 's3cr@t-pässword' +TOKEN = 'ghp_frameControlTestToken0123456789' +SECRETS = (PASSWORD, TOKEN, base64.b64encode((USER + ':' + PASSWORD).encode()).decode()) +REAL_BUILD_OPENER = urllib.request.build_opener + + +def basic(user, password): + return 'Basic ' + base64.b64encode((user + ':' + password).encode()).decode() + + +class FakeHTTPS(urllib.request.HTTPSHandler): + """Answers https requests from route(url, authorization) -> (status, headers, body).""" + + def __init__(self, route): + super().__init__() + self.route, self.seen = route, [] + + def https_open(self, req): + auth = req.get_header('Authorization') + self.seen.append((req.full_url, auth)) + status, headers, body = self.route(req.full_url, auth) + message = http.client.HTTPMessage() + for key, value in headers.items(): + message[key] = value + response = urllib.response.addinfourl(io.BytesIO(body), message, req.full_url, status) + response.msg = 'fake' + return response + + +class Base(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + for name, value in [('data_dir', lambda *p: self.root.joinpath('data', *p)), + ('cache_dir', lambda *p: self.root.joinpath('cache', *p))]: + mock = patch.object(fdroid.frame_host, name, value) + mock.start() + self.addCleanup(mock.stop) + self.web = FakeHTTPS(self.route) + mock = patch('urllib.request.build_opener', lambda *handlers: REAL_BUILD_OPENER(*handlers, self.web)) + mock.start() + self.addCleanup(mock.stop) + mock = patch.dict(os.environ, {github.ENV: ''}) + mock.start() + self.addCleanup(mock.stop) + for state in (_web._limited, fdroid._stale, fdroid._retry_at, fdroid._refreshing, github._rejected): + state.clear() + self.addCleanup(state.clear) + self.stderr = io.StringIO() + mock = patch('sys.stderr', self.stderr) + mock.start() + self.addCleanup(mock.stop) + + def route(self, url, auth): + raise AssertionError('unexpected request ' + url) + + def assertNoSecrets(self, *texts): + for text in texts: + for secret in SECRETS + (USER + ':',): + self.assertNotIn(secret, text) + + def disk(self): + """Everything saved under the data and cache folders except the credentials file.""" + out = [] + for path in self.root.rglob('*'): + if path.is_file() and path.name != 'source-credentials.json': + out.append(path.read_bytes().decode('utf-8', 'replace')) + return '\n'.join(out) + + def credentials_file(self): + return self.root / 'data' / 'source-credentials.json' + + +class Repository(Base): + """A repository at REPO that wants USER and PASSWORD.""" + def setUp(self): + super().setUp() + self.login = basic(USER, PASSWORD) + self.redirect = None # file name -> absolute URL it redirects to + self.forbidden = 401 + + def serve(self, name): + return (FIXTURES / ('example.apk' if name.endswith('.apk') else name)).read_bytes() + + def route(self, url, auth): + for mirror in ('https://mirror.example/fdroid/repo/', 'https://repo.example/elsewhere/'): + if url.startswith(mirror): # serves anyone; a login must never get here unless same-origin + return 200, {}, self.serve(url[len(mirror):]) + self.assertTrue(url.startswith(REPO), url) + if auth != self.login: + return self.forbidden, {'WWW-Authenticate': 'Basic realm="repo"'}, b'no' + name = url[len(REPO):] + if self.redirect and name in self.redirect: + return 302, {'Location': self.redirect[name]}, b'' + return 200, {}, self.serve(name) + + def add(self, **kw): + return fdroid.add_repo('https://alice:s3cr%40t-p%C3%A4ssword@repo.example/fdroid/repo', PIN, **kw) + + +class RepositoryLogin(Repository): + def test_add_search_download_with_login_in_the_url(self): + source = self.add() + self.assertEqual(source['url'], REPO) + self.assertEqual(source['fingerprint'], PIN) + saved = fdroid.user_repos()[0] + self.assertEqual(saved['url'], REPO) + self.assertNotIn('login', saved) + self.assertEqual(len(fdroid.search(source, 'example offline')), 1) + downloaded = fdroid.download(source, 'org.example.app', 1) + self.assertTrue(downloaded['verified']) + self.assertTrue(self.web.seen) + self.assertTrue(all(auth == self.login for url, auth in self.web.seen)) + self.assertTrue(all('@' not in url for url, auth in self.web.seen)) # never in the URL + self.assertTrue(next(s for s in fdroid.sources() if s['id'] == source['id'])['login']) + self.assertNoSecrets(json.dumps(source), json.dumps(fdroid.sources()), self.disk(), self.stderr.getvalue()) + # The settings file holds no part of the login; the credentials file holds it, privately. + settings = (self.root / 'data' / 'apk-repos.json').read_text() + self.assertNotIn(USER, settings) + self.assertTrue(self.credentials_file().is_file()) + if os.name != 'nt': + self.assertEqual(stat.S_IMODE(self.credentials_file().stat().st_mode), 0o600) + + def test_login_given_apart_and_saved_login_reused_on_refresh(self): + source = fdroid.add_repo(REPO, PIN, username=USER, password=PASSWORD) + self.assertEqual(source['url'], REPO) + self.web.seen.clear() + fdroid._load(fdroid.user_repos()[0], force=True) # as a background refresh would + self.assertTrue(self.web.seen) + self.assertTrue(all(auth == self.login for url, auth in self.web.seen)) + fdroid.add_repo(REPO) # re-adding without a login keeps the saved one + self.assertTrue(_credentials.has(source['id'])) + + def test_wrong_or_missing_login_saves_nothing(self): + for kw, message in [({'username': USER, 'password': 'wrong'}, 'check the username and password'), + ({}, 'needs a username and password')]: + with self.subTest(kw=kw): + with self.assertRaisesRegex(SourceAuthError, message) as caught: + fdroid.add_repo(REPO, PIN, **kw) + self.assertNoSecrets(str(caught.exception), repr(caught.exception)) + self.assertNotIn('wrong', str(caught.exception)) + self.assertEqual(fdroid.user_repos(), []) + self.assertFalse(self.credentials_file().exists()) + self.assertFalse((self.root / 'data' / 'apk-repo-state.json').exists()) + self.assertEqual(_web._limited, {}) # an auth failure isn't a rate limit + + def test_403_with_a_login_is_a_login_error_not_a_rate_limit(self): + self.forbidden = 403 + with self.assertRaisesRegex(SourceAuthError, 'check the username and password'): + fdroid.add_repo(REPO, PIN, username=USER, password='wrong') + self.assertEqual(_web._limited, {}) + self.assertEqual(fdroid.user_repos(), []) + + def test_signature_checks_still_apply(self): + with self.assertRaisesRegex(SourceError, 'fingerprint mismatch'): + fdroid.add_repo(REPO, '0' * 64, username=USER, password=PASSWORD) + self.assertEqual(fdroid.user_repos(), []) + self.assertFalse(self.credentials_file().exists()) + + def test_cross_origin_redirect_never_gets_the_login(self): + self.redirect = {'entry.jar': 'https://mirror.example/fdroid/repo/entry.jar', + 'index-v2.json': 'https://repo.example/elsewhere/index-v2.json'} + source = self.add() + self.assertEqual(source['fingerprint'], PIN) # the mirror's copy still had to verify + mirror = [auth for url, auth in self.web.seen if url.startswith('https://mirror.example/')] + same_origin = [auth for url, auth in self.web.seen if url.startswith('https://repo.example/elsewhere/')] + self.assertEqual(mirror, [None]) + self.assertEqual(same_origin, [self.login]) # same scheme, host and port: still that repository's + + def test_other_ports_and_http_never_get_the_login(self): + login = _credentials.basic(REPO, USER, PASSWORD) + for url in ('https://repo.example:8443/fdroid/repo/x', 'http://repo.example/fdroid/repo/x', + 'https://other.example/fdroid/repo/x', 'https://repo.example.evil/x'): + self.assertFalse(login.matches(url), url) + self.assertTrue(login.matches('https://REPO.example:443/anything')) + self.assertNoSecrets(repr(login)) + handler = _credentials.Handler(login) + req = urllib.request.Request('https://other.example/x', headers={'Authorization': 'Basic stale'}) + self.assertIsNone(handler.https_request(req).get_header('Authorization')) + + def test_http_and_bad_logins_are_refused_before_any_request(self): + for url, kw in [('http://alice:pw@repo.example/fdroid/repo', {}), + ('http://repo.example/fdroid/repo', {'username': USER, 'password': PASSWORD}), + (REPO, {'username': 'a:b', 'password': PASSWORD}), + (REPO, {'username': USER, 'password': 'x\r\nX-Evil: 1'}), + (REPO, {'password': PASSWORD}), + ('https://alice:pw@repo.example/fdroid/repo', {'username': 'bob'})]: + with self.subTest(url=url, kw=kw), self.assertRaises(SourceError) as caught: + fdroid.add_repo(url, PIN, **kw) + self.assertNotIn('pw', str(caught.exception).replace('password', '')) + self.assertNoSecrets(str(caught.exception)) + self.assertEqual(self.web.seen, []) + + def test_remove_deletes_the_login(self): + source = self.add() + other = _credentials.bearer('https://api.github.com/', TOKEN) + _credentials.put('github', other) + fdroid.remove_repo(source['id']) + self.assertFalse(_credentials.has(source['id'])) + self.assertNotIn(PASSWORD, self.credentials_file().read_text()) + _credentials.remove('github') + self.assertFalse(self.credentials_file().exists()) # nothing left: the file goes + + def test_cli_reads_the_password_from_stdin(self): + out = io.StringIO() + argv = ['fdroid.py', 'add', REPO, '--fingerprint', PIN, '--username', USER, '--password-stdin'] + with patch.object(sys, 'argv', argv), patch.object(sys, 'stdin', io.StringIO(PASSWORD + '\n')), \ + contextlib.redirect_stdout(out): + fdroid.main() + self.assertEqual(json.loads(out.getvalue())['url'], REPO) + self.assertNoSecrets(out.getvalue()) + self.assertTrue(_credentials.has(fdroid.user_repos()[0]['id'])) + + def test_artwork_from_the_repository_gets_the_login_and_only_there(self): + source = self.add() + self.assertEqual(_credentials.for_url(REPO + 'icons/a.png').header(), self.login) + self.assertTrue(_credentials.has(source['id'])) + self.assertIsNone(_credentials.for_url('https://repo.example/other/a.png')) # outside the repository + self.assertIsNone(_credentials.for_url('https://cdn.example/fdroid/repo/a.png')) + + +class GitHubToken(Base): + def setUp(self): + super().setUp() + self.accept = {TOKEN} + self.releases = json.dumps([]).encode() + + def route(self, url, auth): + if url.startswith('https://api.github.com/'): + if auth is not None and auth not in {'Bearer ' + t for t in self.accept}: + return 401, {}, b'{"message": "Bad credentials"}' + if url.endswith('/redirected'): + return 302, {'Location': 'https://github.com/landed'}, b'' + return 200, {}, self.releases + if url == 'https://github.com/landed': + return 200, {}, b'{"ok": true}' + raise AssertionError('unexpected request ' + url) + + def test_saved_token_goes_only_to_the_api(self): + github.set_token(TOKEN) + self.assertEqual(github._api('/repos/a/b/releases'), []) + self.assertEqual(self.web.seen, [('https://api.github.com/repos/a/b/releases', 'Bearer ' + TOKEN)]) + auth = github._credential(github._token()[0]) + self.assertEqual(_web.read('https://api.github.com/redirected', ('api.github.com', 'github.com'), auth=auth), + b'{"ok": true}') + self.assertEqual(self.web.seen[-2:], [('https://api.github.com/redirected', 'Bearer ' + TOKEN), + ('https://github.com/landed', None)]) # redirect: no token + status = github.sources()[0]['token'] + self.assertEqual(status, {'set': True, 'from': 'saved', 'rejected': False}) + self.assertNoSecrets(json.dumps(github.sources()), self.disk(), self.stderr.getvalue()) + if os.name != 'nt': + self.assertEqual(stat.S_IMODE(self.credentials_file().stat().st_mode), 0o600) + + def test_rejected_token_retries_once_without_it(self): + github.set_token(TOKEN) + self.accept = set() + self.assertEqual(github._api('/repos/a/b/releases'), []) + self.assertEqual([auth for _, auth in self.web.seen], ['Bearer ' + TOKEN, None]) + self.assertTrue(github.token_status()['rejected']) + self.assertIn('rejected', self.stderr.getvalue()) + self.assertNoSecrets(self.stderr.getvalue()) + self.web.seen.clear() + github._api('/repos/a/b/other') + self.assertEqual([auth for _, auth in self.web.seen], [None]) # not tried again this run + github.set_token(TOKEN) # saving it again gives it a fresh chance + self.assertFalse(github.token_status()['rejected']) + + def test_environment_variable_wins(self): + github.set_token(TOKEN) + env = 'ghp_fromTheEnvironment' + self.accept = {env} + with patch.dict(os.environ, {github.ENV: env}): + github._api('/repos/a/b/releases') + self.assertEqual(self.web.seen[-1][1], 'Bearer ' + env) + self.assertEqual(github.token_status(), {'set': True, 'from': 'env', 'rejected': False}) + self.assertEqual(github.token_status()['from'], 'saved') + + def test_clear_removes_it_from_disk(self): + github.set_token(TOKEN) + github.clear_token() + self.assertFalse(self.credentials_file().exists()) + self.assertEqual(github.token_status(), {'set': False, 'from': None, 'rejected': False}) + github._api('/repos/a/b/releases') + self.assertEqual(self.web.seen[-1][1], None) + + def test_bad_tokens_are_refused(self): + for token in ('', ' ', 'two words', 'x\nAuthorization: y'): + with self.subTest(token=token), self.assertRaises(SourceError): + github.set_token(token) + self.assertFalse(self.credentials_file().exists()) + + +class Endpoints(Repository): + """The server API: logins in, never out, in responses or the server log.""" + + def setUp(self): + super().setUp() + mock = patch.object(search, 'modules', return_value=([fdroid, github], [])) + mock.start() + self.addCleanup(mock.stop) + self.httpd = server.ThreadingHTTPServer(('127.0.0.1', 0), server.Handler) + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + self.addCleanup(self.httpd.server_close) + self.addCleanup(self.httpd.shutdown) + + def request(self, method, path, body=None): + c = http.client.HTTPConnection('127.0.0.1', self.httpd.server_port) + c.request(method, path, json.dumps(body) if body is not None else None, + {'X-Frame-UI': '1', 'Content-Type': 'application/json'}) + r = c.getresponse() + raw = r.read().decode() + c.close() + self.replies.append(raw) + return r.status, json.loads(raw) + + def job(self, reply): + for _ in range(500): + job = self.request('GET', '/api/job?id=' + reply['job'])[1] + if job['done']: + return job + time.sleep(.01) + self.fail('job did not finish') + + def test_add_with_login_and_token_settings(self): + self.replies = [] + bad = self.job(self.request('POST', '/api/sources', {'action': 'add', 'url': REPO, 'fingerprint': PIN, + 'username': USER, 'password': 'wrong'})[1]) + self.assertIn('check the username and password', bad['error']) + self.assertEqual(fdroid.user_repos(), []) + code, reply = self.request('POST', '/api/sources', {'action': 'add', 'url': REPO, 'fingerprint': PIN, + 'username': USER, 'password': PASSWORD}) + self.assertEqual(code, 200) + job = self.job(reply) + self.assertIsNone(job['error']) + listed = self.request('GET', '/api/sources')[1]['sources'] + repo = next(s for s in listed if s['id'] == job['result']['source']['id']) + self.assertEqual((repo['url'], repo['login']), (REPO, True)) + self.assertEqual(self.request('POST', '/api/sources', {'action': 'set-token', 'source': 'github', + 'token': TOKEN})[0], 200) + github_row = next(s for s in self.request('GET', '/api/sources')[1]['sources'] if s['id'] == 'github') + self.assertEqual(github_row['token'], {'set': True, 'from': 'saved', 'rejected': False}) + self.assertEqual(self.request('POST', '/api/sources', {'action': 'set-token', 'source': 'fdroid', + 'token': TOKEN})[0], 400) + self.assertEqual(self.request('POST', '/api/sources', {'action': 'clear-token', 'source': 'github'})[0], 200) + self.assertFalse(_credentials.has('github')) + self.assertEqual(self.request('POST', '/api/sources', {'action': 'remove', 'source': repo['id']})[0], 200) + self.assertFalse(self.credentials_file().exists()) + self.assertNoSecrets(*self.replies, self.stderr.getvalue(), self.disk()) + + def test_login_in_the_url_is_accepted(self): + self.replies = [] + url = 'https://alice:s3cr%40t-p%C3%A4ssword@repo.example/fdroid/repo' + job = self.job(self.request('POST', '/api/sources', {'action': 'add', 'url': url, 'fingerprint': PIN})[1]) + self.assertIsNone(job['error']) + self.assertEqual(fdroid.user_repos()[0]['url'], REPO) + self.assertNoSecrets(*self.replies, self.stderr.getvalue()) + + +if __name__ == '__main__': + unittest.main() diff --git a/ui/apk_sources/__init__.py b/ui/apk_sources/__init__.py index f7e6b05..be9648c 100644 --- a/ui/apk_sources/__init__.py +++ b/ui/apk_sources/__init__.py @@ -45,6 +45,10 @@ class SourceError(Exception): """User-readable failure from a source (network, format, verification).""" +class SourceAuthError(SourceError): + """The server refused our login (HTTP 401, or 403 when a login was sent).""" + + class SourceLimited(SourceError): """The source's host asked us to slow down; retry_after is in seconds.""" diff --git a/ui/apk_sources/_credentials.py b/ui/apk_sources/_credentials.py new file mode 100644 index 0000000..063b837 --- /dev/null +++ b/ui/apk_sources/_credentials.py @@ -0,0 +1,198 @@ +"""Logins for sources behind one: HTTP basic auth and access tokens. + +Saved apart from the source settings, in frame_host.data_dir('source-credentials.json') +(mode 0600), keyed by a caller-chosen name: an F-Droid source id, 'github'. Each +credential belongs to one exact origin (https, host, port) and is only ever sent +to a request for that origin: Handler adds it to each request, redirects +included, and removes any Authorization header from every other request. + +Secrets leave this module only as an Authorization header value. Credential's +repr hides them, and nothing here puts them in an exception message. +""" +import base64 +import json +import os +import tempfile +import threading +import urllib.parse +import urllib.request + +import frame_host + +_lock = threading.RLock() +MAX = 4096 # longest username, password or token accepted + + +def _path(): + return frame_host.data_dir('source-credentials.json') + + +def origin(url): + """'https://host:port' for an https URL; ValueError for anything else.""" + p = urllib.parse.urlsplit(url) + port = p.port # raises ValueError on a bad port + if p.scheme.lower() != 'https' or not p.hostname: + raise ValueError('credentials are only sent over HTTPS') + host = p.hostname.lower() + return 'https://%s:%d' % ('[%s]' % host if ':' in host else host, port or 443) + + +def _check(value, what, allow_empty=False): + if not isinstance(value, str) or (not value and not allow_empty) or len(value) > MAX: + raise ValueError('invalid ' + what) + if any(ord(c) < 32 or ord(c) == 127 for c in value): + raise ValueError(what + ' must not contain control characters') + return value + + +class Credential: + """A username and password (basic) or a token (bearer) for one origin. + + scope is a URL prefix used only to pick a stored credential for a URL (for_url); + whether a request gets the credential depends on its origin alone.""" + __slots__ = ('origin', 'scope', 'username', '_secret') + + def __init__(self, scope, secret, username=None): + self.origin = origin(scope) + self.scope = scope + if username is not None: + _check(username, 'username') + if ':' in username: + raise ValueError('username must not contain ":"') + _check(secret, 'password', allow_empty=True) + else: + _check(secret, 'token') + if any(c.isspace() for c in secret): + raise ValueError('token must not contain spaces') + self.username, self._secret = username, secret + + @property + def kind(self): + return 'bearer' if self.username is None else 'basic' + + def header(self): + if self.username is None: + return 'Bearer ' + self._secret + return 'Basic ' + base64.b64encode((self.username + ':' + self._secret).encode('utf-8')).decode('ascii') + + def matches(self, url): + """True only for a URL on exactly this credential's origin.""" + try: + return origin(url) == self.origin + except ValueError: + return False + + def same(self, other): + return isinstance(other, Credential) and (self.origin, self.username, self._secret) == ( + other.origin, other.username, other._secret) + + def __repr__(self): + return '' % (self.kind, self.origin) + + def _json(self): + return {'scope': self.scope, 'username': self.username, 'secret': self._secret} + + +def basic(scope, username, password): + """Basic auth for scope's origin, e.g. basic('https://example.org/fdroid/repo/', 'me', 'pw').""" + return Credential(scope, password, username) + + +def bearer(scope, token): + """An access token for scope's origin, sent as 'Authorization: Bearer '.""" + return Credential(scope, token) + + +def _read(): + try: + with open(str(_path()), encoding='utf-8') as f: + data = json.load(f) + if not isinstance(data, dict): + raise ValueError + return data + except FileNotFoundError: + return {} + except (OSError, ValueError): + return {} # unreadable: as if none were saved; the next save rewrites it + + +def _save(data): + path = _path() + if not data: + try: + os.remove(str(path)) + except FileNotFoundError: + pass + return + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix='.credentials-', suffix='.part') # created 0600 + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + json.dump(data, f, separators=(',', ':')) + os.chmod(tmp, 0o600) + os.replace(tmp, str(path)) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def get(key): + """The saved Credential for key, or None.""" + with _lock: + item = _read().get(key) + try: + return Credential(item['scope'], item['secret'], item.get('username')) + except (TypeError, KeyError, ValueError): + return None + + +def has(key): + return get(key) is not None + + +def put(key, credential): + if not isinstance(credential, Credential): + raise TypeError('put() needs a Credential') + with _lock: + data = _read() + data[key] = credential._json() + _save(data) + + +def remove(key): + """Delete key's credential from disk; True if there was one.""" + with _lock: + data = _read() + if key not in data: + return False + del data[key] + _save(data) + return True + + +def for_url(url): + """A saved credential whose origin matches url and whose scope url is under, or None.""" + with _lock: + keys = list(_read()) + for key in keys: + credential = get(key) + if credential and credential.matches(url) and url.startswith(credential.scope): + return credential + return None + + +class Handler(urllib.request.BaseHandler): + """Adds the credential to requests for its origin (redirects included) and strips + Authorization from every other request, so a redirect elsewhere never gets it.""" + handler_order = 900 # after the default processors + + def __init__(self, credential): + self.credential = credential + + def _request(self, req): + req.remove_header('Authorization') + if self.credential is not None and self.credential.matches(req.full_url): + req.add_unredirected_header('Authorization', self.credential.header()) + return req + + http_request = https_request = _request diff --git a/ui/apk_sources/_images.py b/ui/apk_sources/_images.py index da89453..b0244d5 100644 --- a/ui/apk_sources/_images.py +++ b/ui/apk_sources/_images.py @@ -9,7 +9,7 @@ import threading import time from urllib.parse import urljoin, urlsplit -from apk_sources import SourceError +from apk_sources import SourceError, _credentials _lock = threading.Lock() _urls = OrderedDict() @@ -141,7 +141,11 @@ def get(url, headers=None, redirects=3, deadline=None, limit=MAX_IMAGE): path = p.path or '/' if p.query: path += '?' + p.query - conn.request('GET', path, headers={'User-Agent': 'FrameControl/0.3.1', **(headers or {})}) + sent = {'User-Agent': 'FrameControl/0.3.1', **(headers or {})} + login = _credentials.for_url(url) # a repository behind a login: its own artwork only, per hop + if login: + sent['Authorization'] = login.header() + conn.request('GET', path, headers=sent) live[0].settimeout(min(10, left())) response = conn.getresponse() if response.status in (301, 302, 303, 307, 308) and redirects: diff --git a/ui/apk_sources/_web.py b/ui/apk_sources/_web.py index 1952e7d..d24207d 100644 --- a/ui/apk_sources/_web.py +++ b/ui/apk_sources/_web.py @@ -3,7 +3,7 @@ import hashlib, os, shutil, tempfile, threading, time, urllib.error, urllib.pars from email.utils import parsedate_to_datetime import frame_host -from . import SourceError, SourceLimited +from . import SourceAuthError, SourceError, SourceLimited, _credentials UA = 'FrameControl/0.1' APK_CAP = 2 * 1024 ** 3 # cached APKs across all sources, least recently used go first @@ -136,6 +136,8 @@ def checked_url(url, hosts): class Redirect(urllib.request.HTTPRedirectHandler): + """Only to the allowed hosts, and never carrying an Authorization header along: + a credential passed to open_url() is added again, by its Handler, only on its origin.""" def __init__(self, hosts): self.hosts = hosts @@ -147,13 +149,15 @@ class Redirect(urllib.request.HTTPRedirectHandler): return result -def open_url(url, hosts, headers=None): +def open_url(url, hosts, headers=None, auth=None): + """auth: an _credentials.Credential, sent only to requests on its own origin.""" checked_url(url, hosts) - return urllib.request.build_opener(Redirect(hosts)).open( + handlers = [Redirect(hosts)] + ([_credentials.Handler(auth)] if auth else []) + return urllib.request.build_opener(*handlers).open( urllib.request.Request(url, headers={'User-Agent': UA, **(headers or {})}), timeout=60) -def read(url, hosts, headers=None, ttl=3600, name=None, hint=''): +def read(url, hosts, headers=None, ttl=3600, name=None, hint='', auth=None): path = os.path.join(cache(), hashlib.sha256(url.encode()).hexdigest() + '.data') name = name or _host(url) or 'The source' @@ -171,7 +175,7 @@ def read(url, hosts, headers=None, ttl=3600, name=None, hint=''): if data is None: raise limited_error(name, wait, hint) return data - with open_url(url, hosts, headers) as r: + with open_url(url, hosts, headers, auth) as r: data = r.read(8 * 1024 * 1024 + 1) if len(data) > 8 * 1024 * 1024: raise SourceError('Source index is too large') @@ -185,6 +189,8 @@ def read(url, hosts, headers=None, ttl=3600, name=None, hint=''): os.remove(tmp) return data except urllib.error.HTTPError as e: + if e.code == 401: # not 403: GitHub also uses that for rate limits, token or not + raise SourceAuthError('%s refused the login (HTTP 401)' % name) from e if e.code in (403, 429): delay = throttle(url, e.headers) data = cached() diff --git a/ui/apk_sources/fdroid.py b/ui/apk_sources/fdroid.py index ff02cb4..1779bf3 100644 --- a/ui/apk_sources/fdroid.py +++ b/ui/apk_sources/fdroid.py @@ -20,7 +20,7 @@ import zipfile if __package__ in (None, ''): sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from apk_sources import SourceError, SourceLimited, _web +from apk_sources import SourceAuthError, SourceError, SourceLimited, _credentials, _web import frame_host from frame_apk_sign import _der_parts, _cert_key, der from frame_catalog import _IndexReader, _reduce_index, _sha256 @@ -71,6 +71,39 @@ def _url(url, fingerprint=None): return urllib.parse.urlunsplit(('https', p.netloc.lower(), p.path.rstrip('/') + '/', '', '')), pin +def _split_login(url, username=None, password=None): + """(url without user:pass, credential or None). The address may carry the login, as + F-Droid's client accepts (https://user:pass@host/fdroid/repo), or it may be given apart.""" + try: + p = urllib.parse.urlsplit(url) + p.port + except ValueError as e: + raise SourceError('invalid repository URL') from e + if '@' in p.netloc: + if p.username is None: + raise SourceError('invalid repository URL') + linked = urllib.parse.unquote(p.username), None if p.password is None else urllib.parse.unquote(p.password) + if username not in (None, '', linked[0]) or (linked[1] is not None and password not in (None, '', linked[1])): + raise SourceError('the address and the username/password fields disagree') + username = linked[0] + password = linked[1] if linked[1] is not None else password + url = urllib.parse.urlunsplit((p.scheme, p.netloc.rpartition('@')[2], p.path, p.query, p.fragment)) + if username in (None, ''): + if password: + raise SourceError('a password needs a username') + return url, None + clean, _ = _url(url) + try: + return url, _credentials.basic(clean, username, password or '') + except ValueError as e: + raise SourceError('cannot use that login: ' + str(e)) from None + + +def _login(source): + """The saved login for a user repository, or None.""" + return None if source.get('builtin') else _credentials.get(source['id']) + + def _child(base, name): name = str(name).lstrip('/') decoded = urllib.parse.unquote(name) @@ -91,14 +124,16 @@ class _HTTPSRedirect(urllib.request.HTTPRedirectHandler): return super().redirect_request(req, fp, code, msg, headers, newurl) -def _fetch(url, path, maximum): +def _fetch(url, path, maximum, auth=None): + """auth: the repository's login, sent only to its exact origin (never after a redirect elsewhere).""" host = urllib.parse.urlsplit(url).hostname wait = _web.wait_time(url) if wait: raise _web.limited_error(host, wait) request = urllib.request.Request(url, headers={'User-Agent': 'FrameControl/1.0'}) + handlers = [_HTTPSRedirect()] + ([_credentials.Handler(auth)] if auth else []) try: - with urllib.request.build_opener(_HTTPSRedirect()).open(request, timeout=60) as r, open(path, 'wb') as f: + with urllib.request.build_opener(*handlers).open(request, timeout=60) as r, open(path, 'wb') as f: total = 0 while True: chunk = r.read(1 << 20) @@ -109,6 +144,9 @@ def _fetch(url, path, maximum): raise SourceError('repository file exceeds size limit') f.write(chunk) except urllib.error.HTTPError as e: + if e.code == 401 or (e.code == 403 and auth): + raise SourceAuthError('The repository refused the login (HTTP %d); check the username and password' + % e.code if auth else 'This repository needs a username and password') from None if e.code in (403, 429): raise _web.limited_error(host, _web.throttle(url, e.headers)) from e raise @@ -342,7 +380,7 @@ def sources(): return [dict(id=i, kind=KIND, name=n, url=u, fingerprint=p, builtin=True, # The archive only holds superseded versions; it clutters search unless asked for. enabled=settings['enabled'].get(i, i != 'fdroid-archive'), trust='community') - for i, n, u, p in builtins] + settings['repos'] + for i, n, u, p in builtins] + [dict(s, login=_credentials.has(s['id'])) for s in settings['repos']] def _text(value): @@ -514,10 +552,12 @@ def _source_lock(source_id): return _load_locks.setdefault(source_id, threading.Lock()) -def _load(source, force=False): +def _load(source, force=False, auth=None): + """auth: a login to use instead of the saved one (while adding, before anything is saved).""" if not re.fullmatch(r'[a-z0-9-]+', source['id']): raise SourceError('invalid source id') _url(source['url'], source.get('fingerprint')) + auth = auth or _login(source) cache = frame_host.cache_dir('apk-sources', source['id'] + '.json') found = None if force else _cached(source, cache) if found: @@ -535,7 +575,7 @@ def _load(source, force=False): jar, raw = Path(tmp) / 'index.jar', Path(tmp) / 'index.json' v2 = True try: - _fetch(source['url'] + 'entry.jar', jar, 8 * 1024 * 1024) + _fetch(source['url'] + 'entry.jar', jar, 8 * 1024 * 1024, auth=auth) except urllib.error.HTTPError as e: if e.code not in (404, 410): raise @@ -543,7 +583,7 @@ def _load(source, force=False): raise SourceError('repository no longer serves its signed v2 index; ' 'refusing to fall back to the older v1 index') from e v2 = False - _fetch(source['url'] + 'index-v1.jar', jar, 256 * 1024 * 1024) + _fetch(source['url'] + 'index-v1.jar', jar, 256 * 1024 * 1024, auth=auth) content, pin = _jar(jar, 'index-v1.json', source.get('fingerprint')) timestamp = _v1(content, raw) _check_timestamp(source, timestamp) @@ -552,7 +592,7 @@ def _load(source, force=False): signed = json.loads(content) timestamp, entry = signed.get('timestamp'), signed['index'] _check_timestamp(source, timestamp) - _fetch(_child(source['url'], entry['name']), raw, 512 * 1024 * 1024) + _fetch(_child(source['url'], entry['name']), raw, 512 * 1024 * 1024, auth=auth) if _sha256(raw) != entry['sha256'] or (entry.get('size') is not None and raw.stat().st_size != entry['size']): raise SourceError('index SHA-256 or size mismatch') apps = _reduce(raw, source) @@ -572,7 +612,10 @@ def _load(source, force=False): raise SourceError('cannot load repository: ' + str(e)) from e -def add_repo(url, fingerprint=None, name=None): +def add_repo(url, fingerprint=None, name=None, username=None, password=None): + """Check and save a repository. A login (in the URL or apart) is kept out of the + saved URL, in _credentials, and saved only once the repository's index verified.""" + url, login = _split_login(url, username, password) url, pin = _url(url, fingerprint) with _LOCK: existing = next((s for s in _read()['repos'] if s['url'] == url), None) @@ -583,15 +626,26 @@ def add_repo(url, fingerprint=None, name=None): source = dict(id='fdroid-user-' + hashlib.sha256(url.encode()).hexdigest()[:20], kind=KIND, name=name or (existing or {}).get('name') or urllib.parse.urlsplit(url).hostname, url=url, builtin=False, enabled=True, trust='user', fingerprint=pin) - _, source['fingerprint'] = _load(source, force=True) # network work outside the settings lock + # Network work outside the settings lock; without a new login, a saved one (re-adding) is used. + _, source['fingerprint'] = _load(source, force=True, auth=login) source['trust_on_first_use'] = existing.get('trust_on_first_use', False) if existing else pin is None with _LOCK: settings = _read() current = next((s for s in settings['repos'] if s['id'] == source['id']), None) if current and current['fingerprint'] != source['fingerprint']: raise SourceError('repository already has a different pinned fingerprint; remove it first') - settings['repos'] = [s for s in settings['repos'] if s['id'] != source['id']] + [source] - _write(_storage(), settings) + previous = _credentials.get(source['id']) + if login: + _credentials.put(source['id'], login) + try: + settings['repos'] = [s for s in settings['repos'] if s['id'] != source['id']] + [source] + _write(_storage(), settings) + except BaseException: + if login and previous: # not saved after all: leave the login as it was + _credentials.put(source['id'], previous) + elif login: + _credentials.remove(source['id']) + raise return source @@ -602,6 +656,7 @@ def remove_repo(source_id): raise SourceError('unknown user repository') settings['repos'] = [s for s in settings['repos'] if s['id'] != source_id] _write(_storage(), settings) + _credentials.remove(source_id) with _state_file_lock(), _LOCK: # lock order: state file, then _LOCK states = _states() if states.pop(source_id, None) is not None: # re-adding is a deliberate new trust decision @@ -663,7 +718,7 @@ def download(source, entry_id, version_code=None): fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix='.part') os.close(fd) try: - _fetch(_child(source['url'], version['name']), tmp, 4 * 1024 ** 3) + _fetch(_child(source['url'], version['name']), tmp, 4 * 1024 ** 3, auth=_login(source)) if _sha256(tmp) != sha: raise SourceError('APK SHA-256 mismatch; download discarded') os.replace(tmp, path) @@ -684,6 +739,8 @@ def main(): add.add_argument('url') add.add_argument('--fingerprint') add.add_argument('--name') + add.add_argument('--username', help='for a repository behind a login (HTTP basic auth)') + add.add_argument('--password-stdin', action='store_true', help='read the password from the first line of stdin') sub.add_parser('list') remove = sub.add_parser('remove') remove.add_argument('source') @@ -702,7 +759,12 @@ def main(): def _cli(parser, args): try: if args.command == 'add': - result = add_repo(args.url, args.fingerprint, args.name) + password = None + if args.password_stdin: + if not args.username: + raise SourceError('--password-stdin needs --username') + password = sys.stdin.readline().rstrip('\r\n') + result = add_repo(args.url, args.fingerprint, args.name, args.username, password) elif args.command == 'list': result = sources() elif args.command == 'remove': diff --git a/ui/apk_sources/github.py b/ui/apk_sources/github.py index e636f2a..a12ed57 100644 --- a/ui/apk_sources/github.py +++ b/ui/apk_sources/github.py @@ -1,17 +1,77 @@ """Curated publisher releases; optional topic discovery is page-only.""" -import fnmatch, json, os, re, urllib.parse +import fnmatch, hashlib, json, os, re, sys, threading, urllib.parse -from . import SourceError, _web +from . import SourceAuthError, SourceError, _credentials, _web KIND = 'github' API = 'https://api.github.com' +ENV = 'FRAME_GITHUB_TOKEN' # wins over a token saved in source settings +CREDENTIAL = 'github' # _credentials key; only ever sent to api.github.com +_rejected = set() # SHA-256 of tokens GitHub answered 401 to (this run only) +_rejected_lock = threading.Lock() TOPICS = ('oculus-quest', 'openxr', 'quest') HOSTS = ('github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com') def sources(): return [{'id': KIND, 'kind': KIND, 'name': 'GitHub releases', 'url': 'https://github.com', - 'builtin': True, 'enabled': True, 'trust': 'community'}] + 'builtin': True, 'enabled': True, 'trust': 'community', 'token': token_status()}] + + +def _token(): + """(token, where it came from) or (None, None). The environment variable wins.""" + env = os.environ.get(ENV) + if env: + return env, 'env' + saved = _credentials.get(CREDENTIAL) + if saved and saved.kind == 'bearer': + return saved, 'saved' + return None, None + + +def _credential(token): + if isinstance(token, _credentials.Credential): + return token + try: + return _credentials.bearer(API + '/', token) + except ValueError as e: + raise SourceError('%s is not a usable GitHub token: %s' % (ENV, e)) from None + + +def _digest(credential): + return hashlib.sha256(credential.header().encode()).hexdigest() + + +def token_status(): + """Whether a token is set, where from, and whether GitHub rejected it; never the token.""" + token, where = _token() + rejected = False + if token is not None: + try: + with _rejected_lock: + rejected = _digest(_credential(token)) in _rejected + except SourceError: + rejected = True + return {'set': token is not None, 'from': where, 'rejected': rejected} + + +def set_token(token): + """Save a personal access token for api.github.com (FRAME_GITHUB_TOKEN still wins).""" + if not isinstance(token, str) or not token.strip(): + raise SourceError('Paste a GitHub token') + try: + credential = _credentials.bearer(API + '/', token.strip()) + except ValueError as e: + raise SourceError('That is not a usable GitHub token: ' + str(e)) from None + _credentials.put(CREDENTIAL, credential) + with _rejected_lock: + _rejected.discard(_digest(credential)) # a fresh chance + return {'message': 'GitHub token saved' + (' (%s is set and is used instead)' % ENV if os.environ.get(ENV) else '')} + + +def clear_token(): + _credentials.remove(CREDENTIAL) + return {'message': 'GitHub token removed'} def _curated(): @@ -20,13 +80,33 @@ def _curated(): def _api(path): + """GET from the GitHub API, with the user's token if there is one. If GitHub rejects + the token (401), retry once without it, like Obtainium, and remember the rejection.""" headers = {'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'} - token = os.environ.get('FRAME_GITHUB_TOKEN') - if token: - headers['Authorization'] = 'Bearer ' + token + token, _ = _token() + auth = _credential(token) if token is not None else None + with _rejected_lock: + if auth and _digest(auth) in _rejected: + auth = None # already refused this run: don't spend a request on it again + hint = ' (add a GitHub token in the store\'s source settings, or set %s, to raise the limit)' % ENV + if token is not None and not auth: + hint = ' (GitHub rejected your token; replace it in the store\'s source settings)' + + def get(auth): + return _web.read(API + path, ('api.github.com',), headers, name='GitHub', + hint='' if auth else hint, auth=auth) try: - return json.loads(_web.read(API + path, ('api.github.com',), headers, name='GitHub', - hint='' if token else ' (set FRAME_GITHUB_TOKEN to raise the limit)')) + try: + data = get(auth) + except SourceAuthError: + if not auth: + raise + with _rejected_lock: + _rejected.add(_digest(auth)) + print('GitHub rejected the access token (HTTP 401); continuing without it', file=sys.stderr) + hint = ' (GitHub rejected your token; replace it in the store\'s source settings)' + data = get(None) + return json.loads(data) except (ValueError, TypeError) as e: raise SourceError('Invalid GitHub response') from e diff --git a/ui/apk_sources/search.py b/ui/apk_sources/search.py index 883c08b..e987f03 100644 --- a/ui/apk_sources/search.py +++ b/ui/apk_sources/search.py @@ -113,6 +113,14 @@ def manage_repo(action, **kwargs): return getattr(module, action)(**kwargs) +def set_token(source_id, token): + """Save (token) or remove (None) a source's access token; status is in sources().""" + module, _ = resolve(source_id) + if not hasattr(module, 'set_token'): + raise SourceError('This source does not use an access token') + return module.set_token(token) if token is not None else module.clear_token() + + def normalise(name): return ' '.join(re.findall(r'\w+', unicodedata.normalize('NFKC', name or '').casefold())) diff --git a/ui/index.html b/ui/index.html index 8c5a705..62d93ee 100644 --- a/ui/index.html +++ b/ui/index.html @@ -611,6 +611,8 @@ .source-repo-form input { margin-top:7px; display:block; border-radius:6px; } .source-repo-form details { font-size:11px; color:var(--muted); } .source-repo-form button { justify-self:start; border-radius:6px; } + .source-token { display:flex; flex-wrap:wrap; align-items:center; gap:7px; margin:8px 0 4px; } + .source-token input { flex:1; min-width:150px; border-radius:6px; font-size:12px; } #appStore :focus-visible, .store-dialog :focus-visible, .store-tabs :focus-visible { outline:2px solid var(--link); outline-offset:4px; } .store-search input:focus-visible { outline:0!important; } @media(min-width:1400px) { .store-grid { grid-template-columns:repeat(5,minmax(0,1fr)); } } @@ -1207,7 +1209,7 @@
Your app shelves

A world of apps.

Choose where you discover games and apps. We bring them together for you.

-

Add a source

Have a favorite F-Droid repository? Add its address to browse its apps here.

Verify this source (optional)
+

Add a source

Have a favorite F-Droid repository? Add its address to browse its apps here.

Verify this source (optional)
This repository needs a login (optional)

Kept on this computer, apart from the address, and only sent to this repository over HTTPS.

@@ -3139,12 +3141,18 @@ function storeIcon(o) { return ` { if (e.target.matches?.(".store-art img, .store-icon img")) e.target.remove(); }, true); function sourceMark(s) { const name=storeName(s); return ``; } function sourceDescription(s) { return s.description || ({fdroid:"Independent, open-source apps",sidequest:"Discover something new in VR",itch:"Games from independent creators",github:"Direct from the people who make it"})[s.kind] || (s.name.includes("GitHub") ? "Direct from the people who make it" : s.name.includes("Droid") ? "Independent, open-source apps" : s.name.includes("SideQuest") ? "Discover something new in VR" : s.name.includes("itch") ? "Games from independent creators" : "Apps shared by this community"); } +// GitHub's optional access token: only whether one is set is ever shown, never its value. +function sourceToken(s) { + const t=s.token;if(!t)return ""; + const note=t.rejected ? "GitHub rejected this token, so searches continue without it. Replace or clear it." : t.from==="env" ? "Using the token from FRAME_GITHUB_TOKEN." : t.set ? "A token is saved." : "Optional: a token raises GitHub's limit of 60 requests an hour."; + return `
${t.from==="saved" ? `` : ""}
${esc(note)}`; +} async function loadSources() { try { const {sources} = await api("/api/sources"); sourceState.sources = sources; $("sourceFilter").innerHTML = `` + sources.filter(s=>s.enabled).map(s=>``).join(""); $("sourceFilter").value = sourceState.source; - $("sourcesList").innerHTML = sources.map(s=>`
${sourceMark(s.name)}
${esc(storeName(s.name))}

${esc(sourceDescription(s))}

${esc(({official:"Official store",community:"Community",user:"Added by you"})[s.trust] || "Community")}${s.builtin === false ? `` : ""}
`).join("") || `

Your sources will appear here when they are available.

`; + $("sourcesList").innerHTML = sources.map(s=>`
${sourceMark(s.name)}
${esc(storeName(s.name))}

${esc(sourceDescription(s))}

${esc(({official:"Official store",community:"Community",user:"Added by you"})[s.trust] || "Community")}${s.login ? " · Uses a login" : ""}${sourceToken(s)}${s.builtin === false ? `` : ""}
`).join("") || `

Your sources will appear here when they are available.

`; } catch (e) { $("sourceError").textContent = e.message; } } function storeCard(app) { @@ -3271,7 +3279,8 @@ $("detailBody").onclick=e=>{if(e.target.closest('[data-close]'))return $("appDet $("detailBody").onchange=e=>{if(e.target.name==='storeOffer'){sourceState.detail.choice=+e.target.value;renderAppDetail();loadOfferDetails();}}; $("sourcesOpen").onclick=()=>{loadSources();$('sourcesDialog').showModal();}; $("sourcesList").onchange=e=>{if(e.target.matches('[data-enable]'))changeSource({action:'enable',source:e.target.dataset.enable,enabled:e.target.checked});}; -$("sourcesList").onclick=e=>{const b=e.target.closest('[data-remove]');if(b)changeSource({action:'remove',source:b.dataset.remove});}; +$("sourcesList").onclick=e=>{const b=e.target.closest('[data-remove]');if(b)changeSource({action:'remove',source:b.dataset.remove});const c=e.target.closest('[data-clear-token]');if(c)changeSource({action:'clear-token',source:c.dataset.clearToken});}; +$("sourcesList").onsubmit=e=>{const f=e.target.closest('[data-token]');if(!f)return;e.preventDefault();const token=f.elements.token.value.trim();f.reset();if(token)changeSource({action:'set-token',source:f.dataset.token,token});}; async function addSource(body) { $("sourceError").textContent="Checking the repository. Large ones can take a minute…"; let error=""; @@ -3279,7 +3288,12 @@ async function addSource(body) { $("sourceError").textContent=result ? result.message : error || "We couldn't add that repository."; if(result){$("addSource").reset();sourceState.source='';await loadSources();await searchSources();} } -$("addSource").onsubmit=e=>{e.preventDefault();const f=new FormData(e.target);addSource({action:'add',url:f.get('url'),fingerprint:f.get('fingerprint')});}; +$("addSource").onsubmit=e=>{ + e.preventDefault();const f=new FormData(e.target);let url=f.get('url'),username=f.get('username'),password=f.get('password'); + // A login typed into the address moves to the login fields: the address is shown and logged, the login never. + try { const u=new URL(url); if(u.username||u.password){ username=username||decodeURIComponent(u.username); password=password||decodeURIComponent(u.password); u.username='';u.password='';url=u.href; } } catch {} + addSource({action:'add',url,fingerprint:f.get('fingerprint'),...(username ? {username,password} : {})}); +}; document.querySelectorAll('[data-close]').forEach(b=>b.onclick=()=>$(b.dataset.close).close()); $('appDetail').addEventListener('close',()=>{sourceState.detailRequest++;}); document.querySelectorAll('[data-store-tab]').forEach(b=>b.onclick=()=>{const browse=b.dataset.storeTab==='browse';$('appStore').hidden=!browse;$('androidLibrary').hidden=browse;$('repNew').hidden=browse;$('sourcesOpen').hidden=!browse;document.querySelectorAll('[data-store-tab]').forEach(t=>{t.classList.toggle('on',t===b);t.setAttribute('aria-pressed',t===b);});if(!browse)loadAndroid();}); diff --git a/ui/server.py b/ui/server.py index 07ca9f3..0d30986 100755 --- a/ui/server.py +++ b/ui/server.py @@ -2153,6 +2153,16 @@ def source_text(body, key, optional=False): return value.strip() +def login_text(body, key): + """A username, password or token: kept exactly (no stripping) and never echoed back.""" + value = body.get(key) + if value in (None, ''): + return None + if not isinstance(value, str) or len(value) > 4096: + raise Failure('Provide a valid ' + key, 400) + return value + + def source_search(query): args = parse_qs(query) q = args.get('q', [''])[0] @@ -2191,19 +2201,31 @@ def source_manage(body): if action == 'add': url = source_text(body, 'url') fingerprint, name = source_text(body, 'fingerprint', True), source_text(body, 'name', True) - parts = urlparse(url) - if parts.scheme not in ('https', 'fdroidrepos') or not parts.hostname or parts.username: - raise Failure('Use an HTTPS or fdroidrepos:// repository URL without credentials', 400) + username, password = login_text(body, 'username'), login_text(body, 'password') + try: + parts = urlparse(url) + parts.port + except ValueError: + raise Failure('Use an HTTPS or fdroidrepos:// repository URL', 400) from None + if parts.scheme not in ('https', 'fdroidrepos') or not parts.hostname: + raise Failure('Use an HTTPS or fdroidrepos:// repository URL', 400) apk_search.repo_module() # fail now if this build can't manage repositories def add(): # downloads and verifies the whole index: a job, not a request - source = apk_search.manage_repo('add_repo', url=url, fingerprint=fingerprint, name=name) + # A login (here or in the URL) is stored apart and never comes back in a response. + login = {'username': username, 'password': password} if username or password else {} + source = apk_search.manage_repo('add_repo', url=url, fingerprint=fingerprint, name=name, **login) message = 'Added ' + source['name'] if source.get('trust_on_first_use'): message += '. Trusted on first use: ' + source['fingerprint'].upper() return {'message': message, 'source': {k: source.get(k) for k in ('id', 'name', 'fingerprint', 'trust_on_first_use')}} return start_job('Add repository', add) + if action in ('set-token', 'clear-token'): + token = login_text(body, 'token') if action == 'set-token' else None + if action == 'set-token' and not token: + raise Failure('Paste a token', 400) + return apk_search.set_token(source_text(body, 'source'), token) if action == 'game-data': package = source_text(body, 'package') return start_job('Add game data', lambda: apk_search.add_game_data(package))