Add generic APK URL source

tests/test_sideload.py

@@ -9,6 +9,7 @@ import subprocess
 import sys
 import urllib.error
 import urllib.request
+from contextlib import contextmanager
 from pathlib import Path
 from unittest import mock
 
@@ -43,6 +44,22 @@ def _mock_urlopen_response(data: bytes):
     return m
 
 
+def _generic_response(data: str | bytes, url: str):
+    response = _mock_urlopen_response(data.encode() if isinstance(data, str) else data)
+    response.geturl.return_value = url
+    return response
+
+
+@contextmanager
+def _mock_download_opener(response=None, *, error=None):
+    with mock.patch("sideload._generic_opener") as opener_factory:
+        if error is not None:
+            opener_factory.return_value.open.side_effect = error
+        else:
+            opener_factory.return_value.open.return_value = response
+        yield
+
+
 # ── fixtures ──────────────────────────────────────────────────────────────────
 
 
@@ -212,6 +229,20 @@ asset_pattern = ".*\\\\.apk$"
         assert cfg.apps[0].app_id == "org.schabi.newpipe"
         assert cfg.apps[0].config["url"].startswith("https://github.com/")
 
+    def test_generic_requires_https_url_and_asset_pattern(self, tmp_path: Path):
+        p = _write_config(
+            tmp_path / "cfg.toml",
+            '[[apps]]\nname="X"\nsource="generic"\npackage="x"\nurl="http://example.test/"\nasset_pattern=".*\\\\.apk"',
+        )
+        with pytest.raises(ValueError, match="absolute HTTPS"):
+            sideload.Config.from_file(p)
+        p.write_text(
+            '[[apps]]\nname="X"\nsource="generic"\npackage="x"\n'
+            'url="https://example.test/"\nasset_pattern=".*\\\\.apk"\nsha256="not-a-sha"'
+        )
+        with pytest.raises(ValueError, match="64 hexadecimal"):
+            sideload.Config.from_file(p)
+
 
 # ── DownloadCache tests ───────────────────────────────────────────────────────
 
@@ -375,7 +406,7 @@ class TestFdroidSource:
             expected_sha256=wrong_sha,
         )
         mock_resp = _mock_urlopen_response(apk_data)
-        with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+        with _mock_download_opener(mock_resp):
             with pytest.raises(ValueError, match="SHA-256 mismatch"):
                 source.download_apk(release, cache)
 
@@ -435,7 +466,7 @@ class TestGitHubSource:
         )
         apk_data = b"github-apk-content"
         mock_resp = _mock_urlopen_response(apk_data)
-        with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+        with _mock_download_opener(mock_resp):
             result = source.download_apk(release, cache)
         assert result.is_file()
         assert result.read_bytes() == apk_data
@@ -581,7 +612,7 @@ class TestGitLabSource:
         )
         apk_data = b"gitlab-apk-content"
         mock_resp = _mock_urlopen_response(apk_data)
-        with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+        with _mock_download_opener(mock_resp):
             result = source.download_apk(release, cache)
         assert result.is_file()
         assert result.read_bytes() == apk_data
@@ -590,6 +621,148 @@ class TestGitLabSource:
 # ── ApkValidator tests ────────────────────────────────────────────────────────
 
 
+class TestGenericSource:
+    def _entry(self, **config):
+        return sideload.AppEntry(
+            name="Firefox", source="generic", app_id="org.mozilla.firefox",
+            config={
+                "url": "https://downloads.example/releases/",
+                "intermediate_patterns": [r"(\d+)\/$", "android/$"],
+                "asset_pattern": r"^fenix-.*\.apk$",
+                **config,
+            },
+        )
+
+    def test_directory_navigation_and_numeric_version_selection(self):
+        entry = self._entry()
+        pages = [
+            _generic_response('<a href="1/">1</a><a href="10/">10</a><a href="http://bad/">bad</a>', entry.config["url"]),
+            _generic_response('<a href="android/">android</a>', "https://downloads.example/releases/10/"),
+            _generic_response('<a href="fenix-1.multi.android-arm64-v8a.apk?download=1">apk</a>', "https://downloads.example/releases/10/android/"),
+        ]
+        with mock.patch("sideload._generic_opener") as opener_factory:
+            opener_factory.return_value.open.side_effect = pages
+            release = sideload.GenericSource(entry).get_latest_release()
+        assert release.download_url.endswith(".apk?download=1")
+        assert release.version == "fenix-1.multi.android-arm64-v8a.apk"
+
+    def test_ambiguous_intermediate_links(self):
+        entry = self._entry(intermediate_patterns=[r"release/$"])
+        page = _generic_response('<a href="a/release/">a</a><a href="b/release/">b</a>', entry.config["url"])
+        with mock.patch("sideload._generic_opener") as opener_factory:
+            opener_factory.return_value.open.return_value = page
+            with pytest.raises(ValueError, match="multiple intermediate"):
+                sideload.GenericSource(entry).get_latest_release()
+
+    def test_excludes_debug_and_rejects_http_assets(self):
+        entry = self._entry(intermediate_patterns=[], asset_pattern=r"^good\.apk$")
+        page = _generic_response(
+            '<a href="http://downloads.example/good.apk">http</a>'
+            '<a href="good-debug.apk">debug</a><a href="good.apk">good</a>',
+            entry.config["url"],
+        )
+        with mock.patch("sideload._generic_opener") as opener_factory:
+            opener_factory.return_value.open.return_value = page
+            release = sideload.GenericSource(entry).get_latest_release()
+        assert release.download_url.endswith("good.apk")
+
+    def test_initial_http_and_network_error(self):
+        bad = self._entry(url="http://downloads.example/releases/")
+        with pytest.raises(ValueError, match="absolute HTTPS"):
+            sideload.GenericSource(bad)
+        entry = self._entry()
+        with mock.patch("sideload._generic_opener") as opener_factory:
+            opener_factory.return_value.open.side_effect = urllib.error.URLError("offline")
+            with pytest.raises(RuntimeError, match="network error"):
+                sideload.GenericSource(entry).get_latest_release()
+
+    def test_headers_are_sent_but_not_exposed(self, capsys):
+        entry = self._entry(intermediate_patterns=[], asset_pattern=r"^good\.apk$", headers={"Authorization": "secret-token"})
+        response = _generic_response('<a href="good.apk">good</a>', entry.config["url"])
+        with mock.patch("sideload._generic_opener") as opener_factory:
+            opener_factory.return_value.open.return_value = response
+            sideload.GenericSource(entry).get_latest_release()
+        request = opener_factory.return_value.open.call_args.args[0]
+        assert request.get_header("Authorization") == "secret-token"
+        captured = capsys.readouterr()
+        assert "secret-token" not in captured.out + captured.err
+
+    def test_https_redirect_is_blocked_before_following(self):
+        request = urllib.request.Request(
+            "https://downloads.example/start", headers={"Authorization": "secret-token"}
+        )
+        with pytest.raises(ValueError, match="non-HTTPS"):
+            sideload._HttpsOnlyRedirectHandler().redirect_request(
+                request, mock.Mock(), 302, "Found", "http://evil.example/", {}
+            )
+
+    def test_cross_host_redirect_is_blocked_and_same_origin_keeps_headers(self):
+        request = urllib.request.Request(
+            "https://downloads.example/start", headers={"Authorization": "secret-token"}
+        )
+        handler = sideload._HttpsOnlyRedirectHandler(
+            "https://downloads.example/start", same_origin=True
+        )
+        with pytest.raises(ValueError, match="changed origin"):
+            handler.redirect_request(
+                request, mock.Mock(), 302, "Found", "https://cdn.example/file", {}
+            )
+        redirected = handler.redirect_request(
+            request, mock.Mock(), 302, "Found", "https://downloads.example/file", {}
+        )
+        assert redirected.get_header("Authorization") == "secret-token"
+
+    def test_atomic_download_rejects_downgrade_before_response_body(self):
+        request = urllib.request.Request("https://downloads.example/file.apk")
+        handler = sideload._HttpsOnlyRedirectHandler(
+            "https://downloads.example/file.apk", same_origin=False
+        )
+        with pytest.raises(ValueError, match="non-HTTPS"):
+            handler.redirect_request(
+                request, mock.Mock(read=mock.Mock(side_effect=AssertionError)),
+                302, "Found", "http://downloads.example/file.apk", {}
+            )
+
+    def test_redirected_page_url_is_used_for_relative_links(self):
+        entry = self._entry(intermediate_patterns=[] , asset_pattern=r"^good\.apk$")
+        response = _generic_response('<a href="good.apk">good</a>', "https://cdn.example/final/")
+        with mock.patch("sideload._generic_opener") as opener_factory:
+            opener_factory.return_value.open.return_value = response
+            release = sideload.GenericSource(entry).get_latest_release()
+        assert release.download_url == "https://cdn.example/final/good.apk"
+
+    def test_generic_cache_uses_conditional_request_without_sha(self, tmp_path: Path):
+        entry = self._entry(intermediate_patterns=[] , asset_pattern=r"^good\.apk$")
+        source = sideload.GenericSource(entry)
+        release = sideload.ReleaseInfo(entry, "good.apk", "https://downloads.example/good.apk", None)
+        first = _generic_response(b"apk-content", release.download_url)
+        first.headers = {"ETag": "v1"}
+        first.getcode.return_value = 200
+        second = _generic_response(b"", release.download_url)
+        second.headers = {}
+        second.getcode.return_value = 304
+        with mock.patch("sideload._generic_opener") as opener_factory:
+            opener_factory.return_value.open.side_effect = [first, second]
+            cache = sideload.DownloadCache(tmp_path / "cache")
+            cached_first = source.download_apk(release, cache)
+            cached_second = source.download_apk(release, cache)
+        assert cached_second == cached_first
+        assert cached_second.read_bytes() == b"apk-content"
+        second_request = opener_factory.return_value.open.call_args_list[1].args[0]
+        assert second_request.get_header("If-none-match") == "v1"
+
+    def test_generic_cache_rejects_304_without_valid_cache(self, tmp_path: Path):
+        entry = self._entry(intermediate_patterns=[], asset_pattern=r"^good\.apk$")
+        release = sideload.ReleaseInfo(entry, "good.apk", "https://downloads.example/good.apk", None)
+        response = _generic_response(b"", release.download_url)
+        response.getcode.return_value = 304
+        with _mock_download_opener(response):
+            with pytest.raises(ValueError, match="304"):
+                sideload.GenericSource(entry).download_apk(
+                    release, sideload.DownloadCache(tmp_path / "cache")
+                )
+
+
 class TestApkValidator:
     def test_validate_success(self, apk_file: Path):
         validator = sideload.ApkValidator(aapt2_bin="fake-aapt2", aapt_bin="fake-aapt")
@@ -747,6 +920,13 @@ class TestMakeSource:
         s = sideload._make_source(codeberg_entry)
         assert isinstance(s, sideload.CodebergSource)
 
+    def test_generic(self):
+        entry = sideload.AppEntry(
+            name="X", source="generic", app_id="x",
+            config={"url": "https://example.test/", "asset_pattern": r"x\.apk"},
+        )
+        assert isinstance(sideload._make_source(entry), sideload.GenericSource)
+
     def test_unknown_source_raises(self):
         entry = sideload.AppEntry(name="X", source="unknown", app_id="x", config={})
         with pytest.raises(ValueError, match="unknown source type"):
@@ -783,7 +963,7 @@ class TestSideload:
         with mock.patch(
             "sideload._http_get", return_value=json.dumps(fdroid_index).encode()
         ):
-            with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+            with _mock_download_opener(mock_resp):
                 with mock.patch("sideload._run", side_effect=mock_run):
                     result = sideload.sideload(
                         app_entry, config, dry_run=True,
@@ -821,7 +1001,7 @@ class TestSideload:
         with mock.patch(
             "sideload._http_get", return_value=json.dumps(fdroid_index).encode()
         ):
-            with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+            with _mock_download_opener(mock_resp):
                 with mock.patch("sideload._run", side_effect=mock_run):
                     result = sideload.sideload(
                         app_entry, config, dry_run=True,
@@ -865,7 +1045,7 @@ class TestSideload:
         with mock.patch(
             "sideload._http_get", return_value=json.dumps(fdroid_index).encode()
         ):
-            with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+            with _mock_download_opener(mock_resp):
                 with mock.patch("sideload._run", side_effect=mock_run):
                     result = sideload.sideload(
                         app_entry, config,
@@ -910,7 +1090,7 @@ class TestSideload:
         with mock.patch(
             "sideload._http_get", return_value=json.dumps(fdroid_index).encode()
         ):
-            with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+            with _mock_download_opener(mock_resp):
                 with mock.patch("sideload._run", side_effect=mock_run):
                     result = sideload.sideload(
                         app_entry, config,
@@ -963,7 +1143,7 @@ class TestSideload:
         with mock.patch(
             "sideload._http_get", return_value=json.dumps(fdroid_index).encode()
         ):
-            with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+            with _mock_download_opener(mock_resp):
                 with mock.patch("sideload._run", side_effect=mock_run):
                     result = sideload.sideload(
                         app_entry, config,
@@ -1008,7 +1188,7 @@ class TestCLI:
         }
         mock_resp = _mock_urlopen_response(apk_data)
         with mock.patch("sideload._http_get", return_value=json.dumps(fdroid_index).encode()):
-            with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+            with _mock_download_opener(mock_resp):
                 with mock.patch("sideload._run", return_value=subprocess.CompletedProcess(
                     args=[], returncode=0, stdout="org.mozilla.fennec_fdroid\n", stderr=""
                 )):
@@ -1064,7 +1244,7 @@ class TestUtilities:
         dest = tmp_path / "sub" / "file.apk"
         data = b"downloaded-content"
         mock_resp = _mock_urlopen_response(data)
-        with mock.patch("urllib.request.urlopen", return_value=mock_resp):
+        with _mock_download_opener(mock_resp):
             result = sideload._atomic_download("https://example.com/x.apk", dest)
         assert result == dest
         assert dest.read_bytes() == data
@@ -1072,7 +1252,7 @@ class TestUtilities:
 
     def test_atomic_download_cleans_up_on_error(self, tmp_path: Path):
         dest = tmp_path / "file.apk"
-        with mock.patch("urllib.request.urlopen", side_effect=RuntimeError("fail")):
+        with _mock_download_opener(error=RuntimeError("fail")):
             with pytest.raises(RuntimeError):
                 sideload._atomic_download("https://example.com/x.apk", dest)
         assert not dest.exists()