diff --git a/Addon.py b/Addon.py index e1acda9..3daad4d 100644 --- a/Addon.py +++ b/Addon.py @@ -189,6 +189,11 @@ def __init__( self.curated = True self.score = 0 + # True if this Addon comes from a repository URL that the user entered themselves, rather + # than from the addon catalog. The user has chosen to trust that repository, so its Python + # dependencies are not held to the catalog's allow-list of reviewed packages. + self.from_custom_repository = False + # In cases where there are multiple versions/branches/installations available for an addon, # this dictionary is the mapping from the displayed name in the UI (as given in the # catalog) to the Addon object. diff --git a/AddonCatalogCacheCreator.py b/AddonCatalogCacheCreator.py index 18f9fc4..bdb6fe4 100644 --- a/AddonCatalogCacheCreator.py +++ b/AddonCatalogCacheCreator.py @@ -613,24 +613,10 @@ def find_file( @staticmethod def get_icon_from_metadata(metadata: addonmanager_metadata.Metadata) -> Optional[str]: - """Try to locate the icon file specified for this Addon. Recursively search through the - levels of the metadata and return the first specified icon file path. Returns None of there - is no icon specified for this Addon (which is not allowed by the standard, but we don't want - to crash the cache writer).""" - if metadata.icon: - if ( - metadata.subdirectory - and metadata.subdirectory != "." - and metadata.subdirectory != "./" - ): - return metadata.subdirectory + "/" + metadata.icon - return metadata.icon - for content_type in metadata.content: - for content_item in metadata.content[content_type]: - icon = CacheWriter.get_icon_from_metadata(content_item) - if icon: - return icon - return None + """Try to locate the icon file specified for this Addon. Returns None if there is no icon + specified for this Addon (which is not allowed by the standard, but we don't want to crash + the cache writer).""" + return addonmanager_metadata.get_icon_from_metadata(metadata) @staticmethod def determine_git_ref_type(name: str, _url: str, branch: str) -> GitRefType: diff --git a/AddonManagerTest/app/test_utilities.py b/AddonManagerTest/app/test_utilities.py index dff372a..c10086f 100644 --- a/AddonManagerTest/app/test_utilities.py +++ b/AddonManagerTest/app/test_utilities.py @@ -20,6 +20,7 @@ ################################################################################ from datetime import datetime +import json import unittest from unittest.mock import MagicMock, patch, mock_open import os @@ -27,12 +28,26 @@ from AddonManagerTest.app.mocks import MockAddon as Addon +from addonmanager_freecad_interface import Preferences from addonmanager_utilities import ( + GITEA, + GITHUB, + GITLAB, + IDENTIFIED_HOSTS_PREFERENCE, + construct_git_url, + forget_git_host, + forget_git_hosts, get_assigned_string_literal, get_macro_version_from_file, + get_readme_html_url, get_readme_url, + get_zip_url, + git_host_of, + identify_git_host, process_date_string_to_python_datetime, recognized_git_location, + reload_git_hosts, + remember_git_host, run_interruptable_subprocess, ) @@ -94,6 +109,36 @@ def test_get_readme_url(self): actual_result = get_readme_url(repo) self.assertEqual(actual_result, expected_result) + def test_get_readme_html_url(self): + """Each git host displays a file at its own URL: the Gitea hosts, including Codeberg, do + not use the same one as GitHub.""" + expected_urls = { + "https://github.com/FreeCAD/FreeCAD": "https://github.com/FreeCAD/FreeCAD/blob/main/README.md", + "https://codeberg.org/user/addon": "https://codeberg.org/user/addon/src/branch/main/README.md", + "https://gitlab.com/freecad/FreeCAD": "https://gitlab.com/freecad/FreeCAD/-/blob/main/README.md", + "https://unknown.host/user/addon": "https://unknown.host/user/addon/-/blob/main/README.md", + } + for url, expected_result in expected_urls.items(): + repo = Addon("Test Repo", url, "Addon.Status.NOT_INSTALLED", "main") + self.assertEqual(expected_result, get_readme_html_url(repo)) + + def test_get_zip_url(self): + expected_urls = { + "https://github.com/FreeCAD/FreeCAD": "https://github.com/FreeCAD/FreeCAD/archive/main.zip", + "https://codeberg.org/user/addon": "https://codeberg.org/user/addon/archive/main.zip", + "https://gitlab.com/freecad/FreeCAD": "https://gitlab.com/freecad/FreeCAD/-/archive/main/Test Repo-main.zip", + "https://unknown.host/user/addon": "https://unknown.host/user/addon/-/archive/main/Test Repo-main.zip", + } + for url, expected_result in expected_urls.items(): + repo = Addon("Test Repo", url, "Addon.Status.NOT_INSTALLED", "main") + self.assertEqual(expected_result, get_zip_url(repo)) + + def test_construct_git_url_for_a_local_path(self): + """A repo that is a local path, rather than a remote host, is used as-is.""" + repo = Addon("Test Repo", "/home/user/addon", "Addon.Status.NOT_INSTALLED", "main") + + self.assertEqual("/home/user/addon/package.xml", construct_git_url(repo, "package.xml")) + def test_get_assigned_string_literal(self): good_lines = [ ["my_var = 'Single-quoted literal'", "Single-quoted literal"], @@ -269,5 +314,206 @@ def test_process_date_string_to_python_datetime_invalid_separators(self): process_date_string_to_python_datetime(f"2024{separator}01{separator}31") +class TestGitHostDetection(unittest.TestCase): + """Tests for identifying the software that an unrecognized git host is running.""" + + _UNKNOWN_URL = "https://git.example.com/user/addon" + + def setUp(self): + forget_git_hosts() + self.addCleanup(forget_git_hosts) + + def _repo(self, url: str = _UNKNOWN_URL): + return Addon("Test Repo", url, "Addon.Status.NOT_INSTALLED", "main") + + def _answers_at(self, *layouts): + """A serves_file() that answers only at the raw file layouts of the given hosts, and + records every URL it was asked about.""" + answering_urls = { + f"{self._UNKNOWN_URL}{suffix}" + for suffix in ( + { + GITHUB: "/raw/main/package.xml", + GITEA: "/raw/branch/main/package.xml", + GITLAB: "/-/raw/main/package.xml", + }[host] + for host in layouts + ) + } + self.asked = [] + + def serves_file(url): + self.asked.append(url) + return url in answering_urls + + return serves_file + + # The layouts each host was measured to serve. All three serve GitHub's, so it identifies + # nothing on its own; the other two layouts are each served only by their own software. + + def test_a_host_serving_only_the_github_layout_is_github(self): + self.assertEqual(GITHUB, identify_git_host(self._repo(), self._answers_at(GITHUB))) + + def test_a_host_serving_the_gitea_layout_is_gitea(self): + """Gitea serves GitHub's layout as well as its own: the exclusive layout decides.""" + self.assertEqual(GITEA, identify_git_host(self._repo(), self._answers_at(GITEA, GITHUB))) + + def test_a_host_serving_the_gitlab_layout_is_gitlab(self): + """GitLab serves GitHub's layout as well as its own: the exclusive layout decides.""" + self.assertEqual(GITLAB, identify_git_host(self._repo(), self._answers_at(GITLAB, GITHUB))) + + def test_the_answer_does_not_depend_on_the_order_of_the_layouts(self): + """Every layout is asked before anything is decided, so no host can win by being asked + first. A Gitea host is identified as Gitea whichever way round the answers arrive.""" + for order in ((GITEA, GITHUB), (GITHUB, GITEA)): + with self.subTest(order=[host.name for host in order]): + self.assertEqual(GITEA, identify_git_host(self._repo(), self._answers_at(*order))) + + def test_every_layout_is_asked_about(self): + """No early exit: the decision is made from the complete set of answers.""" + identify_git_host(self._repo(), self._answers_at(GITEA, GITHUB)) + + self.assertEqual( + { + f"{self._UNKNOWN_URL}/raw/main/package.xml", + f"{self._UNKNOWN_URL}/raw/branch/main/package.xml", + f"{self._UNKNOWN_URL}/-/raw/main/package.xml", + }, + set(self.asked), + ) + + def test_a_host_serving_an_exclusive_layout_alone_is_still_identified(self): + """If a GitLab ever stops serving GitHub's legacy layout, it is still a GitLab.""" + self.assertEqual(GITLAB, identify_git_host(self._repo(), self._answers_at(GITLAB))) + + def test_a_host_that_contradicts_itself_is_not_identified(self): + """Nothing can be both a Gitea and a GitLab. A host that answers at both exclusive layouts + is answering everything it is asked, so its answers mean nothing and it is left alone.""" + self.assertIsNone(identify_git_host(self._repo(), self._answers_at(GITHUB, GITEA, GITLAB))) + + def test_a_host_that_serves_nothing_is_not_identified(self): + self.assertIsNone(identify_git_host(self._repo(), self._answers_at())) + + def test_a_local_path_is_not_identified(self): + repo = self._repo("/home/user/addon") + + self.assertIsNone(identify_git_host(repo, self._answers_at(GITHUB))) + + def test_identified_host_is_used_for_every_url(self): + """A host identified while fetching one file is used for the zip and readme URLs too.""" + repo = self._repo() + + remember_git_host(repo, GITEA) + + self.assertEqual(f"{self._UNKNOWN_URL}/archive/main.zip", get_zip_url(repo)) + self.assertEqual( + f"{self._UNKNOWN_URL}/src/branch/main/README.md", get_readme_html_url(repo) + ) + self.assertEqual(f"{self._UNKNOWN_URL}/raw/branch/main/README.md", get_readme_url(repo)) + + def test_a_known_host_cannot_be_overridden(self): + """The layouts of the hosts the Addon Manager ships are not up for redefinition.""" + repo = self._repo("https://github.com/FreeCAD/FreeCAD") + + remember_git_host(repo, GITLAB) + + self.assertEqual(GITHUB, git_host_of(repo)) + + def test_an_unidentified_host_has_no_layout(self): + self.assertIsNone(git_host_of(self._repo())) + + def test_forgetting_hosts_makes_them_be_identified_again(self): + repo = self._repo() + remember_git_host(repo, GITEA) + + forget_git_hosts() + + self.assertIsNone(git_host_of(repo)) + + +class TestGitHostPersistence(unittest.TestCase): + """A git host does not change which software it runs from one run of FreeCAD to the next, so + what was worked out about it is stored in the preferences rather than probed for every time.""" + + _UNKNOWN_URL = "https://git.example.com/user/addon" + + def setUp(self): + forget_git_hosts() + self.addCleanup(forget_git_hosts) + + def _repo(self, url: str = _UNKNOWN_URL): + return Addon("Test Repo", url, "Addon.Status.NOT_INSTALLED", "main") + + @staticmethod + def _restart(): + """Simulate the next run of FreeCAD: everything held in memory is gone, and only what was + written to the preferences is left.""" + reload_git_hosts() + + def test_an_identified_host_is_stored_in_the_preferences(self): + remember_git_host(self._repo(), GITEA) + + stored = json.loads(Preferences().get(IDENTIFIED_HOSTS_PREFERENCE)) + + self.assertEqual({"git.example.com": "Gitea"}, stored) + + def test_an_identified_host_survives_a_restart(self): + remember_git_host(self._repo(), GITEA) + + self._restart() + + self.assertEqual(GITEA, git_host_of(self._repo())) + + def test_a_stored_host_is_not_probed_again(self): + """A host read back from the preferences is used as-is: nothing is asked of it.""" + remember_git_host(self._repo(), GITEA) + + self._restart() + + def must_not_be_asked(url): + raise AssertionError(f"A host already stored in the preferences was probed at {url}") + + self.assertEqual(GITEA, git_host_of(self._repo())) + self.assertEqual(GITEA, identify_git_host(self._repo(), must_not_be_asked)) + + def test_forgetting_a_host_removes_it_from_the_preferences(self): + repo = self._repo() + remember_git_host(repo, GITEA) + + self.assertTrue(forget_git_host(repo)) + + self._restart() + self.assertIsNone(git_host_of(repo)) + + def test_forgetting_a_host_that_was_never_identified_does_nothing(self): + """Nothing was learned about this host, so there is no point in trying again.""" + self.assertFalse(forget_git_host(self._repo())) + + def test_a_host_stored_under_a_name_we_do_not_know_is_ignored(self): + """A preference naming host software this version does not support, perhaps written by a + newer version, is discarded rather than trusted.""" + Preferences().set(IDENTIFIED_HOSTS_PREFERENCE, '{"git.example.com": "Fossil"}') + + self._restart() + + self.assertIsNone(git_host_of(self._repo())) + + def test_a_corrupt_preference_is_ignored(self): + Preferences().set(IDENTIFIED_HOSTS_PREFERENCE, "this is not JSON") + + self._restart() + + with patch("addonmanager_utilities.fci.Console"): + self.assertIsNone(git_host_of(self._repo())) + + def test_a_shipped_host_cannot_be_overridden_by_the_preference(self): + """A stored preference cannot redefine the layout of a host we ship support for.""" + Preferences().set(IDENTIFIED_HOSTS_PREFERENCE, '{"github.com": "GitLab"}') + + self._restart() + + self.assertEqual(GITHUB, git_host_of(self._repo("https://github.com/FreeCAD/FreeCAD"))) + + if __name__ == "__main__": unittest.main() diff --git a/AddonManagerTest/app/test_workers_startup.py b/AddonManagerTest/app/test_workers_startup.py index 38c89ba..84d13c9 100644 --- a/AddonManagerTest/app/test_workers_startup.py +++ b/AddonManagerTest/app/test_workers_startup.py @@ -20,8 +20,12 @@ ################################################################################ import json +import os +import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch, MagicMock +import addonmanager_utilities as utils import addonmanager_workers_startup from Addon import Addon from PySideWrapper import QtCore @@ -157,37 +161,29 @@ def test_process_addon_catalog_with_user_override( self.assertEqual(8, mock_addon_repo_signal.emit.call_count) -# --------------------------------------------------------------------------- -# Minimal package.xml used by the tests below. The tag intentionally -# carries a *different* branch than the custom-repo entry so we can verify -# that the original values are preserved. -# --------------------------------------------------------------------------- -_PACKAGE_XML_WITH_ICON = b"""\ - +# The tag of these package.xml files intentionally carries a different repository and branch +# than the custom repository does, so that the tests can verify that the location the user +# configured is the one that is kept. +def _package_xml(version: str, icon: bool = True, python_dependency: str = "") -> bytes: + icon_tag = "Resources/icons/MyIcon.svg" if icon else "" + dependency_tag = ( + f'{python_dependency}' if python_dependency else "" + ) + package_xml = f""" My Custom Addon A description from package.xml. - 1.0.0 + {version} 2024-01-01 Dev LGPL-2.1 https://github.com/example/wrong-repo - Resources/icons/MyIcon.svg + {icon_tag} + {dependency_tag} """ + return package_xml.encode("utf-8") -_PACKAGE_XML_NO_ICON = b"""\ - - - My Custom Addon - A description from package.xml. - 1.0.0 - 2024-01-01 - Dev - LGPL-2.1 - https://github.com/example/wrong-repo - -""" _FAKE_ICON_BYTES = b"\x89PNG\r\n\x1a\nfake-icon-data" @@ -199,173 +195,445 @@ def _make_network_reply(data: bytes) -> MagicMock: return reply -class TestFetchRemoteCustomAddonMetadata(unittest.TestCase): - """Unit tests for CreateAddonListWorker._fetch_remote_custom_addon_metadata.""" +def _serve(files: dict): + """Return a blocking_get_with_retries side effect that serves the given files, keyed on the + path of the file within the repository, and returns None for anything else.""" + + def get(url: str, *_args, **_kwargs): + for path, data in files.items(): + if url.endswith(path): + return _make_network_reply(data) + return None + + return get + + +def _serve_urls(urls: dict): + """Return a blocking_get_with_retries side effect that serves the given files, keyed on the + complete URL, and returns None for every other URL.""" + + def get(url: str, *_args, **_kwargs): + if url in urls: + return _make_network_reply(urls[url]) + return None + + return get + + +class TestCustomAddonOnAnUnknownHost(unittest.TestCase): + """Tests for a custom repository on a git host the Addon Manager does not ship support for, + which is only usable if the Addon Manager works out what software the host is running.""" + + _NAME = "addon" + _URL = "https://git.example.com/user/addon" + _BRANCH = "main" + + # This self-hosted Gitea only serves the Gitea URL layout + _GITEA_PACKAGE_XML = f"{_URL}/raw/branch/{_BRANCH}/package.xml" + _GITEA_ICON = f"{_URL}/raw/branch/{_BRANCH}/Resources/icons/MyIcon.svg" + + def setUp(self): + self.mod_dir = tempfile.TemporaryDirectory() + data_paths_patch = patch("AddonCatalog.fci.DataPaths") + mock_data_paths = data_paths_patch.start() + mock_data_paths.return_value.mod_dir = self.mod_dir.name + self.addCleanup(data_paths_patch.stop) + self.addCleanup(self.mod_dir.cleanup) + + utils.forget_git_hosts() + self.addCleanup(utils.forget_git_hosts) + + def _create_addon(self) -> Addon: + worker = addonmanager_workers_startup.CreateAddonListWorker() + worker.mod_dir = self.mod_dir.name + return worker._create_custom_addon(self._NAME, self._URL, self._BRANCH) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_metadata_is_found_by_probing(self, mock_network_manager, _): + """The metadata of an addon on an unknown host is found by trying each known URL layout.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve_urls( + {self._GITEA_PACKAGE_XML: _package_xml("1.0.0"), self._GITEA_ICON: _FAKE_ICON_BYTES} + ) + + addon = self._create_addon() + + self.assertEqual("My Custom Addon", addon.display_name) + self.assertEqual(_FAKE_ICON_BYTES, addon.icon_data) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_the_host_is_identified(self, mock_network_manager, _): + """The host that answered is remembered, so the addon's other URLs are built for it. This + is what makes the addon installable: its zip lives at a Gitea URL, not a GitLab one.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve_urls( + {self._GITEA_PACKAGE_XML: _package_xml("1.0.0", icon=False)} + ) + + addon = self._create_addon() + + self.assertEqual(utils.GITEA, utils.git_host_of(addon)) + self.assertEqual(f"{self._URL}/archive/{self._BRANCH}.zip", addon.get_zip_url()) + self.assertEqual( + f"{self._URL}/src/branch/{self._BRANCH}/README.md", utils.get_readme_html_url(addon) + ) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_an_identified_host_is_not_probed_again(self, mock_network_manager, _): + """Once the host has been identified from the first file, every later file is fetched from + the layout that worked, rather than probing for it all over again.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve_urls( + {self._GITEA_PACKAGE_XML: _package_xml("1.0.0"), self._GITEA_ICON: _FAKE_ICON_BYTES} + ) + + self._create_addon() + + requested = [ + call[0][0] for call in mock_network_manager.blocking_get_with_retries.call_args_list + ] + self.assertEqual([self._GITEA_ICON], [url for url in requested if url.endswith(".svg")]) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_a_host_that_answers_nothing_is_not_identified(self, mock_network_manager, _): + """A host that provides none of the files is left unidentified, rather than being recorded + as whichever layout was tried last.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve_urls({}) + + addon = self._create_addon() + + self.assertIsNone(utils.git_host_of(addon)) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_a_host_behind_a_login_is_not_identified(self, mock_network_manager, _): + """A git host that requires a login answers every URL, including a nonsense one, with a 200 + and a sign-in page. Its answers say nothing about what software it runs, and the sign-in + page must not be mistaken for the addon's metadata.""" + sign_in_page = b"\nPlease sign in" + mock_network_manager.blocking_get_with_retries.side_effect = lambda url, *_a, **_k: ( + _make_network_reply(sign_in_page) + ) + + addon = self._create_addon() + + self.assertIsNone(utils.git_host_of(addon)) + self.assertIsNone(addon.metadata) + self.assertFalse(addon.python_requires) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_a_sign_in_page_is_never_read_as_a_requirements_file(self, mock_network_manager, _): + """The plain text metadata files cannot be checked by parsing them, so a sign-in page + served in place of one would otherwise be read as a list of Python dependencies, and the + user would be offered its JavaScript to install.""" + sign_in_page = b"\n\n\n" + mock_network_manager.blocking_get_with_retries.side_effect = lambda url, *_a, **_k: ( + _make_network_reply(sign_in_page) + ) + + addon = self._create_addon() + + self.assertFalse(addon.python_requires) + self.assertFalse(addon.requires) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_a_host_that_changed_software_is_identified_again(self, mock_network_manager, _): + """A host identified on an earlier run is stored in the preferences, so if it later moves + to different software it has to be worked out again rather than staying wrong forever.""" + remembered = SimpleNamespace(url=self._URL, branch=self._BRANCH, name=self._NAME) + utils.remember_git_host(remembered, utils.GITLAB) # What it was running last time + mock_network_manager.blocking_get_with_retries.side_effect = _serve_urls( + {self._GITEA_PACKAGE_XML: _package_xml("1.0.0", icon=False)} # What it runs now + ) + + addon = self._create_addon() + + self.assertEqual(utils.GITEA, utils.git_host_of(addon)) + self.assertEqual("My Custom Addon", addon.display_name) + + +class TestCustomAddons(unittest.TestCase): + """Tests for the custom repository handling in CreateAddonListWorker.""" + + _NAME = "my-custom-addon" + _URL = "https://github.com/myorg/my-custom-addon" + _BRANCH = "main" + + def setUp(self): + self.mod_dir = tempfile.TemporaryDirectory() + self.addon_dir = os.path.join(self.mod_dir.name, self._NAME) - _CUSTOM_URL = "https://github.com/myorg/my-custom-addon" - _CUSTOM_BRANCH = "main" + data_paths_patch = patch("AddonCatalog.fci.DataPaths") + mock_data_paths = data_paths_patch.start() + mock_data_paths.return_value.mod_dir = self.mod_dir.name + self.addCleanup(data_paths_patch.stop) + self.addCleanup(self.mod_dir.cleanup) - def _make_repo(self) -> Addon: - return Addon( - name="MyCustomAddon", - url=self._CUSTOM_URL, - branch=self._CUSTOM_BRANCH, + def _install(self, package_xml: bytes) -> None: + """Create an installed copy of the custom addon in the mod directory.""" + os.makedirs(self.addon_dir, exist_ok=True) + with open(os.path.join(self.addon_dir, "package.xml"), "wb") as f: + f.write(package_xml) + + def _worker(self) -> addonmanager_workers_startup.CreateAddonListWorker: + worker = addonmanager_workers_startup.CreateAddonListWorker() + worker.mod_dir = self.mod_dir.name + worker.current_thread = MagicMock() + worker.current_thread.isInterruptionRequested.return_value = False + return worker + + def _create_addon(self) -> Addon: + return self._worker()._create_custom_addon(self._NAME, self._URL, self._BRANCH) + + # ------------------------------------------------------------------ + # Preference parsing + # ------------------------------------------------------------------ + + @patch("addonmanager_workers_startup.fci.Preferences") + def test_parse_custom_repositories(self, mock_preferences): + """Each line is parsed into a URL and a branch, with "master" as the default branch.""" + mock_preferences.return_value.get.return_value = "\n".join( + [ + "https://github.com/myorg/no-branch-given", + "https://github.com/myorg/branch-given other-branch", + "https://github.com/myorg/trailing-slash/", + "https://github.com/myorg/dot-git.git", + "", + ] + ) + + result = addonmanager_workers_startup.CreateAddonListWorker._parse_custom_repositories() + + self.assertEqual( + [ + ("https://github.com/myorg/no-branch-given", "master"), + ("https://github.com/myorg/branch-given", "other-branch"), + ("https://github.com/myorg/trailing-slash", "master"), + ("https://github.com/myorg/dot-git", "master"), + ], + result, ) - def _make_worker(self) -> addonmanager_workers_startup.CreateAddonListWorker: - return addonmanager_workers_startup.CreateAddonListWorker() + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.CreateAddonListWorker.addon_repo") + @patch("addonmanager_workers_startup.fci.Preferences") + def test_duplicate_custom_addon_ignored(self, mock_preferences, mock_addon_repo_signal, _): + """A custom repository whose name is already known is skipped.""" + mock_preferences.return_value.get.return_value = self._URL + worker = self._worker() + worker.package_names = [self._NAME] + + worker._get_custom_addons() + + mock_addon_repo_signal.emit.assert_not_called() # ------------------------------------------------------------------ - # Happy path: package.xml fetched, icon fetched + # Metadata fetched from the remote repository # ------------------------------------------------------------------ @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_metadata_fields_populated(self, mock_nm, _mock_console): - """display_name and description are taken from the fetched package.xml.""" - mock_nm.blocking_get_with_retries.side_effect = [ - _make_network_reply(_PACKAGE_XML_WITH_ICON), # package.xml fetch - _make_network_reply(_FAKE_ICON_BYTES), # icon fetch - ] + def test_remote_metadata_populates_addon(self, mock_network_manager, _): + """The display name, description and icon all come from the remote repository.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + { + "package.xml": _package_xml("1.0.0"), + "Resources/icons/MyIcon.svg": _FAKE_ICON_BYTES, + } + ) - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - self.assertEqual("My Custom Addon", repo.display_name) - self.assertEqual("A description from package.xml.", repo.description) + self.assertEqual("My Custom Addon", addon.display_name) + self.assertEqual("A description from package.xml.", addon.description) + self.assertEqual(_FAKE_ICON_BYTES, addon.icon_data) + self.assertEqual(Addon.Status.NOT_INSTALLED, addon.status()) @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_icon_data_populated(self, mock_nm, _mock_console): - """icon_data is set from the fetched icon bytes.""" - mock_nm.blocking_get_with_retries.side_effect = [ - _make_network_reply(_PACKAGE_XML_WITH_ICON), - _make_network_reply(_FAKE_ICON_BYTES), - ] + def test_configured_url_and_branch_are_kept(self, mock_network_manager, _): + """The URL and branch in package.xml do not override the ones the user configured.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"package.xml": _package_xml("1.0.0", icon=False)} + ) - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - self.assertEqual(_FAKE_ICON_BYTES, repo.icon_data) + self.assertEqual(self._URL, addon.url) + self.assertEqual(self._BRANCH, addon.branch) @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_custom_url_preserved(self, mock_nm, _mock_console): - """repo.url must remain the custom-repo URL, not the one from package.xml.""" - mock_nm.blocking_get_with_retries.side_effect = [ - _make_network_reply(_PACKAGE_XML_WITH_ICON), - _make_network_reply(_FAKE_ICON_BYTES), - ] + def test_files_are_fetched_from_the_configured_branch(self, mock_network_manager, _): + """Every file is fetched from the branch the user configured.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + { + "package.xml": _package_xml("1.0.0"), + "Resources/icons/MyIcon.svg": _FAKE_ICON_BYTES, + } + ) - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + self._create_addon() - self.assertEqual(self._CUSTOM_URL, repo.url) + for call in mock_network_manager.blocking_get_with_retries.call_args_list: + url = call[0][0] + self.assertIn(self._BRANCH, url) + self.assertNotIn("wrong-branch", url) + self.assertNotIn("wrong-repo", url) @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_custom_branch_preserved(self, mock_nm, _mock_console): - """repo.branch must remain the custom-repo branch, not the one from package.xml.""" - mock_nm.blocking_get_with_retries.side_effect = [ - _make_network_reply(_PACKAGE_XML_WITH_ICON), - _make_network_reply(_FAKE_ICON_BYTES), - ] + def test_python_dependencies_from_package_xml(self, mock_network_manager, _): + """A Python dependency declared in package.xml is available for the installer to act on.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"package.xml": _package_xml("1.0.0", icon=False, python_dependency="some_package")} + ) - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - self.assertEqual(self._CUSTOM_BRANCH, repo.branch) + self.assertIn("some_package", addon.python_requires) @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_icon_url_uses_custom_branch(self, mock_nm, _mock_console): - """The icon is fetched using the custom branch, not the one from package.xml.""" - mock_nm.blocking_get_with_retries.side_effect = [ - _make_network_reply(_PACKAGE_XML_WITH_ICON), - _make_network_reply(_FAKE_ICON_BYTES), - ] + def test_python_dependencies_from_requirements_txt(self, mock_network_manager, _): + """A Python dependency declared in requirements.txt is picked up as well.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + { + "package.xml": _package_xml("1.0.0", icon=False), + "requirements.txt": b"some_package>=1.2.3\n", + } + ) + + addon = self._create_addon() + + self.assertIn("some_package", addon.python_requires) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_workbench_dependencies_from_metadata_txt(self, mock_network_manager, _): + """A legacy custom repository with only a metadata.txt file still yields its dependencies.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"metadata.txt": b"workbenches=Part\npylibs=some_package\n"} + ) - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - # The icon URL should contain the custom branch, not "wrong-branch" - icon_call_args = mock_nm.blocking_get_with_retries.call_args_list[1] - icon_url: str = icon_call_args[0][0] - self.assertIn(self._CUSTOM_BRANCH, icon_url) - self.assertNotIn("wrong-branch", icon_url) + self.assertIn("some_package", addon.python_requires) + self.assertIn("Part", addon.requires) # ------------------------------------------------------------------ - # No icon in package.xml + # Update detection for an installed custom addon # ------------------------------------------------------------------ @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_no_icon_field_skips_icon_fetch(self, mock_nm, _mock_console): - """When package.xml has no , only one network call is made.""" - mock_nm.blocking_get_with_retries.return_value = _make_network_reply(_PACKAGE_XML_NO_ICON) + def test_newer_remote_version_flags_an_update(self, mock_network_manager, _): + """A remote version newer than the installed one is reported as an available update.""" + self._install(_package_xml("1.0.0", icon=False)) + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"package.xml": _package_xml("2.0.0", icon=False)} + ) + + addon = self._create_addon() - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + self.assertEqual(Addon.Status.UPDATE_AVAILABLE, addon.status()) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_installed_addon_compares_remote_to_installed_metadata(self, mock_network_manager, _): + """The addon's metadata is the remote copy, and its installed metadata the local one, so + that the two can actually be compared against each other.""" + self._install(_package_xml("1.0.0", icon=False)) + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"package.xml": _package_xml("2.0.0", icon=False)} + ) - self.assertEqual(1, mock_nm.blocking_get_with_retries.call_count) + addon = self._create_addon() + + self.assertEqual("2.0.0", str(addon.metadata.version).strip()) + self.assertEqual("1.0.0", str(addon.installed_metadata.version).strip()) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_same_version_leaves_addon_unchecked(self, mock_network_manager, _): + """When the versions match, the addon is left unchecked so that the update worker can run + a git-based check on it: the version string is not proof that there is no update.""" + self._install(_package_xml("1.0.0", icon=False)) + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"package.xml": _package_xml("1.0.0", icon=False)} + ) + + addon = self._create_addon() + + self.assertEqual(Addon.Status.UNCHECKED, addon.status()) # ------------------------------------------------------------------ - # Failure cases – should be silent / non-fatal + # Failure cases: none of them may be fatal # ------------------------------------------------------------------ @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_no_package_xml_available(self, mock_nm, _mock_console): - """When the network returns None, the repo is left unchanged.""" - mock_nm.blocking_get_with_retries.return_value = None + def test_installed_addon_falls_back_to_local_metadata(self, mock_network_manager, _): + """When the remote repository cannot be reached, an installed addon still displays the + metadata of the installed copy.""" + self._install(_package_xml("1.0.0", icon=False)) + mock_network_manager.blocking_get_with_retries.return_value = None - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - self.assertIsNone(repo.metadata) - self.assertEqual(self._CUSTOM_URL, repo.url) - self.assertEqual(self._CUSTOM_BRANCH, repo.branch) + self.assertEqual("My Custom Addon", addon.display_name) + self.assertEqual("1.0.0", str(addon.metadata.version).strip()) @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_network_exception_leaves_repo_unchanged(self, mock_nm, _mock_console): - """A network error during package.xml fetch leaves the repo unchanged.""" - mock_nm.blocking_get_with_retries.side_effect = RuntimeError("connection refused") + def test_no_metadata_available(self, mock_network_manager, _): + """A custom repository that provides no metadata at all still yields a usable Addon.""" + mock_network_manager.blocking_get_with_retries.return_value = None - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - self.assertIsNone(repo.metadata) - self.assertEqual(self._CUSTOM_URL, repo.url) - self.assertEqual(self._CUSTOM_BRANCH, repo.branch) + self.assertIsNone(addon.metadata) + self.assertEqual(self._NAME, addon.name) + self.assertEqual(self._URL, addon.url) + self.assertEqual(self._BRANCH, addon.branch) @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_corrupt_package_xml_leaves_repo_unchanged(self, mock_nm, _mock_console): - """Invalid XML during parse leaves the repo unchanged.""" - mock_nm.blocking_get_with_retries.return_value = _make_network_reply(b"this is not xml") + def test_network_error_is_not_fatal(self, mock_network_manager, _): + """An exception while fetching leaves the addon without metadata, but does not raise.""" + mock_network_manager.blocking_get_with_retries.side_effect = RuntimeError("no connection") - repo = self._make_repo() - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - self.assertIsNone(repo.metadata) - self.assertEqual(self._CUSTOM_URL, repo.url) - self.assertEqual(self._CUSTOM_BRANCH, repo.branch) + self.assertIsNone(addon.metadata) + self.assertEqual(self._URL, addon.url) @patch("addonmanager_workers_startup.fci.Console") @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") - def test_icon_fetch_failure_does_not_raise(self, mock_nm, _mock_console): - """A network error during icon fetch is silently swallowed.""" - mock_nm.blocking_get_with_retries.side_effect = [ - _make_network_reply(_PACKAGE_XML_WITH_ICON), - RuntimeError("icon fetch failed"), - ] + def test_corrupt_package_xml_is_not_fatal(self, mock_network_manager, _): + """An unparsable package.xml leaves the addon without metadata, but does not raise.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"package.xml": b"this is not xml"} + ) + + addon = self._create_addon() + + self.assertIsNone(addon.metadata) + self.assertEqual(self._URL, addon.url) + self.assertEqual(self._BRANCH, addon.branch) + + @patch("addonmanager_workers_startup.fci.Console") + @patch("addonmanager_workers_startup.NetworkManager.AM_NETWORK_MANAGER") + def test_missing_icon_is_not_fatal(self, mock_network_manager, _): + """An icon that cannot be fetched leaves the rest of the metadata intact.""" + mock_network_manager.blocking_get_with_retries.side_effect = _serve( + {"package.xml": _package_xml("1.0.0")} + ) - repo = self._make_repo() - # Must not raise: - self._make_worker()._fetch_remote_custom_addon_metadata(repo) + addon = self._create_addon() - # Metadata should still be populated, just no icon - self.assertIsNotNone(repo.metadata) - self.assertEqual(self._CUSTOM_URL, repo.url) - self.assertEqual(self._CUSTOM_BRANCH, repo.branch) + self.assertEqual("My Custom Addon", addon.display_name) + self.assertFalse(addon.icon_data) diff --git a/AddonManagerTest/gui/test_installer_gui.py b/AddonManagerTest/gui/test_installer_gui.py index 3105067..df2b877 100644 --- a/AddonManagerTest/gui/test_installer_gui.py +++ b/AddonManagerTest/gui/test_installer_gui.py @@ -326,6 +326,110 @@ def test_run_with_disallowed_optional_python_continues(self): # Assert self.assertTrue(proceed_monitor.good()) + # The allow-list of reviewed Python packages belongs to the addon catalog, so it does not get + # to veto the dependencies of a repository the user entered themselves. They are asked to + # confirm the packages instead of being told to install them by hand. + + @staticmethod + def _addon(from_custom_repository: bool) -> Addon: + addon = Addon( + "TestAddon", "https://git.example.com/user/addon", Addon.Status.NOT_INSTALLED, "main" + ) + addon.from_custom_repository = from_custom_repository + return addon + + def _handle_disallowed_python(self, addons, deps, button): + """Run the disallowed-package check with the given answer to whatever dialog it shows. + Returns whether the installation was stopped, and the dialog it showed.""" + gui = AddonDependencyInstallerGUI(addons, deps) + gui.installer = self.MockAddonInstaller(addons) + with patch( + "addonmanager_installer_gui.MessageDialog.show_modal", return_value=button + ) as mock_dialog: + stop_installation = gui._handle_disallowed_python() + dialog_name = mock_dialog.call_args[0][1] if mock_dialog.call_args else None + return stop_installation, dialog_name + + def test_custom_repo_unreviewed_package_is_installed_when_accepted(self): + """This is what makes a custom addon's dependencies actually get installed: the package + stays in the list, rather than being dropped with an instruction to install it by hand.""" + deps = self.create_mock_deps(python_requires=["not_in_the_allowlist"]) + + stop_installation, dialog_name = self._handle_disallowed_python( + [self._addon(from_custom_repository=True)], deps, QtWidgets.QMessageBox.Ok + ) + + self.assertFalse(stop_installation) + self.assertEqual("AddonManager_UnreviewedPythonDialog", dialog_name) + self.assertIn("not_in_the_allowlist", deps.python_requires) + + def test_custom_repo_unreviewed_package_is_refused_when_cancelled(self): + deps = self.create_mock_deps(python_requires=["not_in_the_allowlist"]) + + stop_installation, dialog_name = self._handle_disallowed_python( + [self._addon(from_custom_repository=True)], deps, QtWidgets.QMessageBox.Cancel + ) + + self.assertTrue(stop_installation) + self.assertEqual("AddonManager_UnreviewedPythonDialog", dialog_name) + + def test_catalog_addon_is_still_held_to_the_allow_list(self): + """The user did not choose where a catalog addon comes from, so nothing changes for it.""" + deps = self.create_mock_deps(python_requires=["not_in_the_allowlist"]) + + stop_installation, dialog_name = self._handle_disallowed_python( + [self._addon(from_custom_repository=False)], deps, QtWidgets.QMessageBox.Ok + ) + + self.assertFalse(stop_installation) + self.assertEqual("AddonManager_RequirementFailedDialog", dialog_name) + self.assertNotIn("not_in_the_allowlist", deps.python_requires) + + def test_one_catalog_addon_holds_the_whole_installation_to_the_allow_list(self): + """Installing a catalog addon alongside a custom one does not launder the catalog addon's + dependencies through the custom repository's exemption.""" + deps = self.create_mock_deps(python_requires=["not_in_the_allowlist"]) + addons = [self._addon(from_custom_repository=True), self._addon(False)] + + stop_installation, dialog_name = self._handle_disallowed_python( + addons, deps, QtWidgets.QMessageBox.Ok + ) + + self.assertFalse(stop_installation) + self.assertEqual("AddonManager_RequirementFailedDialog", dialog_name) + self.assertNotIn("not_in_the_allowlist", deps.python_requires) + + def test_an_empty_addon_list_is_held_to_the_allow_list(self): + """No addon means nothing to trust: the exemption must not apply by default.""" + deps = self.create_mock_deps(python_requires=["not_in_the_allowlist"]) + + stop_installation, dialog_name = self._handle_disallowed_python( + [], deps, QtWidgets.QMessageBox.Ok + ) + + self.assertEqual("AddonManager_RequirementFailedDialog", dialog_name) + self.assertNotIn("not_in_the_allowlist", deps.python_requires) + + def test_custom_repo_unreviewed_optional_package_is_offered(self): + """An optional package is offered in the dependency dialog for the user to accept or + refuse, so a custom repository's optional packages are not dropped either.""" + deps = self.create_mock_deps(python_optional=["not_in_the_allowlist"]) + gui = AddonDependencyInstallerGUI([self._addon(from_custom_repository=True)], deps) + gui.installer = self.MockAddonInstaller([]) + + gui._clean_up_optional() + + self.assertIn("not_in_the_allowlist", deps.python_optional) + + def test_catalog_addon_unreviewed_optional_package_is_dropped(self): + deps = self.create_mock_deps(python_optional=["not_in_the_allowlist"]) + gui = AddonDependencyInstallerGUI([self._addon(from_custom_repository=False)], deps) + gui.installer = self.MockAddonInstaller([]) + + gui._clean_up_optional() + + self.assertEqual([], deps.python_optional) + @patch("addonmanager_installer_gui.utils.blocking_get", MagicMock(return_value=None)) def test_run_incompatible_python_version(self): # Arrange diff --git a/addonmanager_installer_gui.py b/addonmanager_installer_gui.py index f2b3d31..e8ae25a 100644 --- a/addonmanager_installer_gui.py +++ b/addonmanager_installer_gui.py @@ -513,6 +513,9 @@ def _handle_disallowed_python(self) -> bool: if dep.lower() not in self.installer.allowed_packages: bad_packages.append(dep) + if bad_packages and self._all_from_custom_repositories(): + return self._confirm_unreviewed_python(bad_packages) + for dep in bad_packages: self.deps.python_requires.remove(dep) @@ -554,6 +557,59 @@ def _handle_disallowed_python(self) -> bool: return True return False + def _all_from_custom_repositories(self) -> bool: + """Whether every addon being installed comes from a repository the user entered themselves. + The allow-list of reviewed Python packages is the catalog's, so it does not get to veto the + dependencies of a repository the user chose to trust. If even one of the addons comes from + the catalog, the allow-list applies as usual.""" + + return bool(self.addons) and all( + getattr(addon, "from_custom_repository", False) for addon in self.addons + ) + + def _confirm_unreviewed_python(self, packages: List[str]) -> bool: + """Ask the user to confirm the installation of Python packages that FreeCAD has not + reviewed, naming them and the repository that asked for them. Returns True to stop the + installation, or False to proceed with the packages left in place, to be installed.""" + + if len(self.addons) == 1: + core_message = translate( + "AddonsInstaller", + "This addon requires Python packages that are not on FreeCAD's list of reviewed " + "packages:", + ) + else: + core_message = translate( + "AddonsInstaller", + "These addons require Python packages that are not on FreeCAD's list of reviewed " + "packages:", + ) + message = f"

{core_message}

    " + for package in packages: + message += f"
  • {package}
    • " + message += "

" + message += translate( + "AddonsInstaller", + "They will be downloaded from the Python Package Index (PyPI) and installed into your " + "user directory. FreeCAD has not reviewed them. Only continue if you trust the " + "repository that asked for them:", + ) + message += "

    " + for addon in self.addons: + message += f"
  • {addon.url}
    • " + message += "
" + + r = MessageDialog.show_modal( + MessageDialog.DialogType.WARNING, + "AddonManager_UnreviewedPythonDialog", + translate("AddonsInstaller", "Unreviewed Python Packages"), + message, + QtWidgets.QMessageBox.Ok | QtWidgets.QMessageBox.Cancel, + ) + if r == QtWidgets.QMessageBox.Ok: + return False # Leave the packages in python_requires, so that they get installed + return True + def _report_missing_workbenches(self) -> bool: """If there are missing workbenches, display a dialog informing the user. Returns True to stop the installation, or False to proceed.""" @@ -657,6 +713,11 @@ def _check_python_version(self) -> bool: return False def _clean_up_optional(self): + if self._all_from_custom_repositories(): + # The allow-list does not apply to a repository the user chose to trust. Each optional + # package is offered in the dependency dialog for the user to accept or refuse, so + # nothing is installed here without them saying so. + return good_packages = [] for dep in self.deps.python_optional: if dep in self.installer.allowed_packages: diff --git a/addonmanager_metadata.py b/addonmanager_metadata.py index 86e3f7e..0b92ff6 100644 --- a/addonmanager_metadata.py +++ b/addonmanager_metadata.py @@ -242,6 +242,23 @@ def get_repo_url_from_metadata(metadata: Metadata) -> str: return "" +def get_icon_from_metadata(metadata: Metadata) -> Optional[str]: + """Locate the icon file specified for this Addon, as a path relative to the root of the + Addon's repository. Recursively searches through the levels of the metadata and returns the + first specified icon file path, or None if no icon is specified anywhere.""" + + if metadata.icon: + if metadata.subdirectory and metadata.subdirectory not in (".", "./"): + return metadata.subdirectory.rstrip("/") + "/" + metadata.icon + return metadata.icon + for content_type in metadata.content: + for content_item in metadata.content[content_type]: + icon = get_icon_from_metadata(content_item) + if icon: + return icon + return None + + class MetadataReader: """Read metadata XML data and construct a Metadata object""" diff --git a/addonmanager_preferences_defaults.json b/addonmanager_preferences_defaults.json index 0566599..80657f8 100644 --- a/addonmanager_preferences_defaults.json +++ b/addonmanager_preferences_defaults.json @@ -11,6 +11,7 @@ "HideNonFSFFreeLibre": false, "HideNonOSIApproved": false, "HideUnlicensed": false, + "IdentifiedGitHosts": "{}", "KnownPythonVersions": "[]", "MacroUpdateStatsURL": "https://addons.freecad.org/macro_update_stats.json", "NoProxyCheck": false, diff --git a/addonmanager_utilities.py b/addonmanager_utilities.py index 4750f89..836b337 100644 --- a/addonmanager_utilities.py +++ b/addonmanager_utilities.py @@ -24,14 +24,17 @@ # pylint: disable=deprecated-module, ungrouped-imports +from dataclasses import dataclass from datetime import datetime -from typing import Optional, Any, List +from typing import Dict, Optional, Any, List, Tuple +import json import os import platform import shutil import stat import subprocess import sys +import threading import time import re import ctypes @@ -198,58 +201,237 @@ def restart_freecad(): QtCore.QProcess.startDetached(QtWidgets.QApplication.applicationFilePath(), args) +@dataclass(frozen=True) +class GitHost: + """The URL layouts used by a family of git hosting software. Each layout is a format string + that takes the base url of the repository, the branch, the name of the addon, and the path of + a file within the repository. + + raw_file_is_exclusive records whether a host that serves a file at this raw_file layout must be + running this software. Gitea and GitLab each have a raw file layout that only they serve, but + all three of them serve GitHub's, so being answered at GitHub's layout means nothing on its own. + See identify_git_host().""" + + name: str + raw_file: str + archive: str + blob: str + raw_file_is_exclusive: bool + + +GITHUB = GitHost( + name="GitHub", + raw_file="{url}/raw/{branch}/{filename}", + archive="{url}/archive/{branch}.zip", + blob="{url}/blob/{branch}/{filename}", + raw_file_is_exclusive=False, # Gitea and GitLab serve this layout too +) + +GITEA = GitHost( + name="Gitea", # Also Forgejo (and therefore Codeberg) + raw_file="{url}/raw/branch/{branch}/{filename}", + archive="{url}/archive/{branch}.zip", + blob="{url}/src/branch/{branch}/{filename}", + raw_file_is_exclusive=True, +) + +GITLAB = GitHost( + name="GitLab", + raw_file="{url}/-/raw/{branch}/{filename}", + archive="{url}/-/archive/{branch}/{name}-{branch}.zip", + blob="{url}/-/blob/{branch}/{filename}", + raw_file_is_exclusive=True, +) + +KNOWN_GIT_HOSTS: Dict[str, GitHost] = { + "github.com": GITHUB, + "codeberg.org": GITEA, + "gitlab.com": GITLAB, + "framagit.org": GITLAB, + "salsa.debian.org": GITLAB, +} + +# Every host that an unidentified git host might turn out to be running. All of them are asked, and +# the answer is worked out from the complete set of replies, so the order here does not matter. +CANDIDATE_GIT_HOSTS: Tuple[GitHost, ...] = (GITHUB, GITEA, GITLAB) + +# The layout to use for a host that we have neither heard of nor identified +DEFAULT_GIT_HOST = GITLAB + +# The file a host is asked for when identifying it. It has to be one whose contents can be checked, +# so that a login page or an error page served with a 200 status is not mistaken for it. +IDENTIFYING_FILE = "package.xml" + +GIT_HOSTS_BY_NAME: Dict[str, GitHost] = {host.name: host for host in CANDIDATE_GIT_HOSTS} + +# The preference that identified hosts are stored in, so that a host only ever has to be identified +# once: a host does not change which software it runs from one run of FreeCAD to the next. +IDENTIFIED_HOSTS_PREFERENCE = "IdentifiedGitHosts" + +_identified_git_hosts: Optional[Dict[str, GitHost]] = None # Loaded from preferences on first use +_identified_git_hosts_lock = threading.Lock() + + +def _load_identified_git_hosts() -> Dict[str, GitHost]: + """Read the identified hosts from the preferences. Must be called with the lock held.""" + + global _identified_git_hosts + if _identified_git_hosts is not None: + return _identified_git_hosts + + _identified_git_hosts = {} + try: + stored = json.loads(fci.Preferences().get(IDENTIFIED_HOSTS_PREFERENCE)) + except json.JSONDecodeError: + fci.Console.PrintWarning( + f"Could not read the {IDENTIFIED_HOSTS_PREFERENCE} preference: " + "the git hosts it names will be identified again\n" + ) + stored = {} + if isinstance(stored, dict): + for netloc, host_name in stored.items(): + if host_name in GIT_HOSTS_BY_NAME and netloc not in KNOWN_GIT_HOSTS: + _identified_git_hosts[netloc] = GIT_HOSTS_BY_NAME[host_name] + return _identified_git_hosts + + +def _store_identified_git_hosts() -> None: + """Write the identified hosts back to the preferences. Must be called with the lock held.""" + + fci.Preferences().set( + IDENTIFIED_HOSTS_PREFERENCE, + json.dumps({netloc: host.name for netloc, host in _identified_git_hosts.items()}), + ) + + +def git_host_of(repo) -> Optional[GitHost]: + """The software running the git host of this repo, if it is known: either because it is one of + the hosts the Addon Manager ships, or because it was identified during this or an earlier run. + Returns None for a host that has not been identified.""" + + netloc = urlparse(repo.url).netloc + if netloc in KNOWN_GIT_HOSTS: + return KNOWN_GIT_HOSTS[netloc] + with _identified_git_hosts_lock: + return _load_identified_git_hosts().get(netloc) + + +def remember_git_host(repo, host: GitHost) -> None: + """Record the software that a git host turned out to be running. This is stored in the user's + preferences, so that the host does not have to be identified again on the next run.""" + + netloc = urlparse(repo.url).netloc + if not netloc or netloc in KNOWN_GIT_HOSTS: + return + with _identified_git_hosts_lock: + identified = _load_identified_git_hosts() + if identified.get(netloc) == host: + return + fci.Console.PrintLog(f"Identified the git host at {netloc} as {host.name}\n") + identified[netloc] = host + _store_identified_git_hosts() + + +def forget_git_host(repo) -> bool: + """Discard what was previously worked out about this repo's git host, so that it is identified + again. Returns True if there was anything to discard, which means it is worth another try.""" + + netloc = urlparse(repo.url).netloc + with _identified_git_hosts_lock: + identified = _load_identified_git_hosts() + if netloc not in identified: + return False + fci.Console.PrintLog( + f"The git host at {netloc} no longer answers as {identified[netloc].name}: " + "identifying it again\n" + ) + del identified[netloc] + _store_identified_git_hosts() + return True + + +def forget_git_hosts() -> None: + """Discard everything the Addon Manager has worked out about git hosts.""" + + with _identified_git_hosts_lock: + _load_identified_git_hosts().clear() + _store_identified_git_hosts() + + +def reload_git_hosts() -> None: + """Discard the identified hosts held in memory and read them from the preferences again, as + happens the first time they are needed in a run of FreeCAD.""" + + global _identified_git_hosts + with _identified_git_hosts_lock: + _identified_git_hosts = None + + +def _base_url(repo) -> str: + return repo.url[:-4] if repo.url.endswith(".git") else repo.url + + +def _format_url(layout: str, repo, filename: str = "") -> str: + return layout.format( + url=_base_url(repo), + branch=repo.branch, + filename=filename, + name=getattr(repo, "name", ""), + ) + + +def identify_git_host(repo, serves_file) -> Optional[GitHost]: + """Work out which software an unrecognized git host is running, by asking it for the same file + in every layout the Addon Manager knows and deciding from the complete set of answers. + + serves_file(url) must return True only if the host really served the file that was asked for. + Checking the status code is not enough to establish that: a git host that requires a login + answers every URL, including a nonsense one, with a 200 and a sign-in page. + + Only Gitea and GitLab have a raw file layout that is exclusively theirs, so an answer at either + of those identifies the host outright. All three serve GitHub's layout, so an answer there only + means GitHub if neither of the exclusive layouts answered. A host that answers at both of the + exclusive layouts is contradicting itself and is left unidentified rather than guessed at.""" + + known = git_host_of(repo) + if known is not None: + return known # Shipped, or identified on an earlier run: there is nothing to ask + if not urlparse(repo.url).netloc: + return None # A local path, not a git host + + answered = { + candidate + for candidate in CANDIDATE_GIT_HOSTS + if serves_file(_format_url(candidate.raw_file, repo, IDENTIFYING_FILE)) + } + exclusive = {candidate for candidate in answered if candidate.raw_file_is_exclusive} + + if len(exclusive) == 1: + return exclusive.pop() + if not exclusive and GITHUB in answered: + return GITHUB + return None + + def get_zip_url(repo): """Returns the location of a zip file from a repo, if available""" - parsed_url = urlparse(repo.url) - if parsed_url.netloc == "github.com": - return f"{repo.url}/archive/{repo.branch}.zip" - if parsed_url.netloc in ["gitlab.com", "framagit.org", "salsa.debian.org"]: - return f"{repo.url}/-/archive/{repo.branch}/{repo.name}-{repo.branch}.zip" - if parsed_url.netloc in ["codeberg.org"]: - return f"{repo.url}/archive/{repo.branch}.zip" - fci.Console.PrintLog( - "Debug: addonmanager_utilities.get_zip_url: Unknown git host fetching zip URL:" - + parsed_url.netloc - + "\n" - ) - return f"{repo.url}/-/archive/{repo.branch}/{repo.name}-{repo.branch}.zip" + return _format_url(_host_or_default(repo).archive, repo) def recognized_git_location(repo) -> bool: - """Returns whether this repo is based at a known git repo location: works with GitHub, gitlab, - framagit, and salsa.debian.org""" + """Returns whether this repo is based at a git host that the Addon Manager ships support for""" - parsed_url = urlparse(repo.url) - return parsed_url.netloc in [ - "github.com", - "gitlab.com", - "framagit.org", - "salsa.debian.org", - "codeberg.org", - ] + return urlparse(repo.url).netloc in KNOWN_GIT_HOSTS def construct_git_url(repo, filename): """Returns a direct download link to a file in an online Git repo""" parsed_url = urlparse(repo.url) - repo_url = repo.url[:-4] if repo.url.endswith(".git") else repo.url - if parsed_url.netloc == "github.com": - return f"{repo_url}/raw/{repo.branch}/{filename}" - if parsed_url.netloc in ["gitlab.com", "framagit.org", "salsa.debian.org"]: - return f"{repo_url}/-/raw/{repo.branch}/{filename}" - if parsed_url.netloc in ["codeberg.org"]: - return f"{repo_url}/raw/branch/{repo.branch}/{filename}" - if parsed_url.netloc == "": - return f"{parsed_url.path}/{filename}" - fci.Console.PrintLog( - "Debug: addonmanager_utilities.construct_git_url: Unknown git host:" - + parsed_url.netloc - + f" for file {filename}\n" - ) - # Assume it's some kind of local GitLab instance... - return f"{repo_url}/-/raw/{repo.branch}/{filename}" + if not parsed_url.netloc: + return f"{parsed_url.path}/{filename}" # A local path, not a remote host + return _format_url(_host_or_default(repo).raw_file, repo, filename) def get_readme_url(repo): @@ -258,35 +440,24 @@ def get_readme_url(repo): return construct_git_url(repo, "README.md") -def get_desc_regex(repo): - """Returns a regex string that extracts a WB description to be displayed in the description - panel of the Addon manager, if the README could not be found""" +def get_readme_html_url(repo): + """Returns the location of a html file containing readme""" - parsed_url = urlparse(repo.url) - if parsed_url.netloc == "github.com": - return r'' - if parsed_url.netloc in ["codeberg.org"]: - return r'' + return _format_url(_host_or_default(repo).blob, repo, "README.md") -def get_readme_html_url(repo): - """Returns the location of a html file containing readme""" +def _host_or_default(repo) -> GitHost: + """The layout to use for this repo's git host, falling back to the default for a host that has + not been identified.""" - parsed_url = urlparse(repo.url) - if parsed_url.netloc == "github.com": - return f"{repo.url}/blob/{repo.branch}/README.md" - if parsed_url.netloc in ["gitlab.com", "salsa.debian.org", "framagit.org"]: - return f"{repo.url}/-/blob/{repo.branch}/README.md" - if parsed_url.netloc in ["gitlab.com", "salsa.debian.org", "framagit.org"]: - return f"{repo.url}/raw/branch/{repo.branch}/README.md" - fci.Console.PrintLog("Unrecognized git repo location '' -- guessing it is a GitLab instance...") - return f"{repo.url}/-/blob/{repo.branch}/README.md" + host = git_host_of(repo) + if host is not None: + return host + fci.Console.PrintLog( + f"Debug: the git host at {urlparse(repo.url).netloc} has not been identified: " + f"assuming it is running {DEFAULT_GIT_HOST.name}\n" + ) + return DEFAULT_GIT_HOST def is_darkmode() -> bool: diff --git a/addonmanager_workers_startup.py b/addonmanager_workers_startup.py index 0b84635..4c2d1e2 100644 --- a/addonmanager_workers_startup.py +++ b/addonmanager_workers_startup.py @@ -21,11 +21,15 @@ ################################################################################ """Worker thread classes for Addon Manager startup""" + +import base64 import hashlib import io import json import os -from typing import List +from types import SimpleNamespace +from typing import List, Optional, Tuple +from xml.etree.ElementTree import ParseError as XmlParseError import zipfile from PySideWrapper import QtCore @@ -33,11 +37,15 @@ from addonmanager_macro import Macro from Addon import Addon, MissingDependencies -from AddonCatalog import AddonCatalog +from AddonCatalog import AddonCatalog, AddonCatalogEntry, CatalogEntryMetadata from AddonStats import AddonStats import NetworkManager from addonmanager_git import initialize_git, GitFailed -from addonmanager_metadata import MetadataReader, get_branch_from_metadata +from addonmanager_metadata import ( + MetadataReader, + get_branch_from_metadata, + get_icon_from_metadata, +) import addonmanager_utilities as utils import addonmanager_freecad_interface as fci @@ -90,132 +98,220 @@ def run(self): return def _get_custom_addons(self): + """Create and emit an Addon for each custom repository the user has configured.""" - # querying custom addons first - addon_list = fci.Preferences().get("CustomRepositories").split("\n") - custom_addons = [] - for addon in addon_list: - if " " in addon: - addon_and_branch = addon.split(" ") - custom_addons.append({"url": addon_and_branch[0], "branch": addon_and_branch[1]}) - else: - custom_addons.append({"url": addon, "branch": "master"}) - for addon in custom_addons: + for url, branch in self._parse_custom_repositories(): if self.current_thread.isInterruptionRequested(): return - if addon and addon["url"]: - if addon["url"][-1] == "/": - addon["url"] = addon["url"][0:-1] # Strip trailing slash - addon["url"] = addon["url"].split(".git")[0] # Remove .git - name: str = addon["url"].split("/")[-1] - if name in self.package_names: - # We already have something with this name, skip this one - fci.Console.PrintWarning( - translate("AddonsInstaller", "WARNING: Duplicate addon {} ignored").format( - name - ) - ) - continue - fci.Console.PrintLog( - f"Adding custom location {addon['url']} with branch {addon['branch']}\n" + name = url.split("/")[-1] + if name in self.package_names: + # We already have something with this name, skip this one + fci.Console.PrintWarning( + translate("AddonsInstaller", "WARNING: Duplicate addon {} ignored").format(name) + + "\n" ) - self.package_names.append(name) - addon_dir = os.path.join(self.mod_dir, name) - if os.path.exists(addon_dir) and os.listdir(addon_dir): - state = Addon.Status.UNCHECKED - else: - state = Addon.Status.NOT_INSTALLED - repo = Addon(name, addon["url"], state, addon["branch"]) - md_file = os.path.join(addon_dir, "package.xml") - if os.path.isfile(md_file): - # load_metadata_file calls set_metadata(), populating repo.metadata, - # repo.description, repo.display_name etc. for the UI to display. - # It handles ParseError internally, leaving repo.metadata as None on failure. - repo.load_metadata_file(md_file) - if repo.metadata: - repo.installed_metadata = repo.metadata - repo.installed_version = repo.metadata.version - repo.updated_timestamp = os.path.getmtime(md_file) - repo.verify_url_and_branch(addon["url"], addon["branch"]) - # Load icon bytes from the local install directory so the - # list view can display the addon's actual icon. - if repo.metadata.icon: - icon_file = os.path.join(addon_dir, repo.metadata.icon) - if os.path.isfile(icon_file): - try: - with open(icon_file, "rb") as f: - repo.icon_data = f.read() - except OSError as e: - fci.Console.PrintWarning( - f"Could not load icon for custom addon {name}: {e}\n" - ) - else: - # Not installed: try to fetch package.xml from the remote repo so the - # browse view can show description and icon. - self._fetch_remote_custom_addon_metadata(repo) + continue + fci.Console.PrintLog(f"Adding custom location {url} with branch {branch}\n") + self.package_names.append(name) + self.addon_repo.emit(self._create_custom_addon(name, url, branch)) - self.addon_repo.emit(repo) + @staticmethod + def _parse_custom_repositories() -> List[Tuple[str, str]]: + """Parse the CustomRepositories preference into a list of (url, branch) pairs. Each line of + the preference is a repository URL, optionally followed by a space and a branch name.""" + + repositories = [] + for line in fci.Preferences().get("CustomRepositories").split("\n"): + url, _, branch = line.strip().partition(" ") + url = url.rstrip("/").split(".git")[0] + if url: + repositories.append((url, branch.strip() if branch.strip() else "master")) + return repositories + + def _create_custom_addon(self, name: str, url: str, branch: str) -> Addon: + """Create an Addon for a custom repository by synthesizing the catalog entry that the + remote addon catalog would have contained for it, so that a custom addon is constructed by + the same code that constructs an addon from the official catalog.""" + + entry = AddonCatalogEntry( + {"repository": url, "git_ref": branch, "branch_display_name": branch} + ) + location = SimpleNamespace(url=url, branch=branch, name=name) + + entry.metadata = self._fetch_custom_addon_metadata(name, location) + if entry.metadata is None and utils.forget_git_host(location): + # The host we had identified answered nothing at all. It may be running different + # software than it was when it was identified, so work out what it is now and retry. + entry.metadata = self._fetch_custom_addon_metadata(name, location) + if entry.metadata is None: + # Something is wrong: fall back to using the installed metadata + entry.metadata = self._load_installed_addon_metadata(name) + + addon = entry.instantiate_addon(name) + addon.from_custom_repository = True + + if addon.metadata: + # set_metadata() replaces the url and branch with the ones stated in package.xml, but + # for a custom repository the location the user configured is the authoritative one. + addon.verify_url_and_branch(url, branch) + addon.url = url + addon.branch = branch + + if addon.status() == Addon.Status.NO_UPDATE_AVAILABLE: + # There is no cached remote update time for a custom repository, so instantiate_addon() + # could only compare version strings. Leave the addon unchecked so that the update + # worker still gets a chance to run a git-based check on it. + addon.set_status(Addon.Status.UNCHECKED) + + return addon + + def _fetch_custom_addon_metadata(self, name: str, location) -> Optional[CatalogEntryMetadata]: + """Fetch the metadata files of a custom repository directly from its git host.""" + + self._identify_git_host(location) + return self._collect_addon_metadata( + name, + lambda filename: self._fetch_remote_file(utils.construct_git_url(location, filename)), + ) - def _fetch_remote_custom_addon_metadata(self, repo: Addon) -> None: - """For a custom addon that is not yet installed, attempt to fetch its package.xml - from the remote repo so the browse view can show the description and display name. - Failures are non-fatal: the addon will simply show without metadata.""" + def _identify_git_host(self, location) -> None: + """Work out which software this repository's git host is running, unless that is already + known. The answer is stored in the user's preferences, so this happens only once for any + given host, which is why it can afford to ask the host several questions.""" - package_xml_url = utils.construct_git_url(repo, "package.xml") - fci.Console.PrintLog( - f"Fetching remote metadata for custom addon {repo.name} from {package_xml_url}\n" - ) - try: - result = NetworkManager.AM_NETWORK_MANAGER.blocking_get_with_retries( - package_xml_url, - CreateAddonListWorker.ATTEMPT_TIMEOUT_MS, - 1, # single attempt – don't slow down startup for missing files - 0, - quiet=True, # Most addons do not have all of these files, so 404 is not an error - ) - except Exception as e: + host = utils.identify_git_host(location, self._serves_package_xml) + if host is None: fci.Console.PrintLog( - f"Could not fetch remote package.xml for custom addon {repo.name}: {e}\n" + f"Could not work out what software the git host at {location.url} is running. " + f"Add a package.xml file to the repository to let the Addon Manager identify it.\n" ) return + utils.remember_git_host(location, host) - if not result: + def _serves_package_xml(self, url: str) -> bool: + """Whether the repository really served a package.xml at this URL. The contents have to be + checked: a git host that requires a login answers every URL with a 200 and a sign-in page, + and that must not be mistaken for the file.""" + + data = self._fetch_remote_file(url) + if not data: + return False + try: + MetadataReader.from_bytes(data) + except (XmlParseError, RuntimeError): + return False + return True + + def _load_installed_addon_metadata(self, name: str) -> Optional[CatalogEntryMetadata]: + """Load the metadata files of a custom repository from its installed copy, if there is + one.""" + + addon_dir = os.path.join(self.mod_dir, name) + return self._collect_addon_metadata( + name, lambda filename: self._read_installed_file(addon_dir, filename) + ) + + @classmethod + def _collect_addon_metadata(cls, name: str, get_file) -> Optional[CatalogEntryMetadata]: + """Assemble the metadata that the remote cache would have provided for this addon, using + get_file to retrieve the contents of a file given its path relative to the root of the + addon. Returns None if the addon provides none of the metadata files.""" + + metadata = CatalogEntryMetadata() + + package_xml = get_file("package.xml") + if package_xml: + try: + parsed_metadata = MetadataReader.from_bytes(package_xml) + except (XmlParseError, RuntimeError) as e: + parsed_metadata = None + fci.Console.PrintWarning( + translate( + "AddonsInstaller", + "Could not parse the package.xml file of custom addon {}: {}", + ).format(name, str(e)) + + "\n" + ) + if parsed_metadata is not None: + metadata.package_xml = cls._decode(package_xml) + icon_path = get_icon_from_metadata(parsed_metadata) + icon_data = get_file(icon_path) if icon_path else None + if icon_data: + metadata.icon_data = base64.b64encode(icon_data).decode("utf-8") + + requirements_txt = cls._text_file(name, "requirements.txt", get_file) + if requirements_txt: + metadata.requirements_txt = requirements_txt + + metadata_txt = cls._text_file(name, "metadata.txt", get_file) + if metadata_txt: + metadata.metadata_txt = metadata_txt + + if metadata.package_xml or metadata.requirements_txt or metadata.metadata_txt: + return metadata + return None + + @classmethod + def _text_file(cls, name: str, filename: str, get_file) -> Optional[str]: + """Retrieve one of the plain text metadata files, unless what came back is a web page. A + git host that requires a login answers every request with a 200 and a sign-in page: without + this check its HTML would be read as though it were a list of the addon's dependencies.""" + + data = get_file(filename) + if not data: + return None + if cls._is_html(data): fci.Console.PrintLog( - f"No package.xml found at {package_xml_url} for custom addon {repo.name}\n" + f"The {filename} of custom addon {name} came back as a web page, not a file. " + f"Ignoring it: the repository may be private, or may not exist.\n" ) - return + return None + return cls._decode(data) - original_url = repo.url - original_branch = repo.branch + @staticmethod + def _is_html(data: bytes) -> bool: + """Whether these are the contents of a web page rather than of an addon's metadata file.""" + + beginning = data.lstrip()[:64].lower() + return beginning.startswith(b" str: + """Decode the contents of a metadata file, which the standard requires to be UTF-8.""" + + return data.decode("utf-8", errors="replace") + + @classmethod + def _fetch_remote_file(cls, url: str) -> Optional[bytes]: + """Fetch a single file from a remote git host. Returns None if the file could not be + fetched: a custom repository is not required to provide any particular file.""" try: - metadata = MetadataReader.from_bytes(result.data()) - repo.set_metadata(metadata) - except Exception as e: - fci.Console.PrintWarning( - f"Could not parse remote package.xml for custom addon {repo.name}: {e}\n" + result = NetworkManager.AM_NETWORK_MANAGER.blocking_get_with_retries( + url, + cls.ATTEMPT_TIMEOUT_MS, + 1, # A single attempt: do not slow down startup for files that do not exist + 0, + quiet=True, # Most addons do not have all of these files, so 404 is not an error ) - return + except (RuntimeError, OSError) as e: + fci.Console.PrintLog(f"Could not fetch {url}: {e}\n") + return None + if not result: + fci.Console.PrintLog(f"No file found at {url}\n") + return None + return result.data() - repo.verify_url_and_branch(original_url, original_branch) - repo.url = original_url - repo.branch = original_branch + @staticmethod + def _read_installed_file(addon_dir: str, filename: str) -> Optional[bytes]: + """Read a single file from an installed addon. Returns None if it does not exist.""" - if repo.metadata and repo.metadata.icon: - icon_url = utils.construct_git_url(repo, repo.metadata.icon) - try: - icon_result = NetworkManager.AM_NETWORK_MANAGER.blocking_get_with_retries( - icon_url, - CreateAddonListWorker.ATTEMPT_TIMEOUT_MS, - 1, - 0, - ) - if icon_result: - repo.icon_data = icon_result.data() - except Exception as e: - fci.Console.PrintLog( - f"Could not fetch remote icon for custom addon {repo.name}: {e}\n" - ) + path = os.path.join(addon_dir, *filename.split("/")) + try: + with open(path, "rb") as f: + return f.read() + except OSError: + return None def get_cache(self, cache_name: str) -> str: cache_file_name = cache_name + "_cache.json"