[mamba-content-trust] Add integration test (#3234)

* -Fix trusted_channels in ctx not working with more than one item -Add test for mamba-content-trust (mamba client side) using conda-content-trust (server side) -Add missing install with pip (to fix) * Remove repo_signed (supposed to be generated with testserver_pkg_signing.sh) * Add MAMBA_ROOT_PREFIX to Taskfile.dist.yml
mamba-org · Mar 20, 2024 · cad6793 · cad6793
1 parent 4270dd4
commit cad6793
Show file tree

Hide file tree

Showing 18 changed files with 191 additions and 113 deletions.
diff --git a/.github/workflows/unix_impl.yml b/.github/workflows/unix_impl.yml
@@ -163,3 +163,34 @@ jobs:
           unset CONDARC  # Interferes with tests
           python -m pytest micromamba/tests/ \
             ${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }}
+
+  verify_pkg_tests:
+    name: mamba-content-trust tests
+    needs: ["build_shared_unix"]
+    runs-on: ${{ inputs.os }}
+    if: startsWith(inputs.os, 'ubuntu')
+    steps:
+      - name: Checkout mamba repository
+        uses: actions/checkout@v4
+      - name: Restore workspace
+        uses: ./.github/actions/workspace
+        with:
+          action: restore
+          path: build/
+          key_suffix: ${{ inputs.os }}-${{ inputs.build_type }}
+      - name: Create build environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ./build/environment.lock
+          environment-name: build_env
+      - name: Run tests using conda-content-trust (server side)
+        shell: bash -l {0} -euo pipefail -x
+        run: |
+          export TEST_MAMBA_EXE=$(pwd)/build/micromamba/mamba
+          export MAMBA_ROOT_PREFIX="${HOME}/micromamba"
+          unset CONDARC  # Interferes with tests
+          # FIXME this is apparently a bug to be fixed (pip specs are not installed from env yml file)
+          python -m pip install securesystemslib --no-input --no-deps
+          cd micromamba/test-server
+          ./generate_gpg_keys.sh
+          ./testserver_pkg_signing.sh
diff --git a/Taskfile.dist.yml b/Taskfile.dist.yml
@@ -18,6 +18,7 @@ vars:
   DOCS_DOXYGEN_XML_DIR: '{{.PWD}}/{{.DOCS_DIR}}/doxygen-xml'
   MAMBA_NAME: 'mamba'  # Depend on preset...
   TEST_MAMBA_EXE: '{{.PWD}}/{{.CMAKE_BUILD_DIR}}/micromamba/{{.MAMBA_NAME}}'
+  MAMBA_ROOT_PREFIX: '${HOME}/micromamba'
   CPU_PERCENTAGE: 75
   CPU_TOTAL:
     sh: >-
@@ -222,6 +223,7 @@ tasks:
     deps: [{task: '_build', vars: {target: '{{.MAMBA_NAME'}}]
     env:
       TEST_MAMBA_EXE: '{{.TEST_MAMBA_EXE}}'
+      MAMBA_ROOT_PREFIX: '{{.MAMBA_ROOT_PREFIX}}'
     # Explicitly using this as var since env does not override shell environment
     vars:
       GNUPGHOME: '{{.BUILD_DIR}}/gnupg'

diff --git a/libmamba/include/mamba/core/context.hpp b/libmamba/include/mamba/core/context.hpp
@@ -39,10 +39,11 @@ namespace mamba
         bool extra_safety_checks = false;
         bool verify_artifacts = false;
 
-        // TODO test with multiple channels in there and check behavior
-        // i.e uncommenting `conda-forge` when possible
-        // and removing "http://127.0.0.1:8000/get/channel0" (should only be in integration tests)
-        // Should we consider removing this and use `channels` instead?
+        // TODO Uncomment `conda-forge` or whatever trusted_channels when possible
+        // (i.e server side package signing ready)
+        // Remove "http://127.0.0.1:8000/get/channel0"
+        // (should only be used in integration tests,
+        // this one is for testing with quetz)
         std::vector<std::string> trusted_channels = {
             /*"conda-forge", */ "http://127.0.0.1:8000/get/channel0"
         };

diff --git a/libmamba/include/mamba/core/repo_checker_store.hpp b/libmamba/include/mamba/core/repo_checker_store.hpp
@@ -32,11 +32,11 @@ namespace mamba
 
         explicit RepoCheckerStore(repo_checker_list checkers);
 
-        [[nodiscard]] auto find_checker(const Channel& chan) const -> const RepoChecker*;
+        [[nodiscard]] auto find_checker(const Channel& chan) -> RepoChecker*;
 
-        [[nodiscard]] auto contains_checker(const Channel& chan) const -> bool;
+        [[nodiscard]] auto contains_checker(const Channel& chan) -> bool;
 
-        [[nodiscard]] auto at_checker(const Channel& chan) const -> const RepoChecker&;
+        [[nodiscard]] auto at_checker(const Channel& chan) -> RepoChecker&;
 
     private:
 

diff --git a/libmamba/src/core/repo_checker_store.cpp b/libmamba/src/core/repo_checker_store.cpp
@@ -42,7 +42,6 @@ namespace mamba
 
                 // Initialization
                 fs::create_directories(checker.cache_path());
-                checker.generate_index_checker();
 
                 repo_checkers.emplace_back(std::move(chan), std::move(checker));
             }
@@ -55,7 +54,7 @@ namespace mamba
     {
     }
 
-    auto RepoCheckerStore::find_checker(const Channel& chan) const -> const RepoChecker*
+    auto RepoCheckerStore::find_checker(const Channel& chan) -> RepoChecker*
     {
         for (auto& [candidate_chan, checker] : m_repo_checkers)
         {
@@ -67,12 +66,12 @@ namespace mamba
         return nullptr;
     }
 
-    auto RepoCheckerStore::contains_checker(const Channel& chan) const -> bool
+    auto RepoCheckerStore::contains_checker(const Channel& chan) -> bool
     {
         return find_checker(chan) != nullptr;
     }
 
-    auto RepoCheckerStore::at_checker(const Channel& chan) const -> const RepoChecker&
+    auto RepoCheckerStore::at_checker(const Channel& chan) -> RepoChecker&
     {
         if (auto ptr = find_checker(chan))
         {

diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp
@@ -592,6 +592,7 @@ namespace mamba
                             if (repo_checker)
                             {
                                 LOG_INFO << "RepoChecker successfully created.";
+                                repo_checker->generate_index_checker();
                                 repo_checker->verify_package(
                                     pkg.json_signable(),
                                     std::string_view(pkg.signatures)
@@ -600,6 +601,10 @@ namespace mamba
                             else
                             {
                                 LOG_ERROR << "Could not create a valid RepoChecker.";
+                                throw std::runtime_error(fmt::format(
+                                    R"(Could not verify "{}". Please make sure the package signatures are available and 'trusted-channels' are configured correctly. Alternatively, try downloading without '--verify-artifacts' flag.)",
+                                    pkg.name
+                                ));
                             }
                         }
                         LOG_INFO << "'" << pkg.name << "' trusted from '" << pkg.channel << "'";

diff --git a/micromamba/test-server/repo/noarch/current_repodata.json b/micromamba/test-server/repo/noarch/current_repodata.json
@@ -8,14 +8,12 @@
       "build_number": 0,
       "depends": [],
       "license": "BSD",
-      "license_family": "BSD",
-      "md5": "2a8595f37faa2950e1b433acbe91d481",
+      "md5": "719449904d9d4ac9853437a9504f87c5",
       "name": "test-package",
-      "noarch": "generic",
-      "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
-      "size": 5719,
+      "sha256": "1c8942fd0ad0dd3f780bff1a159a6ff8b82968965a630b3e1bff27b7168f68b5",
+      "size": 5747,
       "subdir": "noarch",
-      "timestamp": 1613117294885,
+      "timestamp": 1613117294,
       "version": "0.1"
     }
   },

diff --git a/micromamba/test-server/repo/noarch/current_repodata.json.bz2 b/micromamba/test-server/repo/noarch/current_repodata.json.bz2
diff --git a/micromamba/test-server/repo/noarch/repodata.json b/micromamba/test-server/repo/noarch/repodata.json
@@ -8,14 +8,12 @@
       "build_number": 0,
       "depends": [],
       "license": "BSD",
-      "license_family": "BSD",
-      "md5": "2a8595f37faa2950e1b433acbe91d481",
+      "md5": "719449904d9d4ac9853437a9504f87c5",
       "name": "test-package",
-      "noarch": "generic",
-      "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
-      "size": 5719,
+      "sha256": "1c8942fd0ad0dd3f780bff1a159a6ff8b82968965a630b3e1bff27b7168f68b5",
+      "size": 5747,
       "subdir": "noarch",
-      "timestamp": 1613117294885,
+      "timestamp": 1613117294,
       "version": "0.1"
     }
   },

diff --git a/micromamba/test-server/repo/noarch/repodata.json.bz2 b/micromamba/test-server/repo/noarch/repodata.json.bz2
diff --git a/micromamba/test-server/repo/noarch/repodata_from_packages.json b/micromamba/test-server/repo/noarch/repodata_from_packages.json
@@ -8,14 +8,12 @@
       "build_number": 0,
       "depends": [],
       "license": "BSD",
-      "license_family": "BSD",
-      "md5": "2a8595f37faa2950e1b433acbe91d481",
+      "md5": "719449904d9d4ac9853437a9504f87c5",
       "name": "test-package",
-      "noarch": "generic",
-      "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
-      "size": 5719,
+      "sha256": "1c8942fd0ad0dd3f780bff1a159a6ff8b82968965a630b3e1bff27b7168f68b5",
+      "size": 5747,
       "subdir": "noarch",
-      "timestamp": 1613117294885,
+      "timestamp": 1613117294,
       "version": "0.1"
     }
   },

diff --git a/micromamba/test-server/repo/noarch/repodata_from_packages.json.bz2 b/micromamba/test-server/repo/noarch/repodata_from_packages.json.bz2
diff --git a/micromamba/test-server/repo/noarch/test-package-0.1-0.tar.bz2 b/micromamba/test-server/repo/noarch/test-package-0.1-0.tar.bz2
diff --git a/micromamba/test-server/repo_signed/1.root.json b/micromamba/test-server/repo_signed/1.root.json
diff --git a/micromamba/test-server/repo_signed/key_mgr.json b/micromamba/test-server/repo_signed/key_mgr.json
diff --git a/micromamba/test-server/repo_signed/noarch/repodata.json b/micromamba/test-server/repo_signed/noarch/repodata.json
diff --git a/micromamba/test-server/reposerver.py b/micromamba/test-server/reposerver.py
@@ -89,8 +89,10 @@ def make_signed_repo(self) -> Path:
         self.folder.mkdir(exist_ok=True)
         self.create_root(self.keys)
         self.create_key_mgr(self.keys)
+        self.create_pkg_mgr(self.keys)
         for f in glob.glob(str(self.in_folder / "**" / "repodata.json")):
             self.sign_repodata(Path(f), self.keys)
+        self.copy_signing_root_file()
         return self.folder
 
     def create_root(self, keys):
@@ -168,10 +170,53 @@ def create_key_mgr(self, keys):
         # Doing delegation processing.
         cct_authentication.verify_delegation("key_mgr", key_mgr_metadata, root_metadata)
 
-        print("[reposigner] success: key mgr metadata verified based on root metadata.")
+        print("[reposigner] Success: key_mgr metadata verified based on root metadata.")
 
         return key_mgr
 
+    # Adding this to be compatible with `mamba` (in `conda` they don't seem to have the `pkg_mgr.json` file)
+    # But the signing does use delegation to `pkg_mgr` role in both cases
+    def create_pkg_mgr(self, keys):
+        private_key_pkg_mgr = cct_common.PrivateKey.from_hex(keys["pkg_mgr"][0]["private"])
+        pkg_mgr = cct_metadata_construction.build_delegating_metadata(
+            metadata_type="pkg_mgr",
+            delegations=None,
+            version=1,
+            # timestamp   default: now
+            # expiration  default: now plus root expiration default duration
+        )
+
+        pkg_mgr = cct_signing.wrap_as_signable(pkg_mgr)
+
+        # sign dictionary in place
+        cct_signing.sign_signable(pkg_mgr, private_key_pkg_mgr)
+
+        pkg_mgr_serialized = cct_common.canonserialize(pkg_mgr)
+        with open(self.folder / "pkg_mgr.json", "wb") as fobj:
+            fobj.write(pkg_mgr_serialized)
+
+        # let's run a verification
+        key_mgr_metadata = cct_common.load_metadata_from_file(self.folder / "key_mgr.json")
+        pkg_mgr_metadata = cct_common.load_metadata_from_file(self.folder / "pkg_mgr.json")
+
+        cct_common.checkformat_signable(key_mgr_metadata)
+
+        if "delegations" not in key_mgr_metadata["signed"]:
+            raise ValueError('Expected "delegations" entry in key_mgr metadata.')
+
+        key_mgr_delegations = key_mgr_metadata["signed"]["delegations"]  # for brevity
+        cct_common.checkformat_delegations(key_mgr_delegations)
+        if "pkg_mgr" not in key_mgr_delegations:
+            raise ValueError('Missing expected delegation to "pkg_mgr" in key_mgr metadata.')
+        cct_common.checkformat_delegation(key_mgr_delegations["pkg_mgr"])
+
+        # Doing delegation processing.
+        cct_authentication.verify_delegation("pkg_mgr", pkg_mgr_metadata, key_mgr_metadata)
+
+        print("[reposigner] Success: pkg_mgr metadata verified based on key_mgr metadata.")
+
+        return pkg_mgr
+
     def sign_repodata(self, repodata_fn, keys):
         target_folder = self.folder / repodata_fn.parent.name
         if not target_folder.exists():
@@ -185,6 +230,31 @@ def sign_repodata(self, repodata_fn, keys):
         cct_signing.sign_all_in_repodata(str(final_fn), pkg_mgr_key)
         print(f"[reposigner] Signed {final_fn}")
 
+        # Copy actual 'test-package-0.1-0.tar.bz2' to serving directory ('repo_signed')
+        pkg_bz2_src_fn = repodata_fn.parent / "test-package-0.1-0.tar.bz2"
+        pkg_bz2_dst_fn = target_folder / "test-package-0.1-0.tar.bz2"
+        print("copy", pkg_bz2_src_fn, pkg_bz2_dst_fn)
+        shutil.copyfile(pkg_bz2_src_fn, pkg_bz2_dst_fn)
+        print("[reposigner] 'test-package-0.1-0.tar.bz2' copied")
+
+    def copy_signing_root_file(self):
+        # Copy root json file to 'ref_path'
+        # as this should be available in a safe place locally
+        root_prefix = Path(os.environ["MAMBA_ROOT_PREFIX"])
+        if not root_prefix:
+            fatal_error("MAMBA_ROOT_PREFIX is not set!")
+
+        # '7da7dc10' corresponds to id of channel 'http://localhost:8000/mychannel'
+        channel_initial_trusted_root_role = root_prefix / "etc/trusted-repos/7da7dc10"
+        if not channel_initial_trusted_root_role.exists():
+            os.makedirs(channel_initial_trusted_root_role)
+
+        shutil.copy(
+            self.folder / "1.root.json",
+            channel_initial_trusted_root_role / "root.json",
+        )
+        print("Initial trusted root copied")
+
 
 class ChannelHandler(SimpleHTTPRequestHandler):
     url_pattern = re.compile(r"^/(?:t/[^/]+/)?([^/]+)")
@@ -219,6 +289,12 @@ def do_GET(self) -> None:
 
         self.send_response(404)
 
+    def do_HEAD(self) -> None:
+        if self.path.endswith("_mgr.json"):
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+
     def basic_do_HEAD(self) -> None:
         self.send_response(200)
         self.send_header("Content-type", "text/html")