update tests to use testing mirror

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

conda_oci_mirror/cli.py

@@ -15,9 +15,6 @@ def main():
     pass
 
 
-# If the user is calling the client, set the cache to PWD
-default_cache_dir = os.getcwd()
-
 options = [
     click.option("-s", "--subdir", default=defaults.DEFAULT_SUBDIRS, multiple=True),
     click.option("-p", "--package", help="Select packages", default=[], multiple=True),
@@ -57,7 +54,7 @@ def mirror(channel, subdir, registry, package, cache_dir, dry_run, quiet, debug)
         subdirs=subdir,
         packages=package,
         registry=registry,
-        cache_dir=cache_dir or default_cache_dir,
+        cache_dir=cache_dir,
     )
     m.update(dry_run)
 
@@ -77,7 +74,7 @@ def pull_cache(channel, subdir, registry, package, cache_dir, dry_run, quiet, de
         subdirs=subdir,
         packages=package,
         registry=registry,
-        cache_dir=cache_dir or default_cache_dir,
+        cache_dir=cache_dir,
     )
     m.pull_latest(dry_run)
 
@@ -100,7 +97,7 @@ def push_cache(
         subdirs=subdir,
         packages=package,
         registry=registry,
-        cache_dir=cache_dir or default_cache_dir,
+        cache_dir=cache_dir,
     )
     if push_all:
         m.push_all(dry_run)

conda_oci_mirror/mirror.py

@@ -52,8 +52,7 @@ def __init__(
     ):
         self.channel = channel
         self.subdirs = subdirs or defaults.DEFAULT_SUBDIRS
-        self.packages = packages
-
+        self.packages = packages or []
         # TODO consider placing packages on level of functions
         # We should not need to specify them on init.
 
@@ -85,7 +84,7 @@ def announce(self):
         util.print_item("  Packages:", "all" if not self.packages else self.packages)
 
     @decorators.require_registry
-    def update(self, dry_run=False):
+    def update(self, dry_run=False, serial=False):
         """
         Update from a conda mirror (do a mirror) akin to a pull and a push.
         """
@@ -129,6 +128,8 @@ def update(self, dry_run=False):
             )
 
         # Once we get here, run all tasks, this returns all the items
+        if serial:
+            return runner.run_serial()
         return runner.run()
 
     def iter_subdirs(self):

conda_oci_mirror/package.py

@@ -43,7 +43,7 @@ def check_checksum(path, package_dict):
         return True
 
 
-@retry(attempts=3, timeout=2)
+@retry(attempts=5, timeout=2)
 def download_file(url, dest, checksum_content=None, chunk_size=8192):
     """
     Stream download a file!

conda_oci_mirror/tasks.py

@@ -24,7 +24,8 @@ def wait(self):
         with last_upload_time.get_lock():
             lt = last_upload_time.value
             now = time.time()
-            rt = 0.5
+            # Slighly slower than the original 0.5 rate limit
+            rt = 0.7
             if now - lt < rt:
                 print(f"Rate limit sleep for {(lt + rt) - now}")
                 time.sleep((lt + rt) - now)

conda_oci_mirror/tests/helpers.py

@@ -7,12 +7,32 @@
 sys.path.insert(0, root)
 sys.path.insert(0, here)
 
+import conda_oci_mirror.defaults as defaults  # noqa
+
 # import conda_oci_mirror.defaults as defaults
 from conda_oci_mirror.mirror import Mirror  # noqa
 from conda_oci_mirror.oras import oras  # noqa
 
 
-def get_mirror(subdir, cache_dir, package=None, channel=None):
+def check_media_type(layer):
+    """
+    Ensure the layer media type matches what we expect
+    """
+    if layer["path"].endswith("repodata.json"):
+        assert layer["media_type"] == defaults.repodata_media_type_v1
+    elif layer["path"].endswith("conda"):
+        assert layer["media_type"] == defaults.package_conda_media_type
+    elif layer["path"].endswith("bz2"):
+        assert layer["media_type"] == defaults.package_tarbz2_media_type
+    elif layer["path"].endswith("info.tar.gz"):
+        assert layer["media_type"] == defaults.info_archive_media_type
+    elif layer["path"].endswith("index.json"):
+        assert layer["media_type"] == defaults.info_index_media_type
+    else:
+        raise ValueError(f"Unexpected layer content type {layer}")
+
+
+def get_mirror(cache_dir, subdir=None, package=None, channel=None):
     """
     Shared function to get a mirror for a particular subdir.
     """
@@ -21,8 +41,7 @@ def get_mirror(subdir, cache_dir, package=None, channel=None):
     host = f"{registry_host}:{registry_port}"
 
     # Noarch is the only place redo exists
-    channel = channel or "conda-forge"
-    package = package or "redo"
+    channel = channel or "mirror-testing"
     user = "dinosaur"
 
     # Interacting with a package repo means we interact with the
@@ -31,9 +50,9 @@ def get_mirror(subdir, cache_dir, package=None, channel=None):
 
     return Mirror(
         channel=channel,
-        packages=[package],
+        packages=[package] if package else None,
         registry=registry,
-        subdirs=[subdir],
+        subdirs=[subdir] if subdir else None,
         cache_dir=cache_dir,
     )
 

conda_oci_mirror/tests/test_mirror.py

@@ -10,7 +10,7 @@
 root = os.path.dirname(os.path.dirname(here))
 sys.path.insert(0, root)
 
-from helpers import get_mirror, get_tags  # noqa
+from helpers import check_media_type, get_mirror, get_tags  # noqa
 
 import conda_oci_mirror.repo as repository  # noqa
 from conda_oci_mirror.logger import setup_logger  # noqa
@@ -23,13 +23,18 @@
 
 
 @pytest.mark.parametrize(
-    "subdir,package,ext",
+    "subdir,num_updates,package_name",
     [
-        ("noarch", "redo", "bz2"),
-        ("noarch", "zope.event", "conda"),
+        ("linux-64", 5, "xtensor"),
+        ("osx-64", 6, "xtensor"),
+        ("osx-arm64", 5, "xtensor"),
+        ("win-64", 2, "xtensor"),
+        ("linux-aarch64", 2, "xtensor"),
+        ("linux-ppc64le", 2, "xtensor"),
+        ("noarch", 6, "redo"),
     ],
 )
-def test_mirror(tmp_path, subdir, package, ext):
+def test_mirror(tmp_path, subdir, num_updates, package_name):
     """
     Test creation of a mirror
 
@@ -39,8 +44,32 @@ def test_mirror(tmp_path, subdir, package, ext):
     and checking file structure and/or size.
     """
     cache_dir = os.path.join(tmp_path, "cache")
-    m = get_mirror(subdir, cache_dir, package=package)
-    assert len(m.update()) >= 2
+    m = get_mirror(cache_dir, subdir=subdir)
+    cache_subdir = os.path.join(cache_dir, m.channel, m.subdirs[0])
+
+    assert not os.path.exists(cache_subdir)
+    updates = m.update(serial=True)
+    print(subdir)
+    print(len(updates))
+    assert len(updates) == num_updates
+
+    # Sanity check structure of layers
+    for update in updates:
+        for key in "uri", "layers":
+            assert key in update
+        for layer in update["layers"]:
+            for key in ["path", "title", "media_type", "annotations"]:
+                assert key in layer
+            # The title should be the file basename
+            assert layer["path"].endswith(layer["title"])
+
+            check_media_type(layer)
+
+            # The annotation is needed for the path
+            assert "org.opencontainers.image.title" in layer["annotations"]
+            assert (
+                layer["annotations"]["org.opencontainers.image.title"] == layer["title"]
+            )
 
     # Each subdir should have a directory in the cache with repodata
     # and nothing else
@@ -51,11 +80,12 @@ def test_mirror(tmp_path, subdir, package, ext):
     repodata_file = os.path.join(cache_subdir, "repodata.json")
     repodata = repository.RepoData(repodata_file)
 
+    # The repodata is len(packages) + 2 (latest and tag for repodata.json)
+    assert len(list(repodata.packages)) + 2 == num_updates
+
     # Smallest size is osx-arm64
     for key in ["packages", "packages.conda"]:
         assert key in repodata.data
-        # packages has over 40000, conda has > 70000
-        assert len(repodata.data[key]) >= 7000
 
     # We can use oras to get artifacts we should have pushed
     # We should be able to pull the latest tag
@@ -75,20 +105,23 @@ def test_mirror(tmp_path, subdir, package, ext):
     os.stat(result[0]).st_size == os.stat(repodata_file).st_size
     package_names = repodata.package_names
 
-    # Looks like only noarch has redo
-    if subdir != "noarch":
-        assert m.packages[0] not in package_names
+    # Testing mirror has xtensor, except if only 2
+    if num_updates == 2:
+        assert not package_names
         return
-    assert m.packages[0] in package_names
 
-    expected_repo = f"{m.registry}/{m.channel}/{subdir}/{m.packages[0]}"
+    print(package_names)
+    assert package_name in package_names
+
+    expected_repo = f"{m.registry}/{m.channel}/{subdir}/{package_name}"
     tags = get_tags(expected_repo)
     assert len(tags["tags"]) >= 1
 
     # Get the latest tag - should be newer at end (e.g., conda)
     tag = tags["tags"][-1]
     pull_dir = os.path.join(tmp_path, "package")
-    result = oras.pull(target=f"{expected_repo}:{tag}", outdir=pull_dir)
+    uri = f"{expected_repo}:{tag}"
+    result = oras.pull(target=uri, outdir=pull_dir)
     assert result
 
     # This directory has .bz2 or conda and subdirectory
@@ -102,6 +135,3 @@ def test_mirror(tmp_path, subdir, package, ext):
             assert "info.tar.gz" in os.listdir(fullpath)
             fullpath = os.path.join(pull_dir, result, "info")
             assert "index.json" in os.listdir(fullpath)
-            continue
-
-        assert fullpath.endswith(ext)