update to add ability to use http external registry

damona/pull.py

@@ -31,11 +31,13 @@ class Pull():
 
 
     """
-    def __init__(self, dryrun=False):
-        self.registry = Registry().registry
+    def __init__(self, dryrun=False, from_url=None):
+        self.registry = Registry(from_url=from_url).registry
+        self.from_url = from_url
         self.dryrun = dryrun
 
-    def pull(self, name, output_name=None, pull_folder=images_directory, force=False):
+    def pull(self, name, output_name=None, pull_folder=images_directory,
+        force=False):
 
         # the name must be valid. we use _ to separate name and version for the
         # singularity filemane but the image have a tag (version) and name
@@ -45,7 +47,6 @@ def pull(self, name, output_name=None, pull_folder=images_directory, force=False
         # However, for users and image stored on sylabs or singularity hub, we
         # use the character :
         #
-
         if ":" in name:
             logger.info(f"Looking for {name}...")
             # e.G. fastqc:0.11.9
@@ -75,16 +76,31 @@ def pull(self, name, output_name=None, pull_folder=images_directory, force=False
             registry_name = name + '_' + str(version)
             logger.info("pulling {}".format(registry_name))
 
+
         download_name = self.registry[registry_name]['download']
+
         if output_name is None:
             output_name = registry_name + ".img"
 
         if self.dryrun: 
             pass
         else: # pragma: no cover
-            Client.pull(str(download_name), name=output_name, 
-                pull_folder=pull_folder,
-                force=force) 
+            # By default it downlaods from syslab if it can be found, but one
+            # can provide an url
+            if self.from_url:
+                # The client does not support external https link other than
+                # dockker, library, shub. 
+                cmd = f"singularity pull  --dir {pull_folder} "
+                if force:
+                    cmd += " --force "
+                cmd += f"{download_name}"
+                import subprocess
+
+                subprocess.call(cmd.split())
+            else:
+                Client.pull(str(download_name), name=output_name, 
+                    pull_folder=pull_folder,
+                    force=force) 
             logger.info("File {} uploaded to {}".format(name, images_directory))
 
         # Now, create an alias

damona/registry.py

@@ -36,7 +36,10 @@ class Registry():
 
     """
 
-    def __init__(self):
+    def __init__(self, from_url=None):
+        if from_url == "damona":
+            from_url = "https://biomics.pasteur.fr/drylab/damona/registry.txt"
+        self.from_url = from_url
         self.registry = {}
         self.discovery()
 
@@ -60,8 +63,42 @@ def _read_registry(self, registry):
                     logger.warning("missing binaries field in {}".format(registry))
         return data
 
-
     def discovery(self):
+
+        if self.from_url:
+            self._url_discovery()
+        else:
+            self._local_discovery()
+
+    def _url_discovery(self):
+        self.registry = {}
+        import  urllib.request
+
+        response = urllib.request.urlopen(self.from_url)
+        html = response.read()
+        html = html.decode('utf-8')
+        _class = None
+        for registry in html.split("\n"):
+            if registry.startswith("["):
+                _class = registry.replace('[', '').replace(']','')
+                continue
+            if registry.strip() == "":
+                continue
+            else:
+                data = {}
+                name,version = registry.split("_")
+                version = version.rsplit(".", 1)[0]
+                # todo for now only exe fills the binaries assuming name of
+                # package is the name of the binary
+                self.registry[name+"_"+version] = {
+                    "name": name+"_"+version,   
+                    "download": self.from_url.replace("registry.txt", registry),
+                    "version": version,
+                    "class": _class, 
+                    'binaries': [{name:name}]}
+
+    def _local_discovery(self):
+
         from damona.recipes import __path__
         self._registry_files = glob.glob(__path__[0] + '/*/registry.yaml')
         self._singularity_files = glob.glob(__path__[0] + '/*/Singularity.*')
@@ -98,15 +135,21 @@ def create_registry(self, path):
     """.format(recipe.rsplit("/",1)[1], version))
 
     def get_list(self, pattern=None):
-
-        from damona.recipes import __path__
-        recipes = glob.glob(__path__[0] + '/*/Singularity.*')
-        recipes = [os.path.basename(x) for x in recipes]
-        recipes = [x.replace("Singularity.", "").lower() for x in recipes]
-        recipes = [x.replace("_", ":").lower() for x in recipes]
-        recipes = sorted(recipes)
-
-        if pattern:
-            recipes = [x for x in recipes if pattern in x]
-
-        return recipes
+        if self.from_url is None:
+            from damona.recipes import __path__
+            recipes = glob.glob(__path__[0] + '/*/Singularity.*')
+            recipes = [os.path.basename(x) for x in recipes]
+            recipes = [x.replace("Singularity.", "").lower() for x in recipes]
+            recipes = [x.replace("_", ":").lower() for x in recipes]
+            recipes = sorted(recipes)
+
+            if pattern:
+                recipes = [x for x in recipes if pattern in x]
+
+            return recipes
+        else:
+            self._url_discovery()
+            names = []
+            for k,v in self.registry.items():
+                names.append(k + ":" + v['version'])
+            return names

damona/script.py

@@ -55,6 +55,8 @@ def main():
 @click.option('--force', is_flag=True)
 @click.option('--dryrun', is_flag=True, 
     help="If set, do not pull the image and do not create the binary alias")
+@click.option('--from-url', help="""download image from a remote URL. The URL must
+contain a registry.txt as explained on damona.readthedocs.io""") 
 @click.option('--output-directory', default=images_directory, 
     help="""Where to save the image (default: {})""".format(images_directory))
 def install(**kwargs):
@@ -65,7 +67,7 @@ def install(**kwargs):
     """
     name = kwargs['name']
     from damona.pull import Pull
-    p = Pull(dryrun=kwargs['dryrun'])
+    p = Pull(dryrun=kwargs['dryrun'], from_url=kwargs['from_url'])
     p.pull(name, pull_folder=kwargs["output_directory"],
         force=kwargs['force'])
 
@@ -117,10 +119,12 @@ def build(**kwargs): #pragma: no cover
 @main.command()
 @click.option('--pattern', default=None, 
     help="restrict the output list keeping those with this pattern")
+@click.option('--from-url', help="""download image from a remote URL. The URL must
+contain a registry.txt as explained on damona.readthedocs.io""") 
 def list(**kwargs):
     """List all available images from Damona"""
     from damona.registry import Registry
-    modules = Registry().get_list(pattern=kwargs['pattern'])
+    modules = Registry(from_url=kwargs["from_url"]).get_list(pattern=kwargs['pattern'])
     print(", ".join(modules))