Artifacts

ci-build

@@ -17,7 +17,7 @@ omego -h
 #Tests rely on a non-zero error code being returned on failure
 if [ $TEST = install ]; then
   export OMERODIR='./OMERO.server-5.6.4-ice36-b232'
-  omego install --initdb --dbhost localhost --dbname omero --prestartfile $HOME/config.omero -v --release 5.6.4 --ice 3.6 --no-web --no-start
+  omego install --github ome/openmicroscopy --initdb --dbhost localhost --dbname omero --prestartfile $HOME/config.omero -v --release 5.6.4 --no-web --no-start
 
   ls OMERO.server
   # Should return 0 DB_UPTODATE
@@ -34,7 +34,7 @@ fi
 
 #Test a multistage DB upgrade (5.3 -> 5.4) as part of the server upgrade
 if [ $TEST = upgrade ]; then
-  omego download --release 5.3 --ice 3.6 server
+  omego download --github ome/openmicroscopy --release 5.3.5 server
   ln -s OMERO.server-5.3.5-ice36-b73 OMERO.server;
 
   # Should return 3 DB_INIT_NEEDED
@@ -50,7 +50,7 @@ if [ $TEST = upgrade ]; then
   # Should return 0 DB_UPTODATE
   omego db upgrade -n --serverdir OMERO.server
 
-  omego download --release 5.5 --ice 3.6 server --sym download-server-50
+  omego download --github ome/openmicroscopy --release 5.5.1 server --sym download-server-50
   # Should return 2 DB_UPGRADE_NEEDED
   RC=0;
   omego db upgrade -n --dbname omero --serverdir download-server-50 || RC=$?

omego/artifacts.py

@@ -8,6 +8,7 @@
 from builtins import str
 from past.builtins import basestring
 from builtins import object
+import json
 import os
 import logging
 
@@ -27,6 +28,9 @@
 except ImportError:
     from elementtree.ElementTree import XML
 
+from distutils.version import LooseVersion
+import requests
+
 standard_library.install_aliases()  # noqa
 log = logging.getLogger("omego.artifacts")
 
@@ -52,7 +56,9 @@ def __init__(self, args):
             args.ci = 'https://latest-ci.openmicroscopy.org/jenkins'
         if not args.branch:
             args.branch = 'latest'
-        if args.build or re.match(r'[A-Za-z]\w+-\w+', args.branch):
+        if args.github:
+            self.artifacts = GithubArtifacts(args)
+        elif args.build or re.match(r'[A-Za-z]\w+-\w+', args.branch):
             self.artifacts = JenkinsArtifacts(args)
         elif re.match('[0-9]+|latest$', args.branch):
             self.artifacts = ReleaseArtifacts(args)
@@ -473,6 +479,89 @@ def read_downloads(dlurl):
         return dl_icever
 
 
+class GithubArtifacts(ArtifactsList):
+    """
+    Fetch artifacts from GitHub releases
+    """
+
+    def __init__(self, args):
+        super(GithubArtifacts, self).__init__()
+        self.args = args
+        tag = None
+        if re.match(r'[0-9]+\.[0-9]+\.[0-9]+', args.branch):
+            tag = args.branch
+        elif re.match(r'[0-9]+\.[0-9]+', args.branch):
+            tag = self.retrieve_tag(args.github, args.branch, major_only=False)
+        elif re.match(r'[0-9]+', args.branch):
+            tag = self.retrieve_tag(args.github, args.branch)
+        else:
+            raise ArtifactException(
+                'Only GitHub tags are supported', args.branch)
+
+        if args.ice:
+            raise ArtifactException(
+                'Ice argument not supported for GitHub releases', args.ice)
+
+        if tag is None:
+            raise ArtifactException(
+                'No tag matching the specified value found. \
+                 Please check the GitHub releases page.', args.branch)
+
+        dl_url = 'https://api.github.com/repos/%s/releases/tags/v%s' % (
+                args.github, tag)
+        artifacturls = self.read_downloads(dl_url)
+        if len(artifacturls) <= 0:
+            raise AttributeError(
+                "No artifacts, please check the GitHub releases page.")
+        self.find_artifacts(artifacturls)
+
+    @staticmethod
+    def retrieve_tag(github_repo, value, major_only=True):
+        '''
+        First find all the tags associated to a release.
+        Find the tag corresponding to the specified value
+        either major or major.minor
+        '''
+        url = 'https://api.github.com/repos/%s/releases' % github_repo
+        json = requests.get(url).json()
+        tags = []
+        for i in range(len(json)):
+            tags.append(json[i]['tag_name'].replace("v", ""))
+
+        sorted(tags, key=LooseVersion)
+        for tag in tags:
+            major, minor, patch = re.search(r'(\d+)\.(\d+)\.(\d+)', str(tag)).groups() # noqa
+            if major_only:
+                if value == major:
+                    return tag
+            else:
+                if value == major + "." + minor:
+                    return tag
+
+    @staticmethod
+    def read_downloads(dlurl):
+        url = None
+        d = None
+        try:
+            url = fileutils.open_url(dlurl)
+            log.debug('Fetching html from %s code:%d', url.url, url.code)
+            if url.code != 200:
+                log.error('Failed to get HTML from %s (code %d)',
+                          url.url, url.code)
+                raise Stop(
+                    20, 'Downloads page failed, is the version correct?')
+
+            d = json.load(url)
+        except HTTPError as e:
+            log.error('Failed to get HTML from %s (%s)', dlurl, e)
+            raise Stop(20, 'Downloads page failed, is the version correct?')
+        finally:
+            if url:
+                url.close()
+
+        return [a['browser_download_url'] for a in d['assets']]
+
+
 class DownloadCommand(Command):
     """
     Download an OMERO artifact from either a downloads or a Continuous

omego/env.py

@@ -131,6 +131,11 @@ def __init__(self, parser):
             " DOWNLOADURL/omero/<version>/artifacts. Default: "
             "http://downloads.openmicroscopy.org")
 
+        Add(group, "github", "",
+            help="GitHub repository containing release artifacts in form "
+            "'org/repository', if set this will override all other artifact "
+            "sources, default disabled")
+
         Add(group, "ice",
             "", help="Ice version, default is the latest (release only)")
 

requirements.txt

@@ -1,2 +1,3 @@
 future
 yaclifw>=0.1.1
+requests

setup.py

@@ -121,7 +121,8 @@ def run_tests(self):
       # Skipping argparse for Python 2.7 and greater.
       install_requires=[
           'future',
-          'yaclifw>=0.1.1'
+          'yaclifw>=0.1.1',
+          'requests'
       ],
 
       # Using global variables

test/integration/test_download.py

@@ -25,7 +25,7 @@
 import pytest  # noqa
 
 from yaclifw.framework import main
-from omego.artifacts import DownloadCommand
+from omego.artifacts import ArtifactException, DownloadCommand
 
 
 class Downloader(object):
@@ -136,3 +136,60 @@ def testDownloadJar(self, tmpdir):
             assert len(files) == 1
             assert files[0].basename.endswith(".jar")
             assert files[0].basename.startswith('formats-api')
+
+
+class TestDownloadGithub(Downloader):
+
+    def setup_class(self):
+        self.artifact = 'insight'
+
+    def testDownloadGithub(self, tmpdir):
+        with tmpdir.as_cwd():
+            self.download(
+                '--release', '5.5.8',
+                '--github', 'ome/omero-insight',
+                '--sym', 'auto')
+        files = tmpdir.listdir()
+        assert len(files) == 3
+        print([f.basename for f in files])
+        assert sorted(f.basename for f in files) == [
+            'OMERO.insight',
+            'OMERO.insight-5.5.8',
+            'OMERO.insight-5.5.8.zip',
+        ]
+
+    def testDownloadGithub_major(self, tmpdir):
+        with tmpdir.as_cwd():
+            self.download(
+                '--release', '5',
+                '--github', 'ome/omero-insight',
+                '--sym', 'auto')
+        files = tmpdir.listdir()
+        assert len(files) > 0
+        print([f.basename for f in files])
+
+    def testDownloadGithub_major_minor(self, tmpdir):
+        with tmpdir.as_cwd():
+            self.download(
+                '--release', '5.7',
+                '--github', 'ome/omero-insight',
+                '--sym', 'auto')
+        files = tmpdir.listdir()
+        assert len(files) > 0
+        print([f.basename for f in files])
+
+    def testDownloadGithub_major_invalid(self):
+        with pytest.raises(ArtifactException) as exc:
+            self.download(
+                '--release', '100',
+                '--github', 'ome/omero-insight',
+                '--sym', 'auto')
+        assert 'No tag' in exc.value.args[0]
+
+    def testDownloadGithub_major_minor_invalid(self):
+        with pytest.raises(ArtifactException) as exc:
+            self.download(
+                '--release', '100.1',
+                '--github', 'ome/omero-insight',
+                '--sym', 'auto')
+        assert 'No tag' in exc.value.args[0]