Merge pull request #6 from jdcasey/master

Added tests and finished staging + command impls (needs tests)

README.rst

@@ -1,6 +1,12 @@
 nexup - Synchronize Maven repositories to Sonatype Nexus
 ============================================================
 
+.. image:: https://coveralls.io/repos/github/release-engineering/nexup/badge.svg
+:target: https://coveralls.io/github/release-engineering/nexup
+
+.. image:: https://travis-ci.org/release-engineering/nexup.svg?branch=master
+    :target: https://travis-ci.org/release-engineering/nexup
+
 .. split here
 
 ``nexup`` is a command-line utility for uploading Maven repositories to Sonatype's Nexus repository manager.

nexup/command.py

@@ -3,10 +3,13 @@
 import config
 import repo as repos
 import group as groups
+import archive
+import staging
 import os.path
 import sys
 import re
 import click
+import tempfile
 
 RELEASE_GROUP_NAME = 'product-ga'
 TECHPREVIEW_GROUP_NAME = 'product-techpreview'
@@ -15,10 +18,12 @@
 @click.command()
 @click.argument('repo', type=click.Path(exists=True))
 @click.option('--environment', '-e', help='The target Nexus environment (from ~/.config/nexup/nexup.yaml)')
+@click.option('--product', '-p', help='The product key, used to lookup profileId from the configuration')
+@click.option('--version', '-v', help='The product version, used in repository definition metadata')
 @click.option('--ga', '-g', is_flag=True, default=False, help='Push content to the GA group (as opposed to earlyaccess)')
 @click.option('--debug', '-D', is_flag=True, default=False)
-def push(repo, environment, ga=False, debug=False):
-    "Push maven repository content to a Nexus staging repository."
+def push(repo, environment, product, version, ga=False, debug=False):
+    "Push maven repository content to a Nexus staging repository, and add the staging repository to the appropriate content groups."
 
     nexus_config = config.load(environment)
 
@@ -33,32 +38,41 @@ def push(repo, environment, ga=False, debug=False):
         print "Pushing: %s content to: %s" % (repo, environment)
 
         # produce a set of clean repository zips for PUT upload.
+        zips_dir = tempfile.mkdtemp()
+        print "Creating ZIP archives in: %s" % zips_dir
         if os.path.isdir(repo):
             print "Processing repository directory: %s" % repo
 
-            # TODO: Walk the directory tree, and create a zip.
+            # Walk the directory tree, and create a zip.
+            zip_paths = archive.create_partitioned_zips_from_dir(repo, zips_dir)
         else:
             print "Processing repository zip archive: %s" % repo
-            # TODO: Open the zip, walk the entries and normalize the structure to clean zip (if necessary)
+
+            # Open the zip, walk the entries and normalize the structure to clean zip (if necessary)
+            zip_paths = archive.create_partitioned_zips_from_zip(repo, zips_dir)
 
 
-        # TODO: Open new staging repository with description
+        # Open new staging repository with description
+        staging_repo_id = staging.start_staging_repo(session, nexus_config, product, version, ga)
 
-        # TODO: HTTP PUT clean repository zips to Nexus.
+        # HTTP PUT clean repository zips to Nexus.
+        delete_first = True
+        for zipfile in zip_paths:
+            repo.push_zip(session, staging_repo_id, zipfile, delete_first)
+            delete_first = False
 
-        # TODO: Close staging repository
-        staging_repo_name = "FIXME"
+        # Close staging repository
+        staging.finish_staging_repo(session, nexus_config, staging_repo_id, product, version, ga)
 
-        for group_name in groups:
-            group = groups.load(session, group_name, ignore_missing=True)
+        for group_id in groups:
+            group = groups.load(session, group_id, ignore_missing=True)
             if group is not None:
-                print "Adding %s to group: %s" % (staging_repo_name, group_name)
+                print "Adding %s to group: %s" % (staging_repo_id, group_id)
 
-                # TODO: How do you reference a staging repository for group membership??
-                group.append_member(session, staging_repo_name).save(session)
+                group.append_member(session, staging_repo_id).save(session)
             else:
-                print "No such group: %s" % group_name
-                raise Exception("No such group: %s" % group_name)
+                print "No such group: %s" % group_id
+                raise Exception("No such group: %s" % group_id)
     finally:
         if session is not None:
             session.close()

nexup/config.py

@@ -12,21 +12,42 @@
 SSL_VERIFY='ssl-verify'
 PREEMPTIVE_AUTH='preemptive-auth'
 INTERACTIVE='interactive'
+PROFILE_MAP = 'profile-map'
+
+GA_PROFILE = 'ga'
+EA_PROFILE = 'ea'
 
 class NexusConfig(object):
-    def __init__(self, data):
+    def __init__(self, name, data):
+        self.name = name
         self.url = data[URL]
         self.ssl_verify = data.get(SSL_VERIFY, True)
         self.preemptive_auth = data.get(PREEMPTIVE_AUTH, False)
         self.username = data.get(USERNAME, None)
         self.password = data.get(PASSWORD, None)
         self.interactive = data.get(INTERACTIVE, True)
+        self.profile_map = data.get(PROFILE_MAP, {})
 
     def get_password(self):
         if self.password and self.password.startswith("@oracle:"):
             return eval_password(self.username, oracle=self.password, interactive=self.interactive)
         return self.password
 
+    def get_profile_id(self, product, is_ga):
+        profiles = self.profile_map.get(product)
+        if profiles is None:
+            raise Exception( "Product %s not configured in 'profile-maps' of configuration: %s (case-sensitive)" % (product, config.name) )
+
+        quality_level = GA_PROFILE if is_ga is True else EA_PROFILE
+        profile_id = profiles.get(quality_level)
+        if profile_id is None:
+            raise Exception( 
+                "ProfileID not configured for quality level: %s in 'profile-maps' of configuration: %s for the product: '%s' (case-sensitive)" % 
+                (quality_level, config.name, product) )
+
+        return profile_id
+
+
     def __str__(self):
         return """RCMNexusConfig [
     URL: %(url)s
@@ -57,7 +78,7 @@ def load(environment, cli_overrides=None):
 	if cli_overrides is not None:
 		data.update(cli_overrides)
 
-	return NexusConfig(data)
+	return NexusConfig(environment, data)
 
 
 #############################################################################

nexup/group.py

@@ -14,6 +14,10 @@
 from lxml import (objectify,etree)
 from session import (nexus_boolean, python_boolean)
 import repo as repos
+import os
+import re
+
+GROUP_CONTENT_URI_RE='(.+)/content/groups/.+'
 
 GROUPS_PATH = '/service/local/repo_groups'
 NAMED_GROUP_PATH = GROUPS_PATH + '/{key}'
@@ -28,38 +32,45 @@ def load(session, group_key, ignore_missing=True):
     path = NAMED_GROUP_PATH.format(key=group_key)
     response, group_xml = session.get(path, ignore_404=ignore_missing)
 
-    if ignore_missing and response.status == 404:
+    if ignore_missing and response.status_code == 404:
 #        print "Group %s not found. Returning None" % group_key
         return None
 
     doc = objectify.fromstring(group_xml)
-    return Group(doc.data.id, doc.data.name, debug=session.debug)._set_xml_obj(doc)
+    return Group(doc)
 
 class Group(object):
     """Convenience wrapper class around group xml document (via objectify.fromstring(..)).
        Provides methods for accessing data without knowledge of the xml document structure.
     """
-    def __init__(self, key, name, debug=False):
-        self.new = True
-        self.debug = debug
-        self.xml = objectify.Element('repo-group')
-        self.data = etree.SubElement(self.xml, 'data')
-        self.data.id = key
-        self.data.name = name
-        self.data.provider='maven2'
-        self.data.format='maven2'
-        self.data.repoType = 'group'
-        self.data.exposed = nexus_boolean(True)
+    def __init__(self, key_or_doc, name=None, debug=False):
+        if type(key_or_doc) is objectify.ObjectifiedElement:
+            self.new=False
+            self._set_xml_obj(key_or_doc)
+        elif name is None:
+            raise Exception('Invalid new repository; must supply key AND name (name is missing)')
+        else:
+            self.new = True
+            self.debug = debug
+            self.xml = objectify.Element('repo-group')
+            self.data = etree.SubElement(self.xml, 'data')
+            self.data.id = key_or_doc
+            self.data.name = name
+            self.data.provider='maven2'
+            self.data.format='maven2'
+            self.data.repoType = 'group'
+            self.data.exposed = nexus_boolean(True)
 
-    def get_exposed(self):
+    def exposed(self):
         exposed = self.data.exposed
         pyval = python_boolean(exposed)
         if self.debug is True:
             print "Got group exposed value: %s (type: %s, converted to boolean: %s)" % (exposed, type(exposed), pyval)
         return pyval
 
     def set_exposed(self, exposed):
-        self.data.exposed = nexus_boolean(exposed)
+        # self.data.exposed = nexus_boolean(exposed)
+        self.data.exposed = exposed
         return self
 
     def _set_xml_string(self, xml):
@@ -79,11 +90,14 @@ def _set_xml_obj(self, xml):
         self._backup_xml = self.render()
         return self
 
-    def get_name(self):
+    def name(self):
         return self.data.name
 
-    def get_id(self):
+    def id(self):
         return self.data.id
+
+    def content_uri(self):
+        return self.data.contentResourceURI
 
     def set_name(self, name):
         self.data.name = name
@@ -115,7 +129,16 @@ def append_member(self, session, repo_key):
             member = etree.SubElement(members, 'repo-group-member')
             member.id = repo.data.id
             member.name = repo.data.name
-            member.resourceURI = repo.data.contentResourceURI
+            print "Append: '%s' to group content URI: '%s'" % (repo.data.id, self.data.contentResourceURI)
+
+            match = re.search(GROUP_CONTENT_URI_RE, str(self.content_uri()))
+            base_url = match.group(1)
+
+            repo_id = str(repo.data.id)
+
+            resource_uri = "%s%s/%s" % (base_url, NAMED_GROUP_PATH.format(key=self.id()), repo_id)
+
+            member.resourceURI = resource_uri
 
             if session.debug:
                 print "Added member: %s" % repo_key
@@ -136,6 +159,8 @@ def remove_member(self, session, repo_key):
         return self
 
     def render(self, pretty_print=True):
+        objectify.deannotate(self.xml, xsi_nil=True)
+        etree.cleanup_namespaces(self.xml)
         return etree.tostring(self.xml, pretty_print=pretty_print)
 
     def members(self):

nexup/staging.py

@@ -0,0 +1,40 @@
+from lxml import (objectify,etree)
+import repo
+
+STAGE_START_FORMAT='/service/local/staging/profiles/{profile_id}/start'
+STAGE_FINISH_FORMAT='/service/local/staging/profiles/{profile_id}/finish'
+
+def _get_staging_description(product, version, is_ga):
+	return "%s, ver %s (to %s)" % (product, version, "GA" if is_ga else "Early-Access")	
+
+def start_staging_repo(session, config, product, version, is_ga):
+	profile_id = config.get_profile_id( product, is_ga )
+
+	path = STAGE_START_FORMAT.format(profile_id=profile_id)
+	request_data = etree.Element('promoteRequest')
+	data = etree.SubElement( request_data, 'data')
+	etree.SubElement(data, 'description').text=_get_staging_description(product, version, is_ga)
+
+	xml = etree.tostring( request_data, xml_declaration=True, pretty_print=True, encoding='UTF-8')
+	(response,text) = session.post(path, xml)
+
+	# TODO: Error handling!
+
+	repo_id = etree.fromstring(text).xpath('/promoteRequest/data/stagedRepositoryId/text()')
+	return repo_id[0]
+
+def finish_staging_repo(session, config, repo_id, product, version, is_ga):
+	profile_id = config.get_profile_id( product, is_ga )
+
+	path = STAGE_FINISH_FORMAT.format(profile_id=profile_id)
+	request_data = etree.Element('promoteRequest')
+	data = etree.SubElement( request_data, 'data')
+	etree.SubElement(data, 'description').text=_get_staging_description(product, version, is_ga)
+	etree.SubElement(data, 'stagedRepositoryId').text=repo_id
+
+	xml = etree.tostring( request_data, xml_declaration=True, pretty_print=True, encoding='UTF-8')
+	(response,text) = session.post(path, xml)
+
+	# TODO: Error handling!
+	# FIXME: Handle verification failure!
+

test-input/public-group.xml

@@ -0,0 +1,33 @@
+<repo-group>
+  <data>
+    <contentResourceURI>http://localhost:8081/nexus/content/groups/public</contentResourceURI>
+    <id>public</id>
+    <name>Public Repositories</name>
+    <provider>maven2</provider>
+    <format>maven2</format>
+    <repoType>group</repoType>
+    <exposed>true</exposed>
+    <repositories>
+      <repo-group-member>
+        <id>releases</id>
+        <name>Releases</name>
+        <resourceURI>http://localhost:8081/nexus/service/local/repo_groups/public/releases</resourceURI>
+      </repo-group-member>
+      <repo-group-member>
+        <id>snapshots</id>
+        <name>Snapshots</name>
+        <resourceURI>http://localhost:8081/nexus/service/local/repo_groups/public/snapshots</resourceURI>
+      </repo-group-member>
+      <repo-group-member>
+        <id>thirdparty</id>
+        <name>3rd party</name>
+        <resourceURI>http://localhost:8081/nexus/service/local/repo_groups/public/thirdparty</resourceURI>
+      </repo-group-member>
+      <repo-group-member>
+        <id>central</id>
+        <name>Central</name>
+        <resourceURI>http://localhost:8081/nexus/service/local/repo_groups/public/central</resourceURI>
+      </repo-group-member>
+    </repositories>
+  </data>
+</repo-group>

tests/test_config.py

@@ -20,6 +20,46 @@ def test_minimal_from_default(self):
         nxconfig = config.load('test')
         self.assertEqual(nxconfig.url, url)
 
+    def test_get_profile_id_ga(self):
+        url='http://nowhere.com/nexus'
+        ga_profile = '0123456789'
+        data={
+            'test': {
+                config.URL: url,
+                config.PROFILE_MAP: {
+                    'eap': {
+                        config.GA_PROFILE: str(ga_profile),
+                        config.EA_PROFILE: '9876543210'
+                    }
+                }
+            }
+        }
+        rc = self.write_config(data)
+        nxconfig = config.load('test')
+        profile_id = nxconfig.get_profile_id('eap', is_ga=True)
+
+        self.assertEqual(profile_id, ga_profile)
+
+    def test_get_profile_id_ea(self):
+        url='http://nowhere.com/nexus'
+        ea_profile = '0123456789'
+        data={
+            'test': {
+                config.URL: url,
+                config.PROFILE_MAP: {
+                    'eap': {
+                        config.GA_PROFILE: '9876543210',
+                        config.EA_PROFILE: str(ea_profile)
+                    }
+                }
+            }
+        }
+        rc = self.write_config(data)
+        nxconfig = config.load('test')
+        profile_id = nxconfig.get_profile_id('eap', is_ga=False)
+
+        self.assertEqual(profile_id, ea_profile)
+
     def test_preemptive_auth(self):
         url='http://nowhere.com/nexus'
         data={