From 66d0952f09efa2228aa45e6fe669bb3504415467 Mon Sep 17 00:00:00 2001 From: Jacob Kiefer Date: Thu, 23 Feb 2017 13:20:33 -0500 Subject: [PATCH] feat(build_release): Use gradle to publish builds instead of manual copy. (#1430) --- dev/build_release.py | 154 ++++++++++++++++++++---------------------- dev/refresh_source.py | 8 ++- 2 files changed, 82 insertions(+), 80 deletions(-) mode change 100644 => 100755 dev/build_release.py diff --git a/dev/build_release.py b/dev/build_release.py old mode 100644 new mode 100755 index b66b23314..08ab74731 --- a/dev/build_release.py +++ b/dev/build_release.py @@ -185,6 +185,7 @@ def __init__(self, options): self.__background_processes = [] os.environ['NODE_ENV'] = os.environ.get('NODE_ENV', 'dev') + self.__build_number = options.build_number self.__options = options self.refresher = refresh_source.Refresher(options) if options.bintray_repo: @@ -211,28 +212,69 @@ def determine_gradle_root(self, name): gradle_root = name if name != 'spinnaker' else self.__project_dir return gradle_root - def start_build_target(self, name, target): - """Start a subprocess to build the designated target. + def start_subsystem_build(self, name): + """Start a subprocess to build and publish the designated component. - Args: - name [string]: The name of the subsystem repository. - target [string]: The gradle build target. + This function runs a gradle 'candidate' task using the last git tag as the + package version and the Bintray configuration passed through arguments. The + 'candidate' task release builds the source, packages the debian and jar + files, and publishes those to the respective Bintray '$org/$repository'. - Returns: - BackgroundProcess - """ - extra_args = [] - if name == 'deck' and not 'CHROME_BIN' in os.environ: - extra_args.append('-PskipTests') - - # Currently spinnaker is in a separate location - gradle_root = self.determine_gradle_root(name) - print 'Building {name}...'.format(name=name) - return BackgroundProcess.spawn( - 'Building {name}'.format(name=name), - 'cd "{gradle_root}"; ./gradlew {target} {extra}'.format( - gradle_root=gradle_root, target=target, - extra=' '.join(extra_args))) + The naming of the gradle task is a bit unfortunate because of the + terminology used in the Spinnaker product release process. The artifacts + produced by this script are not 'release candidate' artifacts, they are + pre-validation artifacts. Maybe we can modify the task name at some point. + + The gradle 'candidate' task throws a 409 if the package we are trying to + publish already exists. We'll publish unique package versions using build + numbers. These will be transparent to end users since the only meaningful + version is the Spinnaker product version. + + We will use -Prelease.useLastTag=true and ensure the last git tag is the + version we want to use. This tag has to be of the form 'X.Y.Z-$build' or + 'vX.Y.Z-$build for gradle to use the tag as the version. This script will + assume that the source has been properly tagged to use the latest tag as the + package version for each component. + + Args: + name [string]: Name of the subsystem repository. + + Returns: + BackgroundProcess + """ + jarRepo = self.__options.jar_repo + parts = self.__options.bintray_repo.split('/') + if len(parts) != 2: + raise ValueError( + 'Expected --bintray_repo to be in the form /') + org, packageRepo = parts[0], parts[1] + bintray_key = os.environ['BINTRAY_KEY'] + bintray_user = os.environ['BINTRAY_USER'] + + extra_args = [ + '--stacktrace', + '-Prelease.useLastTag=true', + '-PbintrayPackageBuildNumber={number}'.format(number=self.__build_number), + '-PbintrayOrg="{org}"'.format(org=org), + '-PbintrayPackageRepo="{repo}"'.format(repo=packageRepo), + '-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jarRepo), + '-PbintrayKey="{key}"'.format(key=bintray_key), + '-PbintrayUser="{user}"'.format(user=bintray_user) + ] + if name == 'deck' and not 'CHROME_BIN' in os.environ: + extra_args.append('-PskipTests') + + # Currently spinnaker is in a separate location + gradle_root = self.determine_gradle_root(name) + print 'Building and publishing {name}...'.format(name=name) + # Note: 'candidate' is just the gradle task name. It doesn't indicate + # 'release candidate' status for the artifacts created through this build. + return BackgroundProcess.spawn( + 'Building and publishing {name}...'.format(name=name), + 'cd "{gradle_root}"; ./gradlew {extra} candidate'.format( + gradle_root=gradle_root, extra=' '.join(extra_args) + ) + ) def publish_to_bintray(self, source, package, version, path, debian_tags=''): bintray_key = os.environ['BINTRAY_KEY'] @@ -240,7 +282,7 @@ def publish_to_bintray(self, source, package, version, path, debian_tags=''): parts = self.__options.bintray_repo.split('/') if len(parts) != 2: raise ValueError( - 'Expected --bintray_repo to be in the form //') subject, repo = parts[0], parts[1] deb_filename = os.path.basename(path) @@ -302,7 +344,7 @@ def publish_to_bintray(self, source, package, version, path, debian_tags=''): print 'Retrying {url}'.format(url=url) result = urllib2.urlopen(put_request, data) print 'SUCCESS' - + elif put_error.code != 400: raise @@ -392,53 +434,9 @@ def start_copy_file(self, source, target): shutil.copy(source, target) return NO_PROCESS - def start_copy_debian_target(self, name): - """Copies the debian package for the specified subsystem. - - Args: - name [string]: The name of the subsystem repository. - """ - gradle_root = self.determine_gradle_root(name) - if os.path.exists(os.path.join(name, '{name}-web'.format(name=name))): - submodule = '{name}-web'.format(name=name) - elif os.path.exists(os.path.join(name, '{name}-core'.format(name=name))): - submodule = '{name}-core'.format(name=name) - else: - submodule = '.' - - version = determine_package_version(gradle_root, submodule) - build_dir = '{submodule}/build/distributions'.format(submodule=submodule) - - deb_dir = os.path.join(gradle_root, build_dir) - non_spinnaker_name = '{name}_{version}_all.deb'.format( - name=name, version=version) - - if os.path.exists(os.path.join(deb_dir, - 'spinnaker-' + non_spinnaker_name)): - deb_file = 'spinnaker-' + non_spinnaker_name - else: - deb_file = non_spinnaker_name - - if not os.path.exists(os.path.join(deb_dir, deb_file)): - error = ('.deb for name={name} version={version} is not in {dir}\n' - .format(name=name, version=version, dir=deb_dir)) - raise AssertionError(error) - - from_path = os.path.join(gradle_root, build_dir, deb_file) - print 'Adding {path}'.format(path=from_path) - self.__package_list.append(deb_file) - if self.__options.bintray_repo: - self.publish_file(from_path, name, version) - - if self.__release_dir: - to_path = os.path.join(self.__release_dir, deb_file) - return self.start_copy_file(from_path, to_path) - else: - return NO_PROCESS - def __do_build(self, subsys): try: - self.start_build_target(subsys, 'buildDeb').check_wait() + self.start_subsystem_build(subsys).check_wait() except Exception as ex: self.__build_failures.append(subsys) @@ -456,15 +454,6 @@ def build_packages(self): raise RuntimeError('Builds failed for {0!r}'.format( self.__build_failures)) - # Copy subsystem packages. - processes = [] - for subsys in SUBSYSTEM_LIST: - processes.append(self.start_copy_debian_target(subsys)) - - print 'Waiting for package copying to finish....' - for p in processes: - p.check_wait() - @staticmethod def __zip_dir(zip_file, source_path, arcname=''): """Zip the contents of a directory. @@ -529,6 +518,14 @@ def init_argument_parser(cls, parser): '--bintray_repo', default='', help='Publish to this bintray repo.\n' 'This requires BINTRAY_USER and BINTRAY_KEY are set.') + parser.add_argument( + '--jar_repo', default='', + help='Publish produced jars to this repo.\n' + 'This requires BINTRAY_USER and BINTRAY_KEY are set.') + parser.add_argument( + '--build_number', default=os.environ.get('BUILD_NUMBER', ''), + help='CI system build number. Ideally should be a unique integer' + 'for each build.') parser.add_argument( '--wipe_package_on_409', default=False, action='store_true', @@ -553,9 +550,8 @@ def main(cls): cls.init_argument_parser(parser) options = parser.parse_args() - if not (options.release_path or options.bintray_repo): - sys.stderr.write( - 'ERROR: Missing either a --release_path or --bintray_repo') + if not (options.bintray_repo): + sys.stderr.write('ERROR: Missing a --bintray_repo') return -1 builder = cls(options) diff --git a/dev/refresh_source.py b/dev/refresh_source.py index 8803371e1..3304beb72 100755 --- a/dev/refresh_source.py +++ b/dev/refresh_source.py @@ -538,6 +538,9 @@ def init_argument_parser(cls, parser): help='Pull from this github user\'s repositories.' ' If the user is "default" then use the' ' authoritative (upstream) repository.') + parser.add_argument('--update_run_scripts', default=False, + action='store_true', + help='Update the run script for each component.') @classmethod def main(cls): @@ -579,7 +582,10 @@ def main(cls): if refresher.pull_branch: nothing = False refresher.pull_all_from_origin() - refresher.update_spinnaker_run_scripts() + + if options.update_run_scripts: + print 'Updating Spinnaker component run scripts' + refresher.update_spinnaker_run_scripts() if nothing: sys.stderr.write('No pull/push options were specified.\n')