From 788953dab1b31aed8596bb747da722951dea6dd3 Mon Sep 17 00:00:00 2001 From: Jacob Kiefer Date: Tue, 14 Mar 2017 16:37:44 -0400 Subject: [PATCH] feat(build): Build spinnaker distribution to validate with citest. (#1473) --- dev/annotate_source.py | 6 ++-- dev/build_prevalidation.py | 70 ++++++++++++++++++++++++++++++++++++++ dev/build_release.py | 32 ++++------------- dev/generate_bom.py | 15 ++++++-- 4 files changed, 91 insertions(+), 32 deletions(-) create mode 100755 dev/build_prevalidation.py diff --git a/dev/annotate_source.py b/dev/annotate_source.py index 946edf360..7a085c7bd 100755 --- a/dev/annotate_source.py +++ b/dev/annotate_source.py @@ -118,7 +118,7 @@ def __init__(self, message): self.message = message -class Annotator(Refresher): +class Annotator(object): """Provides semantic version tagging for Spinnaker repositories. Each Spinnaker repository has tags that denote releases. These tags follow @@ -148,7 +148,6 @@ def __init__(self, options, path=None): self.__tags_to_delete = [] self.__filtered_tags = [] self.__current_version = None - super(Annotator, self).__init__(options) @property def build_number(self): @@ -359,7 +358,7 @@ def bump_semver(self, curr_version, commit_hashes, commit_msgs): @classmethod def init_argument_parser(cls, parser): """Initialize command-line arguments.""" - parser.add_argument('--build_number', default='', + parser.add_argument('--build_number', default=os.environ.get('BUILD_NUMBER'), help='The build number to append to the semantic version tag.') parser.add_argument('--initial_branch', default='master', help='Initial branch to create the stable release branch from.') @@ -371,7 +370,6 @@ def init_argument_parser(cls, parser): help='Name of the stable release branch to create.') parser.add_argument('--force_rebuild', default=False, action='store_true', help='Force a rebuild even if there is a git tag at HEAD.') - super(Annotator, cls).init_argument_parser(parser) @classmethod def main(cls): diff --git a/dev/build_prevalidation.py b/dev/build_prevalidation.py new file mode 100755 index 000000000..ff57a09f3 --- /dev/null +++ b/dev/build_prevalidation.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys + +from annotate_source import Annotator +from build_release import Builder +from generate_bom import BomGenerator +from refresh_source import Refresher + + +def __annotate_component(annotator, component): + """Annotate the component's source but don't include it in the BOM. + """ + annotator.path = component + annotator.parse_git_tree() + annotator.tag_head() + annotator.delete_unwanted_tags() + +def init_argument_parser(parser): + # Don't need to init args for Annotator since BomGenerator extends it. + BomGenerator.init_argument_parser(parser) + Builder.init_argument_parser(parser) + +def main(): + """Build a Spinnaker release to be validated by Citest. + """ + parser = argparse.ArgumentParser() + init_argument_parser(parser) + options = parser.parse_args() + + annotator = Annotator(options) + __annotate_component(annotator, 'spinnaker') + __annotate_component(annotator, 'halyard') + __annotate_component(annotator, 'spinnaker-monitoring') + + bom_generator = BomGenerator(options) + bom_generator.determine_and_tag_versions() + if options.container_builder == 'gcb': + bom_generator.write_container_builder_gcr_config() + elif options.container_builder == 'docker': + bom_generator.write_docker_version_files() + else: + raise NotImplementedError('container_builder="{0}"' + .format(options.container_builder)) + Builder.do_build(options, options.build_number, options.container_builder) + # Load version information into memory and write BOM to disk. Don't publish yet. + bom_generator.write_bom() + bom_generator.publish_microservice_configs() + bom_generator.publish_boms() + bom_generator.generate_changelog() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dev/build_release.py b/dev/build_release.py index 9f0d788db..ab502e647 100755 --- a/dev/build_release.py +++ b/dev/build_release.py @@ -117,17 +117,16 @@ def determine_package_version(gradle_root): class Builder(object): """Knows how to coordinate a Spinnaker release.""" - def __init__(self, options): + def __init__(self, options, build_number=None, container_builder=None): self.__package_list = [] self.__build_failures = [] self.__background_processes = [] os.environ['NODE_ENV'] = os.environ.get('NODE_ENV', 'dev') - self.__build_number = options.build_number + self.__build_number = build_number or os.environ.get('BUILD_NUMBER') self.__gcb_service_account = options.gcb_service_account self.__options = options - if (self.__options.container_builder - and self.__options.container_builder not in ['gcb', 'docker']): + if (container_builder and container_builder not in ['gcb', 'docker']): raise ValueError('Invalid container_builder. Must be empty, "gcb" or "docker"') self.refresher = refresh_source.Refresher(options) @@ -203,7 +202,7 @@ def start_deb_build(self, name): # Currently spinnaker is in a separate location gradle_root = self.determine_gradle_root(name) - print 'Building and publishing {name}...'.format(name=name) + print 'Building and publishing Debian for {name}...'.format(name=name) # Note: 'candidate' is just the gradle task name. It doesn't indicate # 'release candidate' status for the artifacts created through this build. return BackgroundProcess.spawn( @@ -393,7 +392,7 @@ def start_copy_debian_target(self, name): version = determine_package_version(gradle_root) if version is None: return [] - + for root in determine_modules_with_debians(gradle_root): deb_dir = '{root}/build/distributions'.format(root=root) @@ -531,10 +530,6 @@ def init_argument_parser(cls, parser): '--jar_repo', default='', help='Publish produced jars to this repo.\n' 'This requires BINTRAY_USER and BINTRAY_KEY are set.') - parser.add_argument( - '--build_number', default=os.environ.get('BUILD_NUMBER', ''), - help='CI system build number. Ideally should be a unique integer' - 'for each build.') parser.add_argument( '--wipe_package_on_409', default=False, action='store_true', @@ -551,11 +546,6 @@ def init_argument_parser(cls, parser): parser.add_argument( '--nonebula', dest='nebula', action='store_false', help='Explicitly "buildDeb" then curl upload them to bintray.') - - parser.add_argument( - '--container_builder', default=None, - help='If specified then build container images using the specified' - ' builder. Supported builders are {"gcb", "docker"}.') parser.add_argument( '--gcb_service_account', default='', help='Google service account to invoke the gcp container builder with.') @@ -569,16 +559,12 @@ def __verify_bintray(self): @classmethod - def main(cls): - parser = argparse.ArgumentParser() - cls.init_argument_parser(parser) - options = parser.parse_args() - + def do_build(cls, options, build_number, container_builder): if options.build and not (options.bintray_repo): sys.stderr.write('ERROR: Missing a --bintray_repo') return -1 - builder = cls(options) + builder = cls(options, build_number=build_number, container_builder=container_builder) if options.pull_origin: builder.refresher.pull_all_from_origin() @@ -607,7 +593,3 @@ def main(cls): print '\nFINISHED writing release to {rep}'.format( rep=options.bintray_repo) - - -if __name__ == '__main__': - sys.exit(Builder.main()) diff --git a/dev/generate_bom.py b/dev/generate_bom.py index 5e4d8e467..0c331bfdb 100755 --- a/dev/generate_bom.py +++ b/dev/generate_bom.py @@ -70,6 +70,7 @@ def __init__(self, options): self.__changelog_start_hashes = {} # Hashes to start from when generating changelogs. self.__toplevel_version = '' self.__changelog_output = options.changelog_output + self.__alias = options.bom_alias super(BomGenerator, self).__init__(options) @classmethod @@ -83,6 +84,8 @@ def init_argument_parser(cls, parser): help="Docker registry to push the container images to.") parser.add_argument('--changelog_output', default='', help="Output file to write the changelog to.") + parser.add_argument('--bom_alias', default='', + help="Alias to rename the 'real' BOM as. This also sets the Spinnaker version as the alias.") super(BomGenerator, cls).init_argument_parser(parser) def write_container_builder_gcr_config(self): @@ -202,10 +205,16 @@ def write_bom(self): output_yaml[VERSION] = toplevel_with_build self.__bom_file = '{0}.yml'.format(toplevel_with_build) self.write_bom_file(self.__bom_file, output_yaml) + if self.__alias: + output_yaml[VERSION] = self.__alias + self.write_bom_file(self.__alias + '.yml', output_yaml) + + def publish_boms(self): + """Pushes the generated BOMs to a public GCS bucket for Halyard to use. + """ self.publish_bom(self.__bom_file) - output_yaml[VERSION] = 'nightly' - self.write_bom_file('nightly.yml', output_yaml) # Publish a 'nightly' BOM for folks wanting to run bleeding-edge Spinnaker. - self.publish_bom('nightly.yml') + if self.__alias: + self.publish_bom(self.__alias + '.yml') def write_bom_file(self, filename, output_yaml): """Helper function to write the calculated BOM to files.