feat(build): Build spinnaker distribution to validate with citest. (#…

…1473)

dev/annotate_source.py

@@ -118,7 +118,7 @@ def __init__(self, message):
     self.message = message
 
 
-class Annotator(Refresher):
+class Annotator(object):
   """Provides semantic version tagging for Spinnaker repositories.
 
   Each Spinnaker repository has tags that denote releases. These tags follow
@@ -148,7 +148,6 @@ def __init__(self, options, path=None):
     self.__tags_to_delete = []
     self.__filtered_tags = []
     self.__current_version = None
-    super(Annotator, self).__init__(options)
 
   @property
   def build_number(self):
@@ -359,7 +358,7 @@ def bump_semver(self, curr_version, commit_hashes, commit_msgs):
   @classmethod
   def init_argument_parser(cls, parser):
     """Initialize command-line arguments."""
-    parser.add_argument('--build_number', default='',
+    parser.add_argument('--build_number', default=os.environ.get('BUILD_NUMBER'),
                         help='The build number to append to the semantic version tag.')
     parser.add_argument('--initial_branch', default='master',
                         help='Initial branch to create the stable release branch from.')
@@ -371,7 +370,6 @@ def init_argument_parser(cls, parser):
                         help='Name of the stable release branch to create.')
     parser.add_argument('--force_rebuild', default=False, action='store_true',
                         help='Force a rebuild even if there is a git tag at HEAD.')
-    super(Annotator, cls).init_argument_parser(parser)
 
   @classmethod
   def main(cls):

dev/build_prevalidation.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+#
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+
+from annotate_source import Annotator
+from build_release import Builder
+from generate_bom import BomGenerator
+from refresh_source import Refresher
+
+
+def __annotate_component(annotator, component):
+  """Annotate the component's source but don't include it in the BOM.
+  """
+  annotator.path = component
+  annotator.parse_git_tree()
+  annotator.tag_head()
+  annotator.delete_unwanted_tags()
+
+def init_argument_parser(parser):
+  # Don't need to init args for Annotator since BomGenerator extends it.
+  BomGenerator.init_argument_parser(parser)
+  Builder.init_argument_parser(parser)
+
+def main():
+  """Build a Spinnaker release to be validated by Citest.
+  """
+  parser = argparse.ArgumentParser()
+  init_argument_parser(parser)
+  options = parser.parse_args()
+
+  annotator = Annotator(options)
+  __annotate_component(annotator, 'spinnaker')
+  __annotate_component(annotator, 'halyard')
+  __annotate_component(annotator, 'spinnaker-monitoring')
+
+  bom_generator = BomGenerator(options)
+  bom_generator.determine_and_tag_versions()
+  if options.container_builder == 'gcb':
+    bom_generator.write_container_builder_gcr_config()
+  elif options.container_builder == 'docker':
+    bom_generator.write_docker_version_files()
+  else:
+    raise NotImplementedError('container_builder="{0}"'
+                              .format(options.container_builder))
+  Builder.do_build(options, options.build_number, options.container_builder)
+  # Load version information into memory and write BOM to disk. Don't publish yet.
+  bom_generator.write_bom()
+  bom_generator.publish_microservice_configs()
+  bom_generator.publish_boms()
+  bom_generator.generate_changelog()
+
+
+if __name__ == '__main__':
+  sys.exit(main())

dev/build_release.py

@@ -117,17 +117,16 @@ def determine_package_version(gradle_root):
 class Builder(object):
   """Knows how to coordinate a Spinnaker release."""
 
-  def __init__(self, options):
+  def __init__(self, options, build_number=None, container_builder=None):
       self.__package_list = []
       self.__build_failures = []
       self.__background_processes = []
 
       os.environ['NODE_ENV'] = os.environ.get('NODE_ENV', 'dev')
-      self.__build_number = options.build_number
+      self.__build_number = build_number or os.environ.get('BUILD_NUMBER')
       self.__gcb_service_account = options.gcb_service_account
       self.__options = options
-      if (self.__options.container_builder
-          and self.__options.container_builder not in ['gcb', 'docker']):
+      if (container_builder and container_builder not in ['gcb', 'docker']):
         raise ValueError('Invalid container_builder. Must be empty, "gcb" or "docker"')
 
       self.refresher = refresh_source.Refresher(options)
@@ -203,7 +202,7 @@ def start_deb_build(self, name):
 
     # Currently spinnaker is in a separate location
     gradle_root = self.determine_gradle_root(name)
-    print 'Building and publishing {name}...'.format(name=name)
+    print 'Building and publishing Debian for {name}...'.format(name=name)
     # Note: 'candidate' is just the gradle task name. It doesn't indicate
     # 'release candidate' status for the artifacts created through this build.
     return BackgroundProcess.spawn(
@@ -393,7 +392,7 @@ def start_copy_debian_target(self, name):
       version = determine_package_version(gradle_root)
       if version is None:
         return []
-        
+
       for root in determine_modules_with_debians(gradle_root):
         deb_dir = '{root}/build/distributions'.format(root=root)
 
@@ -531,10 +530,6 @@ def init_argument_parser(cls, parser):
           '--jar_repo', default='',
           help='Publish produced jars to this repo.\n'
                'This requires BINTRAY_USER and BINTRAY_KEY are set.')
-      parser.add_argument(
-          '--build_number', default=os.environ.get('BUILD_NUMBER', ''),
-          help='CI system build number. Ideally should be a unique integer'
-               'for each build.')
 
       parser.add_argument(
           '--wipe_package_on_409', default=False, action='store_true',
@@ -551,11 +546,6 @@ def init_argument_parser(cls, parser):
       parser.add_argument(
           '--nonebula', dest='nebula', action='store_false',
           help='Explicitly "buildDeb" then curl upload them to bintray.')
-
-      parser.add_argument(
-          '--container_builder', default=None,
-          help='If specified then build container images using the specified'
-               ' builder. Supported builders are {"gcb", "docker"}.')
       parser.add_argument(
           '--gcb_service_account', default='',
           help='Google service account to invoke the gcp container builder with.')
@@ -569,16 +559,12 @@ def __verify_bintray(self):
 
 
   @classmethod
-  def main(cls):
-    parser = argparse.ArgumentParser()
-    cls.init_argument_parser(parser)
-    options = parser.parse_args()
-
+  def do_build(cls, options, build_number, container_builder):
     if options.build and not (options.bintray_repo):
       sys.stderr.write('ERROR: Missing a --bintray_repo')
       return -1
 
-    builder = cls(options)
+    builder = cls(options, build_number=build_number, container_builder=container_builder)
     if options.pull_origin:
         builder.refresher.pull_all_from_origin()
 
@@ -607,7 +593,3 @@ def main(cls):
 
       print '\nFINISHED writing release to {rep}'.format(
         rep=options.bintray_repo)
-
-
-if __name__ == '__main__':
-  sys.exit(Builder.main())

dev/generate_bom.py

@@ -70,6 +70,7 @@ def __init__(self, options):
     self.__changelog_start_hashes = {} # Hashes to start from when generating changelogs.
     self.__toplevel_version = ''
     self.__changelog_output = options.changelog_output
+    self.__alias = options.bom_alias
     super(BomGenerator, self).__init__(options)
 
   @classmethod
@@ -83,6 +84,8 @@ def init_argument_parser(cls, parser):
                         help="Docker registry to push the container images to.")
     parser.add_argument('--changelog_output', default='',
                         help="Output file to write the changelog to.")
+    parser.add_argument('--bom_alias', default='',
+                        help="Alias to rename the 'real' BOM as. This also sets the Spinnaker version as the alias.")
     super(BomGenerator, cls).init_argument_parser(parser)
 
   def write_container_builder_gcr_config(self):
@@ -202,10 +205,16 @@ def write_bom(self):
     output_yaml[VERSION] = toplevel_with_build
     self.__bom_file = '{0}.yml'.format(toplevel_with_build)
     self.write_bom_file(self.__bom_file, output_yaml)
+    if self.__alias:
+      output_yaml[VERSION] = self.__alias
+      self.write_bom_file(self.__alias + '.yml', output_yaml)
+
+  def publish_boms(self):
+    """Pushes the generated BOMs to a public GCS bucket for Halyard to use.
+    """
     self.publish_bom(self.__bom_file)
-    output_yaml[VERSION] = 'nightly'
-    self.write_bom_file('nightly.yml', output_yaml) # Publish a 'nightly' BOM for folks wanting to run bleeding-edge Spinnaker.
-    self.publish_bom('nightly.yml')
+    if self.__alias:
+      self.publish_bom(self.__alias + '.yml')
 
   def write_bom_file(self, filename, output_yaml):
     """Helper function to write the calculated BOM to files.