feat(monitoring): Optionally install spinnaker-monitoring from Instal…

…lSpinnaker

This is disabled by default for now but can be enabled with --monitoring.
Note that monitoring installs the monitoring components but still does not
activate them because they need further deployment customization.

Other internal changes:
 - Deprecated the old --google_monitoring flag.
 - Install --monitoring when baking GCE images and added hook to trigger on
first_google_boot.
  - Updated the dev scripts in order to build spinnaker-monitoring.
    In doing so I added back the old curl upload as an option because the
    nebula upload mechanism requires a clean git client, which isnt developer
    friendly. The spinnaker-monitoring package is currently considered optional.

InstallSpinnaker.sh

@@ -29,6 +29,7 @@ QUIET=false
 # Install dependencies but not spinnaker itself.
 DEPENDENCIES_ONLY=false
 
+MONITORING_CHOICES=(datadog, prometheus, stackdriver, undef)
 
 # We can only currently support limited releases
 # First guess what sort of operating system
@@ -75,7 +76,7 @@ function print_usage() {
 usage: $0 [--cloud_provider <aws|google|azure|none>]
     [--aws_region <region>] [--google_region <region>] [--azure_region <region>]
     [--quiet] [--dependencies_only]
-    [--google_cloud_logging] [--google_cloud_monitoring]
+    [--google_cloud_logging] [--monitoring]
     [--repository <debian repository url>]
     [--local-install] [--home_dir <path>]
 
@@ -110,10 +111,13 @@ usage: $0 [--cloud_provider <aws|google|azure|none>]
                                 Platform, but you may require additional
                                 authorization. See https://cloud.google.com/logging/docs/agent/authorization#install_private-key_authorization
 
-    --google_cloud_monitoring   Install Google Cloud Monitoring support. This
-                                is independent of installing on Google Cloud
-                                Platform, but you may require additional
-                                authorization. See https://cloud.google.com/monitoring/api/authentication
+    --monitoring (datadog|prometheus|stackdriver|undef)
+                                Install the Spinnaker Monitoring Daemon. This
+                                installs support to /opt/spinnaker-monitoring
+                                and --client_only support for the given system.
+                                This may not be fully configured or require
+                                additional values per system. For more info, see
+                                http://www.spinnaker.io/docs/monitoring-a-spinnaker-deployment
 
     --local-install             For Spinnaker and Java packages, download
                                 packages and install using dpkg instead of
@@ -217,8 +221,14 @@ function process_args() {
       --google_cloud_logging)
         GOOGLE_CLOUD_LOGGING="true"
         ;;
-      --google_cloud_monitoring)
-        GOOGLE_CLOUD_MONITORING="true"
+      --monitoring)
+        MONITORING="$1"
+        shift
+        if [[ ! ${MONITORING_CHOICES[*]} =~ "$MONITORING" ]]
+        then
+            echo "Unknown monitoring choice '$MONITORING'."
+            exit -1
+        fi
         ;;
       --dependencies_only)
         CLOUD_PROVIDER="none"
@@ -450,8 +460,7 @@ function install_platform_dependencies() {
 
   if [[ -z "$google_scopes" ]]; then
     # Not on GCP
-    if [[ "$GOOGLE_CLOUD_LOGGING" == "true" ]] \
-         || [[ "$GOOGLE_CLOUD_MONITORING" == "true" ]]; then
+    if [[ "$GOOGLE_CLOUD_LOGGING" == "true" ]]; then
       if [[ ! -f /etc/google/auth/application_default_credentials.json ]];
       then
         echo "You may need to add Google Project Credentials."
@@ -466,19 +475,6 @@ function install_platform_dependencies() {
     # The add_google_cloud_logging script checks the scope and warns.
     curl -s -L https://raw.githubusercontent.com/spinnaker/spinnaker/master/google/google_cloud_logging/add_google_cloud_logging.sh | sudo bash
   fi
-
-  if [[ "$GOOGLE_CLOUD_MONITORING" == "true" ]]; then
-    # This can be installed on any platform, so dont scope to google.
-    # However, if on google, then certain scopes are required.
-    curl -s https://repo.stackdriver.com/stack-install.sh | sudo bash
-    if [[ ! -z "$google_scopes" ]] && [[ $scopes != *"monitoring.write"* ]]; then
-      # This is not necessarily bad because we might be using this instance
-      # to create an image (e.g. packer). Only the runtime instances need
-      # this scope.
-      echo "Missing scope 'https://www.googleapis.com/auth/monitoring.write'"
-      echo "Google Cloud Monitoring will not be able to send data upstream."
-    fi
-  fi
 }
 
 function install_dependencies() {
@@ -650,6 +646,16 @@ function install_spinnaker() {
     fi
   fi
 
+  if [[ "$MONITORING" != "" ]] && [[ "$MONITORING" != "none" ]]; then
+     sudo apt-get install -y --force-yes spinnaker-monitoring-daemon
+     sudo apt-get install -y --force-yes spinnaker-monitoring-third-party
+
+    if [[ "$MONITORING" != "undef" ]]; then
+      sudo /opt/spinnaker-monitoring/third_party/$MONITORING/install.sh \
+          --client_only
+      sudo service spinnaker-monitoring restart || true
+    fi
+  fi
 }
 
 set_defaults_from_environ

dev/build_google_image.packer

@@ -10,7 +10,8 @@
     "base_srcdir": null,
     "target_image": "{{env `USER`}}-spinnaker-{{timestamp}}",
     "google_cloud_logging": "true",
-    "google_cloud_monitoring": "true",
+    "google_cloud_monitoring": "deprecated",
+    "monitoring": "undef",
     "enable_cassandra": "false"
  },
 
@@ -35,7 +36,7 @@
     "inline": [
         "flags=\"--cloud_provider google --google_region us-central1 --quiet\"",
         "if [ '{{ user `google_cloud_logging` }}' = 'true' ]; then flags=\"$flags --google_cloud_logging\"; fi",
-        "if [ '{{ user `google_cloud_monitoring` }}' = 'true' ]; then flags=\"$flags --google_cloud_monitoring\"; fi",
+        "if [ '{{ user `monitoring` }}' != '' ]; then flags=\"$flags --monitoring {{ user `monitoring` }}\"; fi",
         "if [ '{{ user `enable_cassandra` }}' = 'true' ]; then flags=\"$flags --install_cassandra\"; fi",
         "chmod +x /tmp/install_spinnaker.sh",
         "/tmp/install_spinnaker.sh --repository {{user `debian_repo`}} $flags",

dev/build_release.py

@@ -45,6 +45,7 @@
 import argparse
 import base64
 import collections
+import glob
 import os
 import multiprocessing
 import multiprocessing.pool
@@ -64,6 +65,7 @@
 
 SUBSYSTEM_LIST = ['clouddriver', 'orca', 'front50',
                   'echo', 'rosco', 'gate', 'igor', 'fiat', 'deck', 'spinnaker']
+ADDITIONAL_SUBSYSTEMS = ['spinnaker-monitoring']
 
 def ensure_gcs_bucket(name, project=''):
   """Ensure that the desired GCS bucket exists, creating it if needed.
@@ -168,9 +170,17 @@ def check_wait(self):
 def determine_project_root():
   return os.path.abspath(os.path.dirname(__file__) + '/..')
 
-def determine_package_version(gradle_root, submodule):
-  with open(os.path.join(gradle_root, submodule,
-                         'build/debian/control')) as f:
+def determine_modules_with_debians(gradle_root):
+  files = glob.glob(os.path.join(gradle_root, '*', 'build', 'debian', 'control'))
+  dirs = [os.path.dirname(os.path.dirname(os.path.dirname(file))) for file in files]
+  if os.path.exists(os.path.join(gradle_root, 'build', 'debian', 'control')):
+    dirs.append(gradle_root)
+  return dirs
+
+def determine_package_version(gradle_root):
+  root = determine_modules_with_debians(gradle_root)
+
+  with open(os.path.join(root[0], 'build', 'debian', 'control')) as f:
      content = f.read()
   match = re.search('(?m)^Version: (.*)', content)
   return match.group(1)
@@ -251,16 +261,23 @@ def start_subsystem_build(self, name):
     bintray_key = os.environ['BINTRAY_KEY']
     bintray_user = os.environ['BINTRAY_USER']
 
-    extra_args = [
-      '--stacktrace',
-      '-Prelease.useLastTag=true',
-      '-PbintrayPackageBuildNumber={number}'.format(number=self.__build_number),
-      '-PbintrayOrg="{org}"'.format(org=org),
-      '-PbintrayPackageRepo="{repo}"'.format(repo=packageRepo),
-      '-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jarRepo),
-      '-PbintrayKey="{key}"'.format(key=bintray_key),
-      '-PbintrayUser="{user}"'.format(user=bintray_user)
-    ]
+    if self.__options.nebula:
+      target = 'candidate'
+      extra_args = [
+          '--stacktrace',
+          '-Prelease.useLastTag=true',
+          '-PbintrayPackageBuildNumber={number}'.format(
+              number=self.__build_number),
+          '-PbintrayOrg="{org}"'.format(org=org),
+          '-PbintrayPackageRepo="{repo}"'.format(repo=packageRepo),
+          '-PbintrayJarRepo="{jarRepo}"'.format(jarRepo=jarRepo),
+          '-PbintrayKey="{key}"'.format(key=bintray_key),
+          '-PbintrayUser="{user}"'.format(user=bintray_user)
+        ]
+    else:
+      target = 'buildDeb'
+      extra_args = []
+
     if name == 'deck' and not 'CHROME_BIN' in os.environ:
       extra_args.append('-PskipTests')
 
@@ -271,9 +288,8 @@ def start_subsystem_build(self, name):
     # 'release candidate' status for the artifacts created through this build.
     return BackgroundProcess.spawn(
       'Building and publishing {name}...'.format(name=name),
-      'cd "{gradle_root}"; ./gradlew {extra} candidate'.format(
-        gradle_root=gradle_root, extra=' '.join(extra_args)
-      )
+      'cd "{gradle_root}"; ./gradlew {extra} {target}'.format(
+          gradle_root=gradle_root, extra=' '.join(extra_args), target=target)
     )
 
   def publish_to_bintray(self, source, package, version, path, debian_tags=''):
@@ -356,6 +372,11 @@ def publish_to_bintray(self, source, package, version, path, debian_tags=''):
                   package=package, pkg_url=pkg_url)
 
               # All the packages are from spinnaker so we'll hardcode it.
+              # Note spinnaker-monitoring is a github repo with two packages.
+              # Neither is "spinnaker-monitoring"; that's only the github repo.
+              gitname = (package.replace('spinnaker-', '')
+                         if not package.startswith('spinnaker-monitoring')
+                         else 'spinnaker-monitoring')
               pkg_data = """{{
                 "name": "{package}",
                 "licenses": ["Apache-2.0"],
@@ -364,8 +385,7 @@ def publish_to_bintray(self, source, package, version, path, debian_tags=''):
                 "github_repo": "spinnaker/{gitname}",
                 "public_download_numbers": false,
                 "public_stats": false
-              }}'""".format(package=package,
-                            gitname=package.replace('spinnaker-', ''))
+              }}'""".format(package=package, gitname=gitname)
 
               pkg_request = urllib2.Request(pkg_url)
               pkg_request.add_header('Authorization', 'Basic ' + encoded_auth)
@@ -385,7 +405,7 @@ def publish_to_bintray(self, source, package, version, path, debian_tags=''):
 
   def publish_install_script(self, source):
     gradle_root = self.determine_gradle_root('spinnaker')
-    version = determine_package_version(gradle_root, '.')
+    version = determine_package_version(gradle_root)
 
     self.publish_to_bintray(source, package='spinnaker', version=version,
                             path='InstallSpinnaker.sh')
@@ -434,6 +454,51 @@ def start_copy_file(self, source, target):
         shutil.copy(source, target)
         return NO_PROCESS
 
+  def start_copy_debian_target(self, name):
+      """Copies the debian package for the specified subsystem.
+
+      Args:
+        name [string]: The name of the subsystem repository.
+      """
+      pids = []
+      gradle_root = self.determine_gradle_root(name)
+      version = determine_package_version(gradle_root)
+      for root in determine_modules_with_debians(gradle_root):
+        deb_dir = '{root}/build/distributions'.format(root=root)
+
+        non_spinnaker_name = '{name}_{version}_all.deb'.format(
+              name=name, version=version)
+
+        if os.path.exists(os.path.join(deb_dir,
+                                       'spinnaker-' + non_spinnaker_name)):
+         deb_file = 'spinnaker-' + non_spinnaker_name
+        elif os.path.exists(os.path.join(deb_dir, non_spinnaker_name)):
+          deb_file = non_spinnaker_name
+        else:
+          module_name = os.path.basename(
+            os.path.dirname(os.path.dirname(deb_dir)))
+          deb_file = '{module_name}_{version}_all.deb'.format(
+            module_name=module_name, version=version)
+
+        if not os.path.exists(os.path.join(deb_dir, deb_file)):
+          error = ('.deb for name={name} version={version} is not in {dir}\n'
+                   .format(name=name, version=version, dir=deb_dir))
+          raise AssertionError(error)
+
+        from_path = os.path.join(deb_dir, deb_file)
+        print 'Adding {path}'.format(path=from_path)
+        self.__package_list.append(from_path)
+        basename = os.path.basename(from_path)
+        module_name = basename[0:basename.find('_')]
+        if self.__options.bintray_repo:
+          self.publish_file(from_path, module_name, version)
+
+        if self.__release_dir:
+          to_path = os.path.join(self.__release_dir, deb_file)
+          pids.append(self.start_copy_file(from_path, to_path))
+
+      return pids
+
   def __do_build(self, subsys):
     try:
       self.start_subsystem_build(subsys).check_wait()
@@ -445,14 +510,37 @@ def build_packages(self):
       if self.__options.build:
         # Build in parallel using half available cores
         # to keep load in check.
+        all_subsystems = []
+        all_subsystems.extend(SUBSYSTEM_LIST)
+        all_subsystems.extend(ADDITIONAL_SUBSYSTEMS)
+        weighted_processes = self.__options.cpu_ratio * multiprocessing.cpu_count()
         pool = multiprocessing.pool.ThreadPool(
-            processes=min(1,
-                        self.__options.cpu_ratio * multiprocessing.cpu_count()))
-        pool.map(self.__do_build, SUBSYSTEM_LIST)
+            processes=int(max(1, weighted_processes)))
+        pool.map(self.__do_build, all_subsystems)
 
       if self.__build_failures:
-        raise RuntimeError('Builds failed for {0!r}'.format(
-          self.__build_failures))
+        if set(self.__build_failures).intersection(set(SUBSYSTEM_LIST)):
+          raise RuntimeError('Builds failed for {0!r}'.format(
+            self.__build_failures))
+        else:
+          print 'Ignoring errors on optional subsystems {0!r}'.format(
+              self.__build_failures)
+
+      if self.__options.nebula:
+        return
+
+      wait_on = set(all_subsystems).difference(set(self.__build_failures))
+      pool = multiprocessing.pool.ThreadPool(processes=len(wait_on))
+      print 'Copying packages...'
+      pool.map(self.__do_copy, wait_on)
+      return
+
+  def __do_copy(self, subsys):
+    print 'Starting to copy {0}...'.format(subsys)
+    pids = self.start_copy_debian_target(subsys)
+    for p in pids:
+      p.check_wait()
+    print 'Finished copying {0}.'.format(subsys)
 
   @staticmethod
   def __zip_dir(zip_file, source_path, arcname=''):
@@ -536,6 +624,13 @@ def init_argument_parser(cls, parser):
         '--nowipe_package_on_409', dest='wipe_package_on_409',
         action='store_false')
 
+      parser.add_argument(
+        '--nebula', default=True, action='store_true',
+        help='Use nebula to build "candidate" target and upload to bintray.')
+      parser.add_argument(
+        '--nonebula', dest='nebula', action='store_false',
+        help='Explicitly "buildDeb" then curl upload them to bintray.')
+
 
   def __verify_bintray(self):
     if not os.environ.get('BINTRAY_KEY', None):

dev/refresh_source.py

@@ -88,7 +88,10 @@ class Refresher(object):
   commands. This class is intended for cross-cutting management.
   """
 
-  __OPTIONAL_REPOSITORIES = [SourceRepository('citest', 'google')]
+  __OPTIONAL_REPOSITORIES = [
+      SourceRepository('citest', 'google'),
+      SourceRepository('spinnaker-monitoring', 'spinnaker')]
+
   __REQUIRED_REPOSITORIES = [
       SourceRepository('spinnaker', 'spinnaker'),
       SourceRepository('clouddriver', 'spinnaker'),

install/first_google_boot.sh

@@ -320,14 +320,20 @@ function extract_spinnaker_gcr_credentials() {
 }
 
 function do_experimental_startup() {
-  local monitor_config=$(get_instance_metadata_attribute "monitor_spinnaker")
-  if [[ ! -z $monitor_config && \
-          -f /opt/spinnaker/install/install_monitor_spinnaker.tz ]]; then
-     echo "$STATUS_PREFIX  Install Monitoring with flags '$monitor_config' "
-     tar xzf /opt/spinnaker/install/install_monitor_spinnaker.tz \
-         -C /opt --no-same-owner
-     /opt/monitor_spinnaker/install_monitoring.sh $monitor_config
-     clear_instance_metadata "monitor_spinnaker"
+  local install_monitoring=$(get_instance_metadata_attribute "install_monitoring")
+  if [[ ! -z $install_monitoring ]]; then
+     IFS=' ' read -r -a all_args <<< "$install_monitoring"
+     local which=${all_args[0]}
+     local flags=${all_args[@]: 1:${#all_args}}
+     /opt/spinnaker-monitoring/third_party/$which/install.sh ${flags[@]}
+
+     if [[ ! -f /opt/spinnaker-monitoring/registry ]]; then
+         mv /opt/spinnaker-monitoring/registry.example \
+            /opt/spinnaker-monitoring/registry
+     fi
+
+     service spinnaker-monitoring restart
+     clear_instance_metadata "install_monitoring"
   fi
 }