fix(config): Fixed some provider-related config

spinnaker-monitoring-daemon/spinnaker-monitoring/datadog_service.py

@@ -27,7 +27,7 @@
   datadog_available = True
 except ImportError:
   datadog_available = False
-  
+
 
 import spectator_client
 
@@ -55,15 +55,16 @@ def api(self):
   def __init__(self, api_key, app_key, host=None):
     """Constructs the object."""
     if not datadog_available:
-      raise ImportError('You must "pip install datadog" to get the datadog client library.')
+      raise ImportError(
+          'You must "pip install datadog" to get the datadog client library.')
     self.__api = None
     self.__host = host
     self.__api_key = api_key
     self.__app_key = app_key
 
   def __append_timeseries_point(
-        self, service, name,
-        instance, metric_metadata, service_metadata, result):
+      self, service, name,
+      instance, metric_metadata, service_metadata, result):
     """Creates a post payload for a DataDog time series data point.
 
        See http://docs.datadoghq.com/api/?lang=python#metrics-post.
@@ -120,37 +121,34 @@ def read_param(param_name, config_text):
     return match.group(1).strip()
 
 
-  app_key = None
-  api_key = None
-  host = None
-  config_path = options['dd_agent_config']
-
-  try:
-    with open(config_path, 'r') as stream:
-      logging.info('Reading Datadog config from %s', config_path)
-      text = stream.read()
-      app_key = read_param('app_key', text)
-      api_key = read_param('api_key', text)
-      host = read_param('hostname', text)
+  datadog_options = options.get('datadog', {})
+  api_key = os.environ.get('DATADOG_API_KEY', datadog_options.get('api_key'))
+  app_key = os.environ.get('DATADOG_APP_KEY', datadog_options.get('app_key'))
+  host = options.get('datadog_host', datadog_options.get('host'))
+  datadog_host = None
+
+  if not api_key or not app_key or host is None:
+    config_path = options['dd_agent_config']
+    try:
+      with open(config_path, 'r') as stream:
+        logging.info('Reading Datadog config from %s', config_path)
+        text = stream.read()
+        app_key = app_key or read_param('app_key', text)
+        api_key = api_key or read_param('api_key', text)
+        datadog_host = read_param('hostname', text)
+    except IOError:
+      logging.warning('Could not read config from datadog "%s": %s',
+                      config_path, traceback.format_exc())
 
-  # pylint: disable=bare-except
-  except:
-    logging.warning('Could not read config from "%s": %s',
-                    config_path, traceback.format_exc())
-
-  api_key = api_key or os.environ.get('DATADOG_API_KEY',
-                                      options.get('datadog', {}).get('api_key'))
   if api_key is None:
     raise ValueError('DATADOG_API_KEY is not defined')
 
-  # This can be None
-  app_key = app_key or os.environ.get('DATADOG_APP_KEY',
-                                      options.get('datadog', {}).get('app_key'))
-  host = (host
-          or socket.getfqdn(
-              options.get('datadog_host',
-                          options.get('datadog', {}).get('host', '')))
-          or '')
+  # This is convoluted because we are only going to fqdn hosts we give.
+  # We probably could regardless, but maybe we should keep datadog as is.
+  if host is None and datadog_host is None:
+    host = ''
+  host = socket.getfqdn(host) if host is not None else datadog_host
+
   return DatadogMetricsService(api_key=api_key, app_key=app_key, host=host)
 
 

spinnaker-monitoring-daemon/spinnaker-monitoring/server_handlers.py

@@ -171,7 +171,8 @@ def __call__(self, options, metric_service_list):
     It is put here in a callable so that we can run this in a separate thread.
     The main thread will be the standard WebServer.
     """
-    period = options['period']
+    period = (options.get('monitor_period')
+              or options.get('monitor', {}).get('period', 60))
     catalog = spectator_client.get_source_catalog(options)
     spectator = spectator_client.SpectatorClient(options)
 
@@ -225,7 +226,8 @@ def add_argparser(self, subparsers):
     parser = super(MonitorCommandHandler, self).add_argparser(subparsers)
     for factory in MonitorCommandHandler._service_factories:
       factory.add_argparser(parser)
-    parser.add_argument('--period', default=60, type=int)
+    parser.add_argument('--period', dest='monitor_period',
+                        default=None, type=int)
     return parser
 
 

spinnaker-monitoring-daemon/spinnaker-monitoring/stackdriver_service.py

@@ -15,16 +15,15 @@
 # pylint: disable=missing-docstring
 
 from datetime import datetime
-import httplib2
 import json
 import logging
 import os
 import re
 import traceback
 import urllib2
+import httplib2
 
 import spectator_client
-import util
 
 
 try:
@@ -126,7 +125,7 @@ def __ec2_monitored_resource_or_none(self):
           'region': doc['region'],
           'aws_account': doc['accountId'],
           'project_id': self.__project
-       }
+      }
     except (IOError, ValueError, KeyError):
       return None
 
@@ -276,7 +275,7 @@ def add_label_and_retry(self, label, metric_type, ts_request):
       logging.info('Retrying create timeseries %s', ts_request)
       (self.stub.projects().timeSeries().create(
           name=self.project_to_resource(self.__project),
-               body={'timeSeries': ts_request})
+          body={'timeSeries': ts_request})
        .execute())
 
   def add_label_to_metric(self, label, metric_type):
@@ -390,12 +389,14 @@ def make_stub(options):
   """Helper function for making a stub to talk to service."""
   if not stackdriver_available:
       raise ImportError(
-           'You must "pip install google-api-python-client oauth2client" to get the stackdriver client library.')
-    
+          'You must "pip install google-api-python-client oauth2client" to get the stackdriver client library.')
+
   stackdriver_config = options.get('stackdriver', {})
   credentials_path = options.get('credentials_path', None)
   if credentials_path is None:
     credentials_path = stackdriver_config.get('credentials_path')
+  if credentials_path:
+    credentials_path = os.path.expandvars(credentials_path)
 
   http = httplib2.Http()
   http = apiclient.http.set_user_agent(

spinnaker-monitoring-daemon/tests/datadog_service_test.py

@@ -42,7 +42,7 @@ def test_initialize_once(self, mock_initialize):
 
   @patch('datadog_service.datadog.initialize')
   def test_initialize_from_dd_agent_config(self, mock_initialize):
-    options = {'dd_agent_config': 'testCONFIG', 'datadog_host': 'testHost'}
+    options = {'dd_agent_config': 'testCONFIG'}
     with patch('datadog_service.open',
                mock.mock_open(read_data='#api_key: COMMENT\n'
                                         'api_key: FOUND_KEY\n'
@@ -56,6 +56,23 @@ def test_initialize_from_dd_agent_config(self, mock_initialize):
     mock_initialize.assert_called_with(
           api_key='FOUND_KEY', app_key=None, host_name='FOUND_HOST')
 
+  @patch('datadog_service.datadog.initialize')
+  def test_initialize_from_options(self, mock_initialize):
+    options = {'dd_agent_config': 'testCONFIG',
+               'datadog': {'api_key': 'testApi', 'host': 'testHost'}}
+    with patch('datadog_service.open',
+               mock.mock_open(read_data='#api_key: COMMENT\n'
+                                        'api_key: FOUND_KEY\n'
+                                        'hostname: FOUND_HOST\n'),
+               create=True) as mock_patch:
+        service = datadog_service.make_datadog_service(options)
+
+    mock_patch.assert_called_with('testCONFIG','r')
+    self.assertIsNotNone(service)
+    self.assertIsNotNone(service.api)  # initialize on demand
+    mock_initialize.assert_called_with(
+          api_key='testApi', app_key=None, host_name='testHost')
+
   @patch('datadog_service.socket.getfqdn')
   @patch('datadog_service.datadog.initialize')
   def test_initialize_from_localhost_config(