AGENT-388 Add extra_config_path as an option (#541)

Under kubernetes, it is not convenient to specify custom configuration snippets
in the agent.d directory without overriding core configuration snippets.

This PR allows users to keep the default configuration snippets we provide, but
also provide their own configuration snippets in an additional configuration
directory.

This additional configuration directory is specified via an environment
variable or a command line parameter.

If present, the agent will process all .json files in this directory and add
them to its current configuration.

This only applies to the main agent, and not the agent config scripts.

scalyr_agent/agent_main.py

@@ -149,6 +149,10 @@ def __init__(self, platform_controller):
         self.__default_paths = platform_controller.default_paths
 
         self.__config_file_path = None
+
+        # An extra directory for config snippets
+        self.__extra_config_dir = None
+
         # If the current contents of the configuration file has errors in it, then this will be set to the config
         # object produced by reading it.
         self.__current_bad_config = None
@@ -245,6 +249,10 @@ def main(self, config_file_path, command, command_options):
         self.__no_fork = command_options.no_fork
         no_check_remote = False
 
+        self.__extra_config_dir = Configuration.get_extra_config_dir(
+            command_options.extra_config_dir
+        )
+
         # We process for the 'version' command early since we do not need the configuration file for it.
         if command == "version":
             print("The Scalyr Agent 2 version is %s" % SCALYR_VERSION)
@@ -363,7 +371,12 @@ def __make_config(self, config_file_path):
         @return: The configuration object.
         @rtype: scalyr_agent.Configuration
         """
-        return Configuration(config_file_path, self.__default_paths, log)
+        return Configuration(
+            config_file_path,
+            self.__default_paths,
+            log,
+            extra_config_dir=self.__extra_config_dir,
+        )
 
     def __verify_config(
         self,
@@ -1750,6 +1763,12 @@ def stop(self):
         help="Read configuration from FILE",
         metavar="FILE",
     )
+    parser.add_option(
+        "--extra-config-dir",
+        default=None,
+        help="An extra directory to check for configuration files",
+        metavar="PATH",
+    )
     parser.add_option(
         "-q",
         "--quiet",

scalyr_agent/config_main.py

@@ -978,6 +978,8 @@ def export_config(config_dest, config_file_path, configuration):
         # raw value for this configuration to avoid it making the path absolute when we want the relative.
         fragment_dir = configuration.config_directory_raw
 
+        # TODO - AGENT-400, should add support for the extra-config-dir here
+
         # If it was absolute, try to make it relative.
         if os.path.isabs(fragment_dir):
             fragment_dir = relative_path(

scalyr_agent/configuration.py

@@ -46,6 +46,7 @@
 
 from scalyr_agent.__scalyr__ import get_install_root
 from scalyr_agent.compat import os_environ_unicode
+from scalyr_agent import compat
 
 
 class Configuration(object):
@@ -75,7 +76,7 @@ class Configuration(object):
     DEFAULT_K8S_IGNORE_NAMESPACES = ["kube-system"]
     DEFAULT_K8S_INCLUDE_NAMESPACES = ["*"]
 
-    def __init__(self, file_path, default_paths, logger):
+    def __init__(self, file_path, default_paths, logger, extra_config_dir=None):
         # Captures all environment aware variables for testing purposes
         self._environment_aware_map = {}
         self.__file_path = os.path.abspath(file_path)
@@ -106,6 +107,9 @@ def __init__(self, file_path, default_paths, logger):
         self.max_retry_time = 15 * 60
         self.max_allowed_checkpoint_age = 15 * 60
 
+        # An additional directory to look for config snippets
+        self.__extra_config_directory = extra_config_dir
+
         self.__logger = logger
 
     def parse(self):
@@ -151,8 +155,14 @@ def parse(self):
                 "server_attributes",
             )
 
+            # Get any configuration snippets in the config directory
+            extra_config = self.__list_files(self.config_directory)
+
+            # Plus any configuration snippets in the additional config directory
+            extra_config.extend(self.__list_files(self.extra_config_directory))
+
             # Now, look for any additional configuration in the config fragment directory.
-            for fp in self.__list_files(self.config_directory):
+            for fp in extra_config:
                 self.__additional_paths.append(fp)
                 content = scalyr_util.read_config_file_as_json(fp)
                 for k in content.keys():
@@ -864,6 +874,26 @@ def config_directory_raw(self):
         """Returns the configuration value for 'config_directory', as recorded in the configuration file."""
         return self.__get_config().get_string("config_directory")
 
+    @property
+    def extra_config_directory(self):
+        """Returns the configuration value for `extra_config_directory`, resolved to full path if
+        necessary.  """
+
+        # If `extra_config_directory` is a relative path, then it will be relative
+        # to the directory containing the main config file
+        if self.__extra_config_directory is None:
+            return None
+
+        return self.__resolve_absolute_path(
+            self.__extra_config_directory,
+            self.__get_parent_directory(self.__file_path),
+        )
+
+    @property
+    def extra_config_directory_raw(self):
+        """Returns the configuration value for 'extra_config_directory'."""
+        return self.__extra_config_directory
+
     @property
     def max_allowed_request_size(self):
         """Returns the configuration value for 'max_allowed_request_size'."""
@@ -1121,6 +1151,20 @@ def equivalent(self, other, exclude_debug_level=False):
             if original_debug_level is not None:
                 other.__config.put("debug_level", original_debug_level)
 
+    @staticmethod
+    def get_extra_config_dir(extra_config_dir):
+        """
+        Returns the value for the additional config directory - either from the value passed
+        in, or from the environment variable `SCALYR_EXTRA_CONFIG_DIR`.
+
+        @param extra_config_dir: the additinal configuration directory.  If this value is
+            None, then the environment variable `SCALYR_EXTRA_CONFIG_DIR` is read for the result
+        """
+        result = extra_config_dir
+        if extra_config_dir is None:
+            result = compat.os_getenv_unicode("SCALYR_EXTRA_CONFIG_DIR")
+        return result
+
     @staticmethod
     def default_ca_cert_path():
         """Returns the default configuration file path for the agent."""
@@ -1182,6 +1226,8 @@ def __list_files(self, directory_path):
         @return: If the directory exists and can be read, the list of files ending in .json (not directories).
         """
         result = []
+        if directory_path is None:
+            return result
         if not os.path.isdir(directory_path):
             return result
         if not os.access(directory_path, os.R_OK):

tests/unit/configuration_test.py

@@ -64,6 +64,8 @@ def setUp(self):
         self._config_file = os.path.join(self._config_dir, "agent.json")
         self._config_fragments_dir = os.path.join(self._config_dir, "agent.d")
         os.makedirs(self._config_fragments_dir)
+        self._extra_config_fragments_dir = tempfile.mkdtemp() + "extra"
+        os.makedirs(self._extra_config_fragments_dir)
         for key in os_environ_unicode.keys():
             if "scalyr" in key.lower():
                 del os.environ[key]
@@ -106,15 +108,18 @@ def _write_file_with_separator_conversion(self, contents):
         fp.close()
 
     def _write_config_fragment_file_with_separator_conversion(
-        self, file_path, contents
+        self, file_path, contents, config_dir=None
     ):
+        if config_dir is None:
+            config_dir = self._config_fragments_dir
+
         contents = scalyr_util.json_encode(
             self.__convert_separators(
                 scalyr_util.json_scalyr_config_decode(contents)
             ).to_dict()
         )
 
-        full_path = os.path.join(self._config_fragments_dir, file_path)
+        full_path = os.path.join(config_dir, file_path)
         fp = open(full_path, "w")
         fp.write(contents)
         fp.close()
@@ -130,7 +135,7 @@ def __init__(self, config):
             self.config = config
             self.log_config = {"path": self.module_name.split(".")[-1] + ".log"}
 
-    def _create_test_configuration_instance(self, logger=None):
+    def _create_test_configuration_instance(self, logger=None, extra_config_dir=None):
         """Creates an instance of a Configuration file for testing.
 
         @return:  The test instance
@@ -144,7 +149,9 @@ def _create_test_configuration_instance(self, logger=None):
             self.convert_path("/var/lib/scalyr-agent-2"),
         )
 
-        return Configuration(self._config_file, default_paths, logger)
+        return Configuration(
+            self._config_file, default_paths, logger, extra_config_dir=extra_config_dir
+        )
 
     # noinspection PyPep8Naming
     def assertPathEquals(self, actual_path, expected_path):
@@ -935,6 +942,64 @@ def test_ignore_non_json_files_in_config_dir(self):
 
         self.assertEquals(len(config.log_configs), 2)
 
+    def test_extra_config_dir_absolute(self):
+
+        self._write_file_with_separator_conversion(""" { api_key: "main-api-key" } """)
+        self._write_config_fragment_file_with_separator_conversion(
+            "extra.json",
+            """ {
+           max_line_size: 10,
+          }
+        """,
+            config_dir=self._extra_config_fragments_dir,
+        )
+        config = self._create_test_configuration_instance(
+            extra_config_dir=self._extra_config_fragments_dir
+        )
+        config.parse()
+        self.assertEquals(config.api_key, "main-api-key")
+        self.assertEquals(config.max_line_size, 10)
+
+    def test_extra_config_dir_relative(self):
+        self._write_file_with_separator_conversion(""" { api_key: "main-api-key" } """)
+        extra_dir = os.path.join(self._config_dir, "extra")
+        os.makedirs(extra_dir)
+        self._write_config_fragment_file_with_separator_conversion(
+            "extra.json",
+            """ {
+           max_line_size: 10,
+          }
+        """,
+            config_dir=extra_dir,
+        )
+        config = self._create_test_configuration_instance(extra_config_dir="extra")
+        config.parse()
+        self.assertEquals(config.api_key, "main-api-key")
+        self.assertEquals(config.max_line_size, 10)
+
+    def test_raw_extra_config(self):
+        self._write_file_with_separator_conversion(""" { api_key: "main-api-key" } """)
+        extra_dir = os.path.join(self._config_dir, "extra")
+        os.makedirs(extra_dir)
+        self._write_config_fragment_file_with_separator_conversion(
+            "extra.json",
+            """ {
+           max_line_size: 10,
+          }
+        """,
+            config_dir=extra_dir,
+        )
+        config = self._create_test_configuration_instance(extra_config_dir="extra")
+        config.parse()
+        self.assertEquals(config.extra_config_directory, extra_dir)
+        self.assertEquals(config.extra_config_directory_raw, "extra")
+
+    def test_no_raw_extra_config(self):
+        self._write_file_with_separator_conversion(""" { api_key: "main-api-key" } """)
+        config = self._create_test_configuration_instance()
+        config.parse()
+        self.assertTrue(config.extra_config_directory_raw is None)
+
     def test_parser_specification(self):
         self._write_file_with_separator_conversion(
             """ {