Merge pull request #2051 from dnephin/extend_compose_files

Extend compose files by allowing multiple files

compose/cli/command.py

@@ -51,57 +51,68 @@ def perform_command(self, options, handler, command_options):
             handler(None, command_options)
             return
 
-        if 'FIG_FILE' in os.environ:
-            log.warn('The FIG_FILE environment variable is deprecated.')
-            log.warn('Please use COMPOSE_FILE instead.')
-
-        explicit_config_path = options.get('--file') or os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')
-        project = self.get_project(
-            explicit_config_path,
+        project = get_project(
+            self.base_dir,
+            get_config_path(options.get('--file')),
             project_name=options.get('--project-name'),
             verbose=options.get('--verbose'))
 
         handler(project, command_options)
 
-    def get_client(self, verbose=False):
-        client = docker_client()
-        if verbose:
-            version_info = six.iteritems(client.version())
-            log.info("Compose version %s", __version__)
-            log.info("Docker base_url: %s", client.base_url)
-            log.info("Docker version: %s",
-                     ", ".join("%s=%s" % item for item in version_info))
-            return verbose_proxy.VerboseProxy('docker', client)
-        return client
 
-    def get_project(self, config_path=None, project_name=None, verbose=False):
-        config_details = config.find(self.base_dir, config_path)
+def get_config_path(file_option):
+    if file_option:
+        return file_option
 
-        try:
-            return Project.from_dicts(
-                self.get_project_name(config_details.working_dir, project_name),
-                config.load(config_details),
-                self.get_client(verbose=verbose))
-        except ConfigError as e:
-            raise errors.UserError(six.text_type(e))
-
-    def get_project_name(self, working_dir, project_name=None):
-        def normalize_name(name):
-            return re.sub(r'[^a-z0-9]', '', name.lower())
-
-        if 'FIG_PROJECT_NAME' in os.environ:
-            log.warn('The FIG_PROJECT_NAME environment variable is deprecated.')
-            log.warn('Please use COMPOSE_PROJECT_NAME instead.')
-
-        project_name = (
-            project_name or
-            os.environ.get('COMPOSE_PROJECT_NAME') or
-            os.environ.get('FIG_PROJECT_NAME'))
-        if project_name is not None:
-            return normalize_name(project_name)
-
-        project = os.path.basename(os.path.abspath(working_dir))
-        if project:
-            return normalize_name(project)
-
-        return 'default'
+    if 'FIG_FILE' in os.environ:
+        log.warn('The FIG_FILE environment variable is deprecated.')
+        log.warn('Please use COMPOSE_FILE instead.')
+
+    config_file = os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')
+    return [config_file] if config_file else None
+
+
+def get_client(verbose=False):
+    client = docker_client()
+    if verbose:
+        version_info = six.iteritems(client.version())
+        log.info("Compose version %s", __version__)
+        log.info("Docker base_url: %s", client.base_url)
+        log.info("Docker version: %s",
+                 ", ".join("%s=%s" % item for item in version_info))
+        return verbose_proxy.VerboseProxy('docker', client)
+    return client
+
+
+def get_project(base_dir, config_path=None, project_name=None, verbose=False):
+    config_details = config.find(base_dir, config_path)
+
+    try:
+        return Project.from_dicts(
+            get_project_name(config_details.working_dir, project_name),
+            config.load(config_details),
+            get_client(verbose=verbose))
+    except ConfigError as e:
+        raise errors.UserError(six.text_type(e))
+
+
+def get_project_name(working_dir, project_name=None):
+    def normalize_name(name):
+        return re.sub(r'[^a-z0-9]', '', name.lower())
+
+    if 'FIG_PROJECT_NAME' in os.environ:
+        log.warn('The FIG_PROJECT_NAME environment variable is deprecated.')
+        log.warn('Please use COMPOSE_PROJECT_NAME instead.')
+
+    project_name = (
+        project_name or
+        os.environ.get('COMPOSE_PROJECT_NAME') or
+        os.environ.get('FIG_PROJECT_NAME'))
+    if project_name is not None:
+        return normalize_name(project_name)
+
+    project = os.path.basename(os.path.abspath(working_dir))
+    if project:
+        return normalize_name(project)
+
+    return 'default'

compose/cli/main.py

@@ -96,7 +96,7 @@ class TopLevelCommand(Command):
     """Define and run multi-container applications with Docker.
 
     Usage:
-      docker-compose [options] [COMMAND] [ARGS...]
+      docker-compose [-f=<arg>...] [options] [COMMAND] [ARGS...]
       docker-compose -h|--help
 
     Options:

compose/cli/utils.py

@@ -36,25 +36,6 @@ def yesno(prompt, default=None):
         return None
 
 
-def find_candidates_in_parent_dirs(filenames, path):
-    """
-    Given a directory path to start, looks for filenames in the
-    directory, and then each parent directory successively,
-    until found.
-
-    Returns tuple (candidates, path).
-    """
-    candidates = [filename for filename in filenames
-                  if os.path.exists(os.path.join(path, filename))]
-
-    if len(candidates) == 0:
-        parent_dir = os.path.join(path, '..')
-        if os.path.abspath(parent_dir) != os.path.abspath(path):
-            return find_candidates_in_parent_dirs(filenames, parent_dir)
-
-    return (candidates, path)
-
-
 def split_buffer(reader, separator):
     """
     Given a generator which yields strings and a separator string,

compose/config/config.py

@@ -16,7 +16,6 @@
 from .validation import validate_extends_file_path
 from .validation import validate_service_names
 from .validation import validate_top_level_object
-from compose.cli.utils import find_candidates_in_parent_dirs
 
 
 DOCKER_CONFIG_KEYS = [
@@ -77,6 +76,7 @@
     'fig.yaml',
 ]
 
+DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml'
 
 PATH_START_CHARS = [
     '/',
@@ -88,24 +88,45 @@
 log = logging.getLogger(__name__)
 
 
-ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename')
+class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files')):
+    """
+    :param working_dir: the directory to use for relative paths in the config
+    :type  working_dir: string
+    :param config_files: list of configuration files to load
+    :type  config_files: list of :class:`ConfigFile`
+     """
+
+
+class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
+    """
+    :param filename: filename of the config file
+    :type  filename: string
+    :param config: contents of the config file
+    :type  config: :class:`dict`
+    """
 
 
-def find(base_dir, filename):
-    if filename == '-':
-        return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None)
+def find(base_dir, filenames):
+    if filenames == ['-']:
+        return ConfigDetails(
+            os.getcwd(),
+            [ConfigFile(None, yaml.safe_load(sys.stdin))])
 
-    if filename:
-        filename = os.path.join(base_dir, filename)
+    if filenames:
+        filenames = [os.path.join(base_dir, f) for f in filenames]
     else:
-        filename = get_config_path(base_dir)
-    return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename)
+        filenames = get_default_config_files(base_dir)
 
+    log.debug("Using configuration files: {}".format(",".join(filenames)))
+    return ConfigDetails(
+        os.path.dirname(filenames[0]),
+        [ConfigFile(f, load_yaml(f)) for f in filenames])
 
-def get_config_path(base_dir):
+
+def get_default_config_files(base_dir):
     (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir)
 
-    if len(candidates) == 0:
+    if not candidates:
         raise ComposeFileNotFound(SUPPORTED_FILENAMES)
 
     winner = candidates[0]
@@ -123,7 +144,31 @@ def get_config_path(base_dir):
         log.warn("%s is deprecated and will not be supported in future. "
                  "Please rename your config file to docker-compose.yml\n" % winner)
 
-    return os.path.join(path, winner)
+    return [os.path.join(path, winner)] + get_default_override_file(path)
+
+
+def get_default_override_file(path):
+    override_filename = os.path.join(path, DEFAULT_OVERRIDE_FILENAME)
+    return [override_filename] if os.path.exists(override_filename) else []
+
+
+def find_candidates_in_parent_dirs(filenames, path):
+    """
+    Given a directory path to start, looks for filenames in the
+    directory, and then each parent directory successively,
+    until found.
+
+    Returns tuple (candidates, path).
+    """
+    candidates = [filename for filename in filenames
+                  if os.path.exists(os.path.join(path, filename))]
+
+    if not candidates:
+        parent_dir = os.path.join(path, '..')
+        if os.path.abspath(parent_dir) != os.path.abspath(path):
+            return find_candidates_in_parent_dirs(filenames, parent_dir)
+
+    return (candidates, path)
 
 
 @validate_top_level_object
@@ -133,29 +178,49 @@ def pre_process_config(config):
     Pre validation checks and processing of the config file to interpolate env
     vars returning a config dict ready to be tested against the schema.
     """
-    config = interpolate_environment_variables(config)
-    return config
+    return interpolate_environment_variables(config)
 
 
 def load(config_details):
-    config, working_dir, filename = config_details
-
-    processed_config = pre_process_config(config)
-    validate_against_fields_schema(processed_config)
+    """Load the configuration from a working directory and a list of
+    configuration files.  Files are loaded in order, and merged on top
+    of each other to create the final configuration.
 
-    service_dicts = []
+    Return a fully interpolated, extended and validated configuration.
+    """
 
-    for service_name, service_dict in list(processed_config.items()):
+    def build_service(filename, service_name, service_dict):
         loader = ServiceLoader(
-            working_dir=working_dir,
-            filename=filename,
-            service_name=service_name,
-            service_dict=service_dict)
+            config_details.working_dir,
+            filename,
+            service_name,
+            service_dict)
         service_dict = loader.make_service_dict()
         validate_paths(service_dict)
-        service_dicts.append(service_dict)
-
-    return service_dicts
+        return service_dict
+
+    def load_file(filename, config):
+        processed_config = pre_process_config(config)
+        validate_against_fields_schema(processed_config)
+        return [
+            build_service(filename, name, service_config)
+            for name, service_config in processed_config.items()
+        ]
+
+    def merge_services(base, override):
+        all_service_names = set(base) | set(override)
+        return {
+            name: merge_service_dicts(base.get(name, {}), override.get(name, {}))
+            for name in all_service_names
+        }
+
+    config_file = config_details.config_files[0]
+    for next_file in config_details.config_files[1:]:
+        config_file = ConfigFile(
+            config_file.filename,
+            merge_services(config_file.config, next_file.config))
+
+    return load_file(config_file.filename, config_file.config)
 
 
 class ServiceLoader(object):