Implement extends

Signed-off-by: Aanand Prasad <aanand.prasad@gmail.com>

Conflicts:
	compose/config.py
	tests/unit/config_test.py

compose/config.py

@@ -79,9 +79,70 @@ def make_service_dict(name, options, working_dir=None):
     service_dict = options.copy()
     service_dict['name'] = name
     service_dict = resolve_environment(service_dict, working_dir=working_dir)
+    service_dict = resolve_extends(service_dict, working_dir=working_dir)
     return process_container_options(service_dict, working_dir=working_dir)
 
 
+def resolve_extends(service_dict, working_dir=None):
+    if 'extends' not in service_dict:
+        return service_dict
+
+    extends_options = process_extends_options(service_dict['name'], service_dict['extends'])
+
+    if working_dir is None:
+        raise Exception("No working_dir passed to resolve_extends()")
+
+    other_config_path = expand_path(working_dir, extends_options['file'])
+    other_working_dir = os.path.dirname(other_config_path)
+
+    other_config = load_yaml(other_config_path)
+    other_service_dict = other_config[extends_options['service']]
+    other_service_dict = make_service_dict(
+        service_dict['name'],
+        other_service_dict,
+        working_dir=other_working_dir,
+    )
+    validate_extended_service_dict(
+        other_service_dict,
+        filename=other_config_path,
+        service=extends_options['service'],
+    )
+
+    return merge_service_dicts(other_service_dict, service_dict)
+
+
+def process_extends_options(service_name, extends_options):
+    error_prefix = "Invalid 'extends' configuration for %s:" % service_name
+
+    if not isinstance(extends_options, dict):
+        raise ConfigurationError("%s must be a dictionary" % error_prefix)
+
+    if 'service' not in extends_options:
+        raise ConfigurationError(
+            "%s you need to specify a service, e.g. 'service: web'" % error_prefix
+        )
+
+    for k, _ in extends_options.items():
+        if k not in ['file', 'service']:
+            raise ConfigurationError(
+                "%s unsupported configuration option '%s'" % (error_prefix, k)
+            )
+
+    return extends_options
+
+
+def validate_extended_service_dict(service_dict, filename, service):
+    error_prefix = "Cannot extend service '%s' in %s:" % (service, filename)
+
+    if 'links' in service_dict:
+        raise ConfigurationError("%s services with 'links' cannot be extended" % error_prefix)
+
+    if 'volumes_from' in service_dict:
+        raise ConfigurationError("%s services with 'volumes_from' cannot be extended" % error_prefix)
+
+    # TODO: net:container
+
+
 def process_container_options(service_dict, working_dir=None):
     for k in service_dict:
         if k not in ALLOWED_KEYS:
@@ -93,6 +154,29 @@ def process_container_options(service_dict, working_dir=None):
     return service_dict
 
 
+def merge_service_dicts(base, override):
+    d = base.copy()
+
+    if 'environment' in base or 'environment' in override:
+        d['environment'] = merge_environment(
+            base.get('environment'),
+            override.get('environment'),
+        )
+
+    for k in ALLOWED_KEYS:
+        if k not in ['environment']:
+            if k in override:
+                d[k] = override[k]
+
+    return d
+
+
+def merge_environment(base, override):
+    env = parse_environment(base)
+    env.update(parse_environment(override))
+    return env
+
+
 def parse_links(links):
     return dict(parse_link(l) for l in links)
 

docs/yml.md

@@ -176,6 +176,83 @@ env_file:
 RACK_ENV: development
 ```
 
+### extends
+
+Extend another service, in the current file or another, optionally overriding
+configuration.
+
+Here's a simple example. Suppose we have 2 files - **common.yml** and
+**development.yml**. We can use `extends` to define a service in
+**development.yml** which uses configuration defined in **common.yml**:
+
+**common.yml**
+
+```
+webapp:
+  build: ./webapp
+  environment:
+    - DEBUG=false
+    - SEND_EMAILS=false
+```
+
+**development.yml**
+
+```
+web:
+  extends:
+    file: common.yml
+    service: webapp
+  ports:
+    - "8000:8000"
+  links:
+    - db
+  environment:
+    - DEBUG=true
+db:
+  image: postgres
+```
+
+Here, the `web` service in **development.yml** inherits the configuration of
+the `webapp` service in **common.yml** - the `build` and `environment` keys -
+and adds `ports` and `links` configuration. It overrides one of the defined
+environment variables (DEBUG) with a new value, and the other one
+(SEND_EMAILS) is left untouched. It's exactly as if you defined `web` like
+this:
+
+```yaml
+web:
+  build: ./webapp
+  ports:
+    - "8000:8000"
+  links:
+    - db
+  environment:
+    - DEBUG=true
+    - SEND_EMAILS=false
+```
+
+The `extends` option is great for sharing configuration between different
+apps, or for configuring the same app differently for different environments.
+You could write a new file for a staging environment, **staging.yml**, which
+binds to a different port and doesn't turn on debugging:
+
+```
+web:
+  extends:
+    file: common.yml
+    service: webapp
+  ports:
+    - "80:8000"
+  links:
+    - db
+db:
+  image: postgres
+```
+
+> **Note:** When you extend a service, `links` and `volumes_from`
+> configuration options are **not** inherited - you will have to define
+> those manually each time you extend it.
+
 ### net
 
 Networking mode. Use the same values as the docker client `--net` parameter.

tests/fixtures/extends/common.yml

@@ -0,0 +1,6 @@
+web:
+  image: busybox
+  command: /bin/true
+  environment:
+    - FOO=1
+    - BAR=1

tests/fixtures/extends/docker-compose.yml

@@ -0,0 +1,16 @@
+myweb:
+  extends:
+    file: common.yml
+    service: web
+  command: sleep 300
+  links:
+    - "mydb:db"
+  environment:
+    # leave FOO alone
+    # override BAR
+    BAR: "2"
+    # add BAZ
+    BAZ: "2"
+mydb:
+  image: busybox
+  command: sleep 300

tests/integration/cli_test.py

@@ -425,3 +425,30 @@ def test_env_file_relative_to_compose_file(self):
         containers = self.project.containers(stopped=True)
         self.assertEqual(len(containers), 1)
         self.assertIn("FOO=1", containers[0].get('Config.Env'))
+
+    def test_up_with_extends(self):
+        self.command.base_dir = 'tests/fixtures/extends'
+        self.command.dispatch(['up', '-d'], None)
+
+        self.assertEqual(
+            set([s.name for s in self.project.services]),
+            set(['mydb', 'myweb']),
+        )
+
+        # Sort by name so we get [db, web]
+        containers = sorted(
+            self.project.containers(stopped=True),
+            key=lambda c: c.name,
+        )
+
+        self.assertEqual(len(containers), 2)
+        web = containers[1]
+
+        self.assertEqual(set(web.links()), set(['db', 'mydb_1', 'extends_mydb_1']))
+
+        expected_env = set([
+            "FOO=1",
+            "BAR=2",
+            "BAZ=2",
+        ])
+        self.assertTrue(expected_env <= set(web.get('Config.Env')))

tests/unit/config_test.py

@@ -38,6 +38,8 @@ def test_config_validation(self):
         )
         config.make_service_dict('foo', {'ports': ['8000']})
 
+
+class EnvTest(unittest.TestCase):
     def test_parse_environment_as_list(self):
         environment =[
             'NORMAL=F1',
@@ -130,3 +132,71 @@ def test_resolve_environment_from_file(self):
             service_dict['environment'],
             {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''},
         )
+
+
+class ExtendsTest(unittest.TestCase):
+    def test_extends(self):
+        service_dicts = config.load('tests/fixtures/extends/docker-compose.yml')
+
+        service_dicts = sorted(
+            service_dicts,
+            key=lambda sd: sd['name'],
+        )
+
+        self.assertEqual(service_dicts, [
+            {
+                'name': 'mydb',
+                'image': 'busybox',
+                'command': 'sleep 300',
+            },
+            {
+                'name': 'myweb',
+                'image': 'busybox',
+                'command': 'sleep 300',
+                'links': ['mydb:db'],
+                'environment': {
+                    "FOO": "1",
+                    "BAR": "2",
+                    "BAZ": "2",
+                },
+            }
+        ])
+
+    def test_extends_validation(self):
+        dictionary = {'extends': None}
+        load_config = lambda: config.make_service_dict('myweb', dictionary, 'tests/fixtures/extends')
+
+        self.assertRaisesRegexp(config.ConfigurationError, 'dictionary', load_config)
+
+        dictionary['extends'] = {}
+        self.assertRaises(config.ConfigurationError, load_config)
+
+        dictionary['extends']['file'] = 'common.yml'
+        self.assertRaisesRegexp(config.ConfigurationError, 'service', load_config)
+
+        dictionary['extends']['service'] = 'web'
+        self.assertIsInstance(load_config(), dict)
+
+        dictionary['extends']['what'] = 'is this'
+        self.assertRaisesRegexp(config.ConfigurationError, 'what', load_config)
+
+    def test_blacklisted_options(self):
+        def load_config():
+            return config.make_service_dict('myweb', {
+                'extends': {
+                    'file': 'whatever',
+                    'service': 'web',
+                }
+            }, '.')
+
+        with self.assertRaisesRegexp(config.ConfigurationError, 'links'):
+            other_config = {'web': {'links': ['db']}}
+
+            with mock.patch.object(config, 'load_yaml', return_value=other_config):
+                print load_config()
+
+        with self.assertRaisesRegexp(config.ConfigurationError, 'volumes_from'):
+            other_config = {'web': {'volumes_from': ['db']}}
+
+            with mock.patch.object(config, 'load_yaml', return_value=other_config):
+                print load_config()