feat(jsonnet): add port conflict solver as jsonnet postprocessor

docker-compose configuration generated by jsonnet is now postprocessed by
a python function to dedupe port mappings of the same `published` value.

When a `published` is conflicting a previously defined one, it is incremented
until the conflict is solved, making all published ports from all services
distinct.

ddb/feature/jsonnet/actions.py

@@ -2,6 +2,7 @@
 import json
 import os
 import re
+from importlib import import_module
 from typing import Tuple, Union, Iterable, Optional
 
 import yaml
@@ -46,9 +47,34 @@ def _render_template(self, template: str, target: str) -> Iterable[Tuple[Union[s
         else:
             ext = os.path.splitext(target)[-1]
             if ext.lower() in ['.yaml', '.yml']:
-                evaluated = yaml.safe_dump(json.loads(evaluated))
+                data = json.loads(evaluated)
+                self._do_postprocess(data)
+                evaluated = yaml.safe_dump(data)
+            else:
+                if '__post_processors__' in evaluated:
+                    data = json.loads(evaluated)
+                    if self._do_postprocess(data):
+                        evaluated = json.dumps(data)
+
             yield evaluated, target
 
+    @staticmethod
+    def _do_postprocess(data):
+        """
+        Post process jsonnet output with python functions.
+        """
+        if isinstance(data, dict) and '__post_processors__' in data.keys():
+            for post_processor_str in data.pop('__post_processors__'):
+                if '__value__' in data:
+                    data = data.pop('__value__')
+                module_name, func_name = post_processor_str.rsplit('.', 1)
+                if module_name == 'ddb.feature.jsonnet.docker':
+                    mod = import_module(module_name)
+                    post_processor = getattr(mod, func_name)
+                    post_processor(data)
+                    return True
+        return False
+
     def _autofix_render_error(self,
                               template: str,
                               target: str,

ddb/feature/jsonnet/docker.py

@@ -0,0 +1,36 @@
+from collections import defaultdict
+
+from ddb.feature.docker.lib.compose.config.types import ServicePort
+
+
+def apply_resolve_ports_conflicts(compose):
+    """
+    Resolve ports conflicts in docker compose configuration.
+    """
+    published = defaultdict(set)
+
+    if 'services' in compose:  # pylint:disable=too-many-nested-blocks
+        for service in compose['services'].values():
+            if 'ports' in service:
+                service_ports = service['ports']
+                for index, port in enumerate(tuple(service_ports)):
+                    parsed_ports = ServicePort.parse(port)
+                    if len(parsed_ports) == 1:
+                        parsed_port = parsed_ports[0]
+                        new_published = parsed_port.published
+                        while new_published in published[parsed_port.protocol]:
+                            new_published = new_published + 1
+                        published[parsed_port.protocol].add(new_published)
+                        if parsed_port.published != new_published:
+                            if isinstance(port, str):
+                                service_port_data = dict(parsed_port._asdict())
+                                service_port_data['published'] = new_published
+                                new_port = ServicePort(**service_port_data)
+                                port = new_port.legacy_repr()
+                                if not new_port.protocol and port.endswith('/tcp'):
+                                    port = port[:-4]
+                            else:
+                                port['published'] = str(new_published)
+                            service_ports[index] = port
+
+    return compose

ddb/feature/jsonnet/lib/ddb.docker.libjsonnet

@@ -144,6 +144,9 @@ local apply_docker_compose_included_services(compose, included_services=_docker_
     else
       compose;
 
+local apply_resolve_ports_conflicts(compose) =
+	compose + {['__post_processors__']+: ['ddb.feature.jsonnet.docker.apply_resolve_ports_conflicts']};
+
 local ServiceName(name=null) = std.join("-", std.prune([_core_project_name, name]));
 
 local Volumes(services) = {
@@ -206,7 +209,7 @@ local Networks(services, networks_names=_docker_networks_names) = {
 };
 
 local Compose(config={}, networks_names=_docker_networks_names, version=_docker_compose_version) =
-    apply_docker_compose_included_services(apply_docker_compose_excluded_services(config)) + {
+    apply_resolve_ports_conflicts(apply_docker_compose_included_services(apply_docker_compose_excluded_services(config))) + {
         "version": version,
         "networks": Networks(self.services, networks_names),
         "volumes": Volumes(self.services)

tests/it/test_jsonnet_it.data/jsonnet-resolve-ports-conflicts/docker-compose.expected.yml

@@ -0,0 +1,66 @@
+networks: {}
+services:
+  db_a:
+    image: postgres
+    init: true
+    ports:
+    - 2933:5433
+    restart: 'no'
+  db_b:
+    image: postgres
+    init: true
+    ports:
+    - 2932:5432
+    restart: 'no'
+  db_c:
+    image: postgres
+    init: true
+    ports:
+    - 2934:5432
+    restart: 'no'
+  db_d:
+    image: postgres
+    init: true
+    ports:
+    - 2931:5431
+    restart: 'no'
+  db_e:
+    image: postgres
+    init: true
+    ports:
+    - 2938:5438
+    restart: 'no'
+  db_f:
+    image: postgres
+    init: true
+    ports:
+    - 2935:5433
+    restart: 'no'
+  db_g:
+    image: postgres
+    init: true
+    ports:
+    - 2932:5432/udp
+    restart: 'no'
+  db_h:
+    image: postgres
+    init: true
+    ports:
+    - 2933:5432/udp
+    restart: 'no'
+  db_i:
+    image: postgres
+    init: true
+    ports:
+    - published: '2939'
+      target: '5438'
+    restart: 'no'
+  db_j:
+    image: postgres
+    init: true
+    ports:
+    - published: '2940'
+      target: '5438'
+    restart: 'no'
+version: '3.7'
+volumes: {}

tests/it/test_jsonnet_it.data/jsonnet-resolve-ports-conflicts/docker-compose.yml.jsonnet

@@ -0,0 +1,20 @@
+local ddb = import 'ddb.docker.libjsonnet';
+
+ddb.Compose({
+	services: {
+		db_a: ddb.Image("postgres") + ddb.Expose("5433"),
+		db_b: ddb.Image("postgres") + ddb.Expose("5432"),
+		db_c: ddb.Image("postgres") + ddb.Expose("5432"),
+		db_d: ddb.Image("postgres") + ddb.Expose("5431"),
+		db_e: ddb.Image("postgres") + ddb.Expose("5438"),
+		db_f: ddb.Image("postgres") + ddb.Expose("5433"),
+		db_g: ddb.Image("postgres") + ddb.Expose("5432", protocol="udp"),
+		db_h: ddb.Image("postgres") + ddb.Expose("5432", protocol="udp"),
+		db_i: ddb.Image("postgres") + {
+			ports: [{published: '2938', target: '5438'}]
+		},
+		db_j: ddb.Image("postgres") + {
+			ports: [{published: '2938', target: '5438'}]
+		},
+	}
+})

tests/it/test_jsonnet_it.py

@@ -57,3 +57,17 @@ def test_binary(self, project_loader):
             docker_compose_expected = yaml.load(f, yaml.SafeLoader)
 
         assert docker_compose == docker_compose_expected
+
+    def test_resolve_ports_conflicts(self, project_loader):
+        project_loader("jsonnet-resolve-ports-conflicts")
+
+        main(["configure"])
+
+        assert os.path.exists('docker-compose.yml')
+        with open('docker-compose.yml', 'r') as f:
+            docker_compose = yaml.safe_load(f)
+
+        with open('docker-compose.expected.yml', 'r') as f:
+            docker_compose_expected = yaml.safe_load(f)
+
+        assert docker_compose == docker_compose_expected