feat(jsonnet): add mount options to mount named volumes inside projec…

…t directory

ddb/feature/docker/actions.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
+import keyword
 import os
 import re
-import keyword
 from pathlib import PurePosixPath, Path
 from typing import Union, Iterable, List, Dict, Set
 
@@ -292,6 +292,9 @@ def execute(self, docker_compose_config: dict):
 
         volume_mappings = self._get_volume_mappings(docker_compose_config, external_volumes)
 
+        volume_mappings = [(source, target) for (source, target) in volume_mappings if
+                           self._check_file_in_project(source)]
+
         for source, target in volume_mappings:
             self._create_local_volume(source, target)
 
@@ -328,14 +331,22 @@ def _fix_owner(relative_path):
                                       config.data.get("docker.user.uid"), config.data.get("docker.user.gid"),
                                       relative_path)
 
+    @staticmethod
+    def _check_file_in_project(target):
+        target_path = os.path.realpath(target)
+        cwd = os.path.realpath(".")
+        return target_path.startswith(cwd)
+
     @staticmethod
     def _create_local_volume(source, target):
         rel_source = os.path.relpath(source, ".")
         if os.path.exists(source):
             context.log.notice("Local volume source: %s (exists)", rel_source)
         else:
             _, source_ext = os.path.splitext(source)
-            _, target_ext = os.path.splitext(target)
+            target_ext = None
+            if target:
+                _, target_ext = os.path.splitext(target)
             if source_ext or target_ext:
                 # Create empty file, because with have an extension in source or target.
                 os.makedirs(str(Path(source).parent), exist_ok=True)
@@ -352,7 +363,9 @@ def _create_local_volume(source, target):
     @staticmethod
     def _get_volume_mappings(docker_compose_config, external_volumes):
         volume_mappings = []
-        for service in docker_compose_config['services'].values():
+        external_volumes_dict = {}
+
+        for service in docker_compose_config.get('services', {}).values():
             if 'volumes' not in service:
                 continue
 
@@ -364,12 +377,26 @@ def _get_volume_mappings(docker_compose_config, external_volumes):
                     source, target, _ = volume_spec.rsplit(':', 2)
 
                 if source in external_volumes:
+                    external_volumes_dict[source] = target
                     continue
 
                 volume_mapping = (source, target)
 
                 if volume_mapping not in volume_mappings:
                     volume_mappings.append(volume_mapping)
+
+        for volume, volume_config in docker_compose_config.get('volumes', {}).items():
+            if volume_config and volume_config.get('driver') == 'local' and volume_config.get('driver_opts'):
+                driver_opts = volume_config.get('driver_opts')
+                device = driver_opts.get('device')
+                if device and driver_opts.get('o') == 'bind':
+                    source = PurePosixPath(device).joinpath(volume)
+                    target = external_volumes_dict.get(volume)
+                    if source and target:
+                        volume_mapping = (source, target)
+                        if volume_mapping not in volume_mappings:
+                            volume_mappings.append(volume_mapping)
+
         return volume_mappings
 
 

ddb/feature/jsonnet/lib/ddb.docker.libjsonnet

@@ -32,6 +32,9 @@ local _docker_compose_excluded_services = std.extVar("jsonnet.docker.compose.exc
 local _docker_compose_included_services = std.extVar("jsonnet.docker.compose.included_services");
 local _docker_expose_disabled = std.extVar("jsonnet.docker.expose.disabled");
 local _docker_expose_port_prefix = std.extVar("jsonnet.docker.expose.port_prefix");
+local _docker_mount_disabled = std.extVar("jsonnet.docker.mount.disabled");
+local _docker_mount_directory = std.extVar("jsonnet.docker.mount.directory");
+local _docker_mount_directories = std.extVar("jsonnet.docker.mount.directories");
 local _core_project_name = std.extVar("core.project.name");
 local _core_os = std.extVar("core.os");
 local _core_env_current = std.extVar("core.env.current");
@@ -52,6 +55,19 @@ local path = {
     home: mapPath(_core_path_home)
 };
 
+local File(thisFile) =
+	local splitFile = if std.isArray(thisFile) then thisFile else std.split(thisFile, "/");
+
+	local name = splitFile[std.length(splitFile) - 1];
+	if std.length(splitFile) >= 2 then
+		local parent = std.slice(splitFile, 0, std.length(splitFile) - 1, 1);
+		{
+		  name: name,
+		  parent: File(parent)
+		}
+	else
+		{name: name};
+
 local Service(restart=_docker_service_restart, init=_docker_service_init) = {
     [ if restart != null then "restart"]: restart,
     [ if init then "init"]: true,
@@ -149,19 +165,44 @@ local apply_resolve_ports_conflicts(compose) =
 
 local ServiceName(name=null) = std.join("-", std.prune([_core_project_name, name]));
 
-local Volumes(services) = {
-        [key]: {} for key in
-        std.set(
-            std.filter(volume_is_named,
-                std.map(volume_source,
-                    std.flatMap(function (f) if std.objectHas(services[f], "volumes") then services[f].volumes else [],
-                        std.objectFields(services)
-                    )
-                )
-            )
-        )
+local _ensure_absolute(path) =
+	if std.startsWith(path, '/') then
+		path
+	else
+		_core_path_project_home + '/' + path;
+
+local apply_volumes_mounts(volumes) = {
+	[key]: volumes[key] +
+	(if _docker_mount_directories != null && std.objectHas(_docker_mount_directories, key) then {
+		driver: 'local',
+		driver_opts: {type: 'none', o: 'bind', device: _ensure_absolute(_docker_mount_directories[key])}
+     } else if _docker_mount_directory != null then {
+		driver: 'local',
+		driver_opts: {type: 'none', o: 'bind', device: _ensure_absolute(_docker_mount_directory + '/' + key)}
+	}
+	else {})
+	for key in std.objectFields(volumes)
 };
 
+local Volumes(services) =
+	local volumes = {
+			[key]: {} for key in
+			std.set(
+				std.filter(volume_is_named,
+					std.map(volume_source,
+						std.flatMap(function (f) if std.objectHas(services[f], "volumes") then services[f].volumes else [],
+							std.objectFields(services)
+						)
+					)
+				)
+			)
+	};
+
+	if !_docker_mount_disabled then
+		apply_volumes_mounts(volumes)
+	else
+		volumes;
+
 local NoBinaryOptionsLabels(name, options, options_condition = null, index = null) = {};
 local BinaryOptionsLabels(name, options, options_condition = null, index = null) = {
     ["ddb.emit.docker:binary[" + name + "](options)" + (if index != null then "(c" + index + ")" else "")]: options,
@@ -319,19 +360,6 @@ local envIndex(env=_core_env_current) =
 local envIs(env) =
     env == _core_env_current;
 
-local File(thisFile) =
-	local splitFile = if std.isArray(thisFile) then thisFile else std.split(thisFile, "/");
-
-	local name = splitFile[std.length(splitFile) - 1];
-	if std.length(splitFile) >= 2 then
-		local parent = std.slice(splitFile, 0, std.length(splitFile) - 1, 1);
-		{
-		  name: name,
-		  parent: File(parent)
-		}
-	else
-		{name: name};
-
 local with(package, params={}, append=null, name=null, when=true) =
 	if when then
 		local effectiveName = if name == null then package.defaultName else name;

ddb/feature/jsonnet/schema.py

@@ -79,6 +79,15 @@ class ExposeSchema(DisableableSchema):
     port_prefix = fields.Integer(required=False)  # default is set in feature _configure_defaults
 
 
+class MountSchema(DisableableSchema):
+    """
+    Mount schema
+    """
+    directory = fields.String(required=False, allow_none=True, default=None)
+    directories = fields.Dict(required=False, allow_none=True, default=None,
+                              keys=fields.String(), values=fields.String())
+
+
 class UserSchema(DisableableSchema):
     """
     User schema
@@ -119,6 +128,7 @@ class DockerSchema(Schema):
     build = fields.Nested(BuildSchema(), default=BuildSchema())
     service = fields.Nested(ServiceSchema(), default=ServiceSchema())
     expose = fields.Nested(ExposeSchema(), default=ExposeSchema())
+    mount = fields.Nested(MountSchema(), default=MountSchema())
     registry = fields.Nested(RegistrySchema(), default=RegistrySchema())
     user = fields.Nested(UserSchema(), default=UserSchema())
     binary = fields.Nested(BinarySchema(), default=BinarySchema())

docs/features/jsonnet.md

@@ -84,6 +84,14 @@ Run `ddb configure` to evaluate templates and generate target files.
     | `port_prefix` | integer<br>`<based on core.project.name>` | Port prefix. |
 
 
+!!! summary "Docker Mount configuration (prefixed with `jsonnet.docker.mount.`)"
+    | Property | Type | Description |
+    | :---------: | :----: | :----------- |
+    | `disabled` | boolean<br>`False` | Should `ddb.Expose()` perform nothing ? |
+    | `directory` | string | Base directory for all named volume mounts, absolute or relative to project home. |
+    | `directories` | dict[string, string] | Directories for named volume mounts, absolute or relative to project home. key is the volume name, value is the local path to mount. |
+
+
 !!! summary "Docker Registry configuration (prefixed with `jsonnet.docker.registry.`)"
     | Property | Type | Description |
     | :---------: | :----: | :----------- |

tests/feature/jsonnet/test_jsonnet.data/docker_compose_expose/docker-compose.expected.yml

@@ -5,9 +5,9 @@ services:
     init: true
     restart: 'no'
     ports:
-      - '14721:21'
-      - '14722:22/udp'
-      - '14799:23/tcp'
+      - '41621:21'
+      - '41622:22/udp'
+      - '41699:23/tcp'
       - '9912:9912'
 volumes: {}
 version: '3.7'

tests/feature/jsonnet/test_jsonnet.data/docker_compose_mount_single_volume/ddb.yml

@@ -0,0 +1,5 @@
+jsonnet:
+  docker:
+    mount:
+      directories:
+        shared-volume: 'volumes/shared-volume'

tests/feature/jsonnet/test_jsonnet.data/docker_compose_mount_single_volume/docker-compose.expected.yml

@@ -0,0 +1,36 @@
+networks: {}
+services:
+  s1:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - shared-volume:/share
+  s2:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - another-volume:/another
+  s3:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - shared-volume:/share
+  s4:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - another-volume2:/share
+version: '3.7'
+volumes:
+  another-volume: {}
+  shared-volume:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '%ddb.path.project%/volumes/shared-volume'
+  another-volume2: {}

tests/feature/jsonnet/test_jsonnet.data/docker_compose_mount_single_volume/docker-compose.yml.jsonnet

@@ -0,0 +1,25 @@
+local ddb = import 'ddb.docker.libjsonnet';
+
+ddb.Compose({
+	services: {
+    s1:
+      ddb.Image("alpine:3.6") +
+      {
+        volumes+: ['shared-volume:/share'],
+      },
+    s2: ddb.Image("alpine:3.6") +
+      {
+        volumes+: ["another-volume:/another"],
+      },
+    s3:
+      ddb.Image("alpine:3.6") +
+      {
+        volumes+: ['shared-volume:/share']
+      },
+	s4:
+	  ddb.Image("alpine:3.6") +
+	  {
+	    volumes+: ['another-volume2:/share']
+	  },
+    },
+})

tests/feature/jsonnet/test_jsonnet.data/docker_compose_mount_single_volume_with_default/ddb.yml

@@ -0,0 +1,6 @@
+jsonnet:
+  docker:
+    mount:
+      directory: '/default'
+      directories:
+        shared-volume: 'shared-volume'

tests/feature/jsonnet/test_jsonnet.data/docker_compose_mount_single_volume_with_default/docker-compose.expected.yml

@@ -0,0 +1,46 @@
+networks: {}
+services:
+  s1:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - shared-volume:/share
+  s2:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - another-volume:/another
+  s3:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - shared-volume:/share
+  s4:
+    image: alpine:3.6
+    init: true
+    restart: 'no'
+    volumes:
+      - another-volume2:/share
+version: '3.7'
+volumes:
+  another-volume:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '/default/another-volume'
+  shared-volume:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '%ddb.path.project%/shared-volume'
+  another-volume2:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '/default/another-volume2'

tests/feature/jsonnet/test_jsonnet.data/docker_compose_mount_single_volume_with_default/docker-compose.yml.jsonnet

@@ -0,0 +1,25 @@
+local ddb = import 'ddb.docker.libjsonnet';
+
+ddb.Compose({
+	services: {
+    s1:
+      ddb.Image("alpine:3.6") +
+      {
+        volumes+: ['shared-volume:/share'],
+      },
+    s2: ddb.Image("alpine:3.6") +
+      {
+        volumes+: ["another-volume:/another"],
+      },
+    s3:
+      ddb.Image("alpine:3.6") +
+      {
+        volumes+: ['shared-volume:/share']
+      },
+	s4:
+	  ddb.Image("alpine:3.6") +
+	  {
+	    volumes+: ['another-volume2:/share']
+	  },
+    },
+})

tests/feature/jsonnet/test_jsonnet.data/docker_compose_mount_volumes/ddb.yml

@@ -0,0 +1,4 @@
+jsonnet:
+  docker:
+    mount:
+      directory: '/custom'