Build Dockerfile from templates for multi-arch support.

paradrop/daemon/MANIFEST.in

@@ -0,0 +1 @@
+include paradrop/core/container/templates/Dockerfile-*.txt

paradrop/daemon/paradrop/core/container/dockerapi.py

@@ -274,7 +274,10 @@ def _build_image(update, service, client, inline, **buildArgs):
         else:
             # Write it out to a file in the working directory.
             path = os.path.join(buildArgs['path'], "Dockerfile")
-            dockerfile.writeFile(path)
+            try:
+                dockerfile.writeFile(path)
+            except Exception as error:
+                update.progress(str(error))
 
     output = client.build(**buildArgs)
 

paradrop/daemon/paradrop/core/container/dockerfile.py

@@ -2,12 +2,52 @@
 This module generates a Dockerfile for use with light chutes.
 """
 
+import os
 import platform
 import re
 
 from io import BytesIO
 
 
+from paradrop.lib.utils.template import TemplateFormatter
+
+
+# Map requested image names to officially supported images in Docker Hub.
+TARGET_IMAGE_MAP = {
+    "go": "golang:1.11",
+    "gradle": "gradle:4.2",
+    "maven": "maven:3.5",
+    "node": "node:8.13",
+    "python2": "python:2.7",
+    "python3": "python:3.7"
+}
+
+
+# Map machine names to officially supported architecture names in Docker Hub.
+TARGET_MACHINE_MAP = {
+    "armv7l": "armv7",
+    "i486": "i386",
+    "i586": "i386",
+    "i686": "i386",
+    "x86_64": "amd64"
+}
+
+
+def get_target_image(requested):
+    if requested in TARGET_IMAGE_MAP:
+        return TARGET_IMAGE_MAP[requested]
+    else:
+        return requested
+
+
+def get_target_machine():
+    machine = platform.machine()
+    if machine in TARGET_MACHINE_MAP:
+        return TARGET_MACHINE_MAP[machine]
+    else:
+        return machine
+
+
 class Dockerfile(object):
     requiredFields = ["image", "command"]
 
@@ -24,6 +64,16 @@ def getBytesIO(self):
         data = self.getString()
         return BytesIO(data.encode("utf-8"))
 
+    def readTemplate(self, language):
+        dirname = os.path.dirname(__file__)
+        path = os.path.join(dirname, "templates/Dockerfile-{}.txt".format(language))
+
+        if not os.path.isfile(path):
+            raise Exception("No Dockerfile template for {}".format(language))
+
+        with open(path, "r") as source:
+            return source.read()
+
     def getString(self):
         """
         Generate a Dockerfile as a multi-line string.
@@ -35,15 +85,15 @@ def getString(self):
 
         # Extra build options.
         build = self.service.build
-        image_source = build.get("image_source", "paradrop")
-        image_version = build.get("image_version", "latest")
+#        image_source = build.get("image_source", "paradrop")
+#        image_version = build.get("image_version", "latest")
         packages = build.get("packages", [])
 
         as_root = self.service.requests.get("as-root", False)
 
-        # Example base image: paradrop/node-x86_64:latest
-        from_image = "{}/{}-{}:{}".format(image_source, language,
-                platform.machine(), image_version)
+        # Example base image: amd64/node:8.13
+        from_image = "{}/{}".format(get_target_machine(),
+                get_target_image(language))
 
         if isinstance(command, basestring):
             cmd_string = command
@@ -53,21 +103,14 @@ def getString(self):
         else:
             raise Exception("command must be either a string or list of strings")
 
-        dockerfile = "FROM {}\n".format(from_image)
-        if len(packages) > 0:
-            # The base images set up an unprivileged user, paradrop.  We will
-            # need to run as root to install packages, though.
-            dockerfile += "USER root\n"
-            dockerfile += "RUN apt-get update && apt-get install -y {}\n".format(
-                    " ".join(packages))
-            # Drop back to user paradrop after installing packages.
-            if not as_root:
-                dockerfile += "USER paradrop\n"
-        elif as_root:
-            # No packages to install but run as root.
-            dockerfile += "USER root\n"
-
-        dockerfile += "CMD {}\n".format(cmd_string)
+        template = self.readTemplate(language)
+        formatter = TemplateFormatter()
+        dockerfile = formatter.format(template,
+                cmd=cmd_string,
+                drop_root=not as_root,
+                has_packages=len(packages) > 0,
+                image=from_image,
+                packages=" ".join(packages))
 
         return dockerfile
 

paradrop/daemon/paradrop/core/container/templates/Dockerfile-go.txt

@@ -0,0 +1,36 @@
+FROM {image}
+
+# Set up an unprivileged user so that we can drop out of root.
+# Make /home/paradrop so that npm can drop some files in there.
+# Make /opt/paradrop/app for installing the app files.
+RUN useradd --system --uid 999 paradrop && \
+    mkdir -p /home/paradrop && \
+    chown paradrop /home/paradrop && \
+    mkdir -p /opt/paradrop/app && \
+    chown paradrop /opt/paradrop/app && \
+    chmod a+s /opt/paradrop/app
+
+WORKDIR /opt/paradrop/app
+
+# Add cap_net_bind_service to node so that it can bind to ports 1-1024.
+# Not all images support this.
+RUN setcap 'cap_net_bind_service=+ep' /usr/local/bin/node || true
+
+{has_packages:if:RUN apt-get install -y {packages}}
+
+# Defang setuid/setgid binaries.
+RUN find / -perm +6000 -type f -exec chmod a-s {{}} \; || true
+
+ENV GOPATH=/opt/paradrop/go
+
+# Now copy the files.
+COPY . /opt/paradrop/app/
+RUN chown paradrop:paradrop -R /opt/paradrop/app
+
+RUN go get -d && \
+    go build && \
+    cp app /usr/local/bin/
+
+{drop_root:if:USER paradrop}
+
+CMD {cmd}

paradrop/daemon/paradrop/core/container/templates/Dockerfile-gradle.txt

@@ -0,0 +1,38 @@
+FROM {image}
+
+USER root
+
+# Set up an unprivileged user so that we can drop out of root.
+# Make /home/paradrop so that npm can drop some files in there.
+# Make /opt/paradrop/app for installing the app files.
+RUN useradd --system --uid 999 paradrop && \
+    mkdir -p /home/paradrop && \
+    chown paradrop /home/paradrop && \
+    mkdir -p /opt/paradrop/app && \
+    chown paradrop /opt/paradrop/app && \
+    chmod a+s /opt/paradrop/app
+
+WORKDIR /opt/paradrop/app
+
+# Add cap_net_bind_service to node so that it can bind to ports 1-1024.
+# Not all images support this.
+RUN setcap 'cap_net_bind_service=+ep' /usr/local/bin/node || true
+
+{has_packages:if:RUN apt-get install -y {packages}}
+
+# Defang setuid/setgid binaries.
+RUN find / -perm +6000 -type f -exec chmod a-s {{}} \; || true
+
+# The default directory (/root/.m2) is not writable by paradrop user.
+# The logger option makes maven a little less verbose.
+ENV MAVEN_CONFIG=/opt/paradrop/.m2 \
+    MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+
+# Now copy the files.
+COPY . /opt/paradrop/app/
+RUN if [ -f build.gradle ]; then gradle build --no-daemon; fi
+RUN chown paradrop:paradrop -R /opt/paradrop/app
+
+{drop_root:if:USER paradrop}
+
+CMD {cmd}

paradrop/daemon/paradrop/core/container/templates/Dockerfile-maven.txt

@@ -0,0 +1,41 @@
+FROM {image}
+
+# Set up an unprivileged user so that we can drop out of root.
+# Make /home/paradrop so that npm can drop some files in there.
+# Make /opt/paradrop/app for installing the app files.
+RUN useradd --system --uid 999 paradrop && \
+    mkdir -p /home/paradrop && \
+    chown paradrop /home/paradrop && \
+    mkdir -p /opt/paradrop/app && \
+    chown paradrop /opt/paradrop/app && \
+    chmod a+s /opt/paradrop/app
+
+WORKDIR /opt/paradrop/app
+
+# Add cap_net_bind_service to node so that it can bind to ports 1-1024.
+# Not all images support this.
+RUN setcap 'cap_net_bind_service=+ep' /usr/local/bin/node || true
+
+{has_packages:if:RUN apt-get install -y {packages}}
+
+# Defang setuid/setgid binaries.
+RUN find / -perm +6000 -type f -exec chmod a-s {{}} \; || true
+
+# The default directory (/root/.m2) is not writable by paradrop user.
+# The logger option makes maven a little less verbose.
+ENV MAVEN_CONFIG=/opt/paradrop/.m2 \
+    MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dmaven.test.skip=true"
+
+# Copy paradrop.yaml and pom.xml, the latter only if it exists.  These
+# layers will be cached as long as the requirements do not change.
+COPY paradrop.yaml pom.xm[l] /opt/paradrop/app/
+RUN if [ -f pom.xml ]; then mvn --batch-mode dependency:resolve; fi
+
+# Now copy the files.
+COPY . /opt/paradrop/app/
+RUN mvn --batch-mode package
+RUN chown paradrop:paradrop -R /opt/paradrop/app
+
+{drop_root:if:USER paradrop}
+
+CMD {cmd}

paradrop/daemon/paradrop/core/container/templates/Dockerfile-node.txt

@@ -0,0 +1,41 @@
+FROM {image}
+
+ENV npm_config_loglevel=warn
+
+# Set up an unprivileged user so that we can drop out of root.
+# Make /home/paradrop so that npm can drop some files in there.
+# Make /opt/paradrop/app for installing the app files.
+RUN useradd --system --uid 999 paradrop && \
+    mkdir -p /home/paradrop && \
+    chown paradrop /home/paradrop && \
+    mkdir -p /opt/paradrop/app && \
+    chown paradrop /opt/paradrop/app && \
+    chmod a+s /opt/paradrop/app
+
+WORKDIR /opt/paradrop/app
+
+# Add cap_net_bind_service to node so that it can bind to ports 1-1024.
+# Not all images support this.
+RUN setcap 'cap_net_bind_service=+ep' /usr/local/bin/node || true
+
+# Install popular tools here.
+RUN npm install --global gulp-cli
+
+{has_packages:if:RUN apt-get update && apt-get install -y {packages}}
+
+# Defang setuid/setgid binaries.
+RUN find / -perm +6000 -type f -exec chmod a-s {{}} \; || true
+
+# Copy paradrop.yaml and package.json, the latter only if it exists. Then call
+# init-app.sh to install dependencies. These layers will be cached as long as
+# the requirements do not change.
+COPY paradrop.yaml package.jso[n] /opt/paradrop/app/
+RUN npm rebuild && \
+    if [ -f package.json ]; then npm install; fi
+
+COPY . /opt/paradrop/app/
+RUN chown paradrop:paradrop -R /opt/paradrop/app
+
+{drop_root:if:USER paradrop}
+
+CMD {cmd}

paradrop/daemon/paradrop/core/container/templates/Dockerfile-python2.txt

@@ -0,0 +1,40 @@
+FROM {image}
+
+# Set up an unprivileged user so that we can drop out of root.
+# Make /home/paradrop so that npm can drop some files in there.
+# Make /opt/paradrop/app for installing the app files.
+RUN useradd --system --uid 999 paradrop && \
+    mkdir -p /home/paradrop && \
+    chown paradrop /home/paradrop && \
+    mkdir -p /opt/paradrop/app && \
+    chown paradrop /opt/paradrop/app && \
+    chmod a+s /opt/paradrop/app
+
+WORKDIR /opt/paradrop/app
+
+# Add cap_net_bind_service to node so that it can bind to ports 1-1024.
+# Not all images support this.
+RUN setcap 'cap_net_bind_service=+ep' /usr/local/bin/node || true
+
+# Install some useful tools and libraries.
+RUN apt-get update && \
+    apt-get install -y \
+        iptables
+
+{has_packages:if:RUN apt-get install -y {packages}}
+
+# Defang setuid/setgid binaries.
+RUN find / -perm +6000 -type f -exec chmod a-s {{}} \; || true
+
+# Copy paradrop.yaml and requirements.txt, the latter only if it exists.  These
+# layers will be cached as long as the requirements do not change.
+COPY paradrop.yaml requirements.tx[t] /opt/paradrop/app/
+RUN if [ -f requirements.txt ]; then pip install --requirement requirements.txt; fi
+
+# Now copy the rest of the files.
+COPY . /opt/paradrop/app/
+RUN chown paradrop:paradrop -R /opt/paradrop/app
+
+{drop_root:if:USER paradrop}
+
+CMD {cmd}

paradrop/daemon/paradrop/lib/utils/template.py

@@ -0,0 +1,22 @@
+import string
+
+
+class TemplateFormatter(string.Formatter):
+    """
+    String formatter that supports method calls, loops, and conditionals.
+
+    See: https://github.com/ebrehault/superformatter
+    """
+
+    def format_field(self, value, spec):
+        if spec.startswith('repeat'):
+            template = spec.partition(':')[-1]
+            if type(value) is dict:
+                value = value.items()
+            return ''.join([template.format(item=item) for item in value])
+        elif spec == 'call':
+            return value()
+        elif spec.startswith('if'):
+            return (value and spec.partition(':')[-1]) or ''
+        else:
+            return super(TemplateFormatter, self).format_field(value, spec)

tests/paradrop/core/container/test_dockerfile.py

@@ -2,6 +2,21 @@
 from paradrop.core.chute.service import Service
 
 
+def test_get_target_image():
+    from paradrop.core.container import dockerfile
+    assert dockerfile.get_target_image("go").startswith("golang:")
+    assert dockerfile.get_target_image("node").startswith("node:")
+    assert dockerfile.get_target_image("python2").startswith("python:2")
+    assert dockerfile.get_target_image("python3").startswith("python:3")
+    assert dockerfile.get_target_image("xxx").startswith("xxx:")
+
+
+def test_get_target_machine():
+    from paradrop.core.container import dockerfile
+    result = dockerfile.get_target_machine()
+    assert isinstance(result, basestring) and len(result) > 0
+
+
 def test_getString():
     service = Service(image="python2", command="python")
     dockerfile = Dockerfile(service)

tools/pdtools/pdtools/helpers/chute.py

@@ -110,7 +110,7 @@ def get_command(name="my-app", use=None):
     elif use == "gradle":
         print("For Java applications, you can accept the default and update")
         print("paradrop.yaml after you have decided on the structure of your project.")
-        default = "java -cp build/libs/gradle-{name}-1.0-SNAPSHOT.jar com.mycompany.{name}.Main".format(name=name)
+        default = "java -jar build/libs/{name}-0.1.0.jar".format(name=name)
     elif use == "maven":
         print("For Java applications, you can accept the default and update")
         print("paradrop.yaml after you have decided on the structure of your project.")