Support multi-platform deployment manifest template (#358)

.env.tmp

@@ -54,11 +54,13 @@ CONTAINER_TAG=""
 #
 
 CONFIG_OUTPUT_DIR="config"
-DEPLOYMENT_CONFIG_FILE="deployment.json"
 DEPLOYMENT_CONFIG_TEMPLATE_FILE="deployment.template.json"
-LOGS_PATH="logs"
+DEPLOYMENT_CONFIG_DEBUG_TEMPLATE_FILE="deployment.debug.template.json"
+DEFAULT_PLATFORM="amd64"
 MODULES_PATH="modules"
 
+LOGS_PATH="logs"
+
 #
 # DOCKER LOGS COMMAND
 #

iotedgedev/azurecli.py

@@ -11,6 +11,10 @@
 from six.moves.queue import Empty, Queue
 
 from . import telemetry
+from .compat import PY2
+
+if PY2:
+    from .compat import FileNotFoundError
 
 output_io_cls = StringIO
 

iotedgedev/cli.py

@@ -112,7 +112,6 @@ def init(module, template, group_id):
 @with_telemetry
 def e2e(ctx):
     ctx.invoke(init)
-    envvars.load(force=True)
     ctx.invoke(push)
     ctx.invoke(deploy)
     ctx.invoke(monitor)
@@ -147,11 +146,24 @@ def add(name, template, group_id):
               required=False,
               is_flag=True,
               help="Deploy modules to Edge device using deployment.json in the config folder")
+@click.option("--file",
+              "-f",
+              "template_file",
+              default=envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE,
+              show_default=True,
+              required=False,
+              help="Specify the deployment manifest template file")
+@click.option("--platform",
+              "-P",
+              default=envvars.DEFAULT_PLATFORM,
+              show_default=True,
+              required=False,
+              help="Specify the platform")
 @click.pass_context
 @with_telemetry
-def build(ctx, push, do_deploy):
+def build(ctx, push, do_deploy, template_file, platform):
     mod = Modules(envvars, output)
-    mod.build_push(no_push=not push)
+    mod.build_push(template_file, platform, no_push=not push)
 
     if do_deploy:
         ctx.invoke(deploy)
@@ -174,12 +186,25 @@ def build(ctx, push, do_deploy):
               show_default=True,
               required=False,
               is_flag=True,
-              help="Inform the push command to not build modules images before pushing to container registry")
+              help="Inform the push command to not build module images before pushing to container registry")
+@click.option("--file",
+              "-f",
+              "template_file",
+              default=envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE,
+              show_default=True,
+              required=False,
+              help="Specify the deployment manifest template file")
+@click.option("--platform",
+              "-P",
+              default=envvars.DEFAULT_PLATFORM,
+              show_default=True,
+              required=False,
+              help="Specify the platform")
 @click.pass_context
 @with_telemetry
-def push(ctx, do_deploy, no_build):
+def push(ctx, do_deploy, no_build, template_file, platform):
     mod = Modules(envvars, output)
-    mod.push(no_build=no_build)
+    mod.push(template_file, platform, no_build=no_build)
 
     if do_deploy:
         ctx.invoke(deploy)
@@ -189,23 +214,43 @@ def push(ctx, do_deploy, no_build):
 
 
 @solution.command(context_settings=CONTEXT_SETTINGS, help="Deploy solution to IoT Edge device")
+@click.option("--file",
+              "-f",
+              "manifest_file",
+              default=envvars.DEPLOYMENT_CONFIG_FILE_PATH,
+              show_default=True,
+              required=False,
+              help="Specify the deployment manifest file")
 @with_telemetry
-def deploy():
+def deploy(manifest_file):
     edge = Edge(envvars, output, azure_cli)
-    edge.deploy()
+    edge.deploy(manifest_file)
 
 
 main.add_command(deploy)
 
 
 @solution.command(context_settings=CONTEXT_SETTINGS,
-                  help="Expand environment variables and placeholders in *.template.json and copy to config folder",
+                  help="Expand environment variables and placeholders in deployment manifest template file and copy to config folder",
                   # hack to prevent Click truncating help messages
-                  short_help="Expand environment variables and placeholders in *.template.json and copy to config folder")
+                  short_help="Expand environment variables and placeholders in deployment manifest template file and copy to config folder")
+@click.option("--file",
+              "-f",
+              "template_file",
+              default=envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE,
+              show_default=True,
+              required=False,
+              help="Specify the deployment manifest template file")
+@click.option("--platform",
+              "-P",
+              default=envvars.DEFAULT_PLATFORM,
+              show_default=True,
+              required=False,
+              help="Specify the platform")
 @with_telemetry
-def genconfig():
+def genconfig(template_file, platform):
     mod = Modules(envvars, output)
-    mod.build_push(no_build=True, no_push=True)
+    mod.build_push(template_file, platform, no_build=True, no_push=True)
 
 
 main.add_command(genconfig)
@@ -261,6 +306,19 @@ def setup_simulator(gateway_host):
               default=False,
               show_default=True,
               help="Build the solution before starting IoT Edge simulator in solution mode.")
+@click.option("--file",
+              "-f",
+              "manifest_file",
+              default=envvars.DEPLOYMENT_CONFIG_FILE_PATH,
+              show_default=True,
+              required=False,
+              help="Specify the deployment manifest file. When `--build` flag is set, specify a deployment manifest template and it will be built.")
+@click.option("--platform",
+              "-P",
+              default=envvars.DEFAULT_PLATFORM,
+              show_default=True,
+              required=False,
+              help="Specify the platform")
 @click.option("--inputs",
               "-i",
               required=False,
@@ -273,14 +331,14 @@ def setup_simulator(gateway_host):
               show_default=True,
               help="Port of the service for sending message.")
 @with_telemetry
-def start_simulator(setup, solution, build, verbose, inputs, port):
+def start_simulator(setup, solution, build, manifest_file, platform, verbose, inputs, port):
     sim = Simulator(envvars, output)
 
     if setup:
         sim.setup(socket.getfqdn())
 
     if solution or not inputs:
-        sim.start_solution(verbose, build)
+        sim.start_solution(manifest_file, platform, verbose, build)
     else:
         sim.start_single(inputs, port)
 

iotedgedev/constants.py

@@ -0,0 +1,8 @@
+class Constants:
+    default_config_folder = "config"
+    default_modules_folder = "modules"
+    default_deployment_template_file = "deployment.template.json"
+    default_deployment_debug_template_file = "deployment.debug.template.json"
+    default_platform = "amd64"
+    deployment_template_suffix = ".template.json"
+    deployment_template_schema_version = "1.0.0"

iotedgedev/deploymentmanifest.py

@@ -11,6 +11,7 @@
 import six
 
 from .compat import PY2
+from .utility import Utility
 
 if PY2:
     from .compat import FileNotFoundError
@@ -21,41 +22,42 @@
 
 class DeploymentManifest:
     def __init__(self, envvars, output, utility, path, is_template):
+        self.envvars = envvars
         self.utility = utility
         self.output = output
         try:
             self.path = path
             self.is_template = is_template
-            self.json = json.loads(self.utility.get_file_contents(path, expandvars=True))
+            self.json = json.loads(Utility.get_file_contents(path, expandvars=True))
         except FileNotFoundError:
             if is_template:
-                deployment_manifest_path = envvars.DEPLOYMENT_CONFIG_FILE_PATH
+                deployment_manifest_path = self.envvars.DEPLOYMENT_CONFIG_FILE_PATH
                 if os.path.exists(deployment_manifest_path):
                     self.output.error('Deployment manifest template file "{0}" not found'.format(path))
                     if output.confirm('Would you like to make a copy of the deployment manifest file "{0}" as the deployment template file?'.format(deployment_manifest_path), default=True):
                         shutil.copyfile(deployment_manifest_path, path)
-                        self.json = json.load(open(envvars.DEPLOYMENT_CONFIG_FILE_PATH))
-                        envvars.save_envvar("DEPLOYMENT_CONFIG_TEMPLATE_FILE", path)
+                        self.json = json.load(open(self.envvars.DEPLOYMENT_CONFIG_FILE_PATH))
+                        self.envvars.save_envvar("DEPLOYMENT_CONFIG_TEMPLATE_FILE", path)
                 else:
                     raise FileNotFoundError('Deployment manifest file "{0}" not found'.format(path))
             else:
                 raise FileNotFoundError('Deployment manifest file "{0}" not found'.format(path))
 
-    def add_module_template(self, module_name):
-        """Add a module template to the deployment manifest with amd64 as the default platform"""
-        new_module = """{
+    def add_module_template(self, module_name, create_options={}, is_debug=False):
+        """Add a module template to the deployment manifest"""
+        new_module = {
             "version": "1.0",
             "type": "docker",
             "status": "running",
             "restartPolicy": "always",
             "settings": {
-              "image": \"${MODULES.""" + module_name + """.amd64}\",
-              "createOptions": {}
+                "image": DeploymentManifest.get_image_placeholder(module_name, is_debug),
+                "createOptions": create_options
             }
-        }"""
+        }
 
         try:
-            self.utility.nested_set(self._get_module_content(), ["$edgeAgent", "properties.desired", "modules", module_name], json.loads(new_module))
+            self.utility.nested_set(self._get_module_content(), ["$edgeAgent", "properties.desired", "modules", module_name], new_module)
         except KeyError as err:
             raise KeyError("Missing key {0} in file {1}".format(err, self.path))
 
@@ -95,6 +97,9 @@ def get_all_modules(self):
     def get_desired_property(self, module, prop):
         return self._get_module_content()[module]["properties.desired"][prop]
 
+    def get_template_schema_ver(self):
+        return self.json.get("$schema-template", "")
+
     def convert_create_options(self):
         modules = self.get_all_modules()
         for module_name, module_info in modules.items():
@@ -120,6 +125,9 @@ def expand_image_placeholders(self, replacements):
             if module_name in replacements:
                 self.utility.nested_set(module_info, ["settings", "image"], replacements[module_name])
 
+    def del_key(self, keys):
+        self.utility.del_key(self.json, keys)
+
     def dump(self, path=None):
         """Dump the JSON to the disk"""
         if path is None:
@@ -128,6 +136,10 @@ def dump(self, path=None):
         with open(path, "w") as deployment_manifest:
             json.dump(self.json, deployment_manifest, indent=2)
 
+    @staticmethod
+    def get_image_placeholder(module_name, is_debug=False):
+        return "${{MODULES.{0}}}".format(module_name + ".debug" if is_debug else module_name)
+
     def _get_module_content(self):
         if "modulesContent" in self.json:
             return self.json["modulesContent"]

iotedgedev/dockercls.py

@@ -5,6 +5,7 @@
 import six
 
 from .deploymentmanifest import DeploymentManifest
+from .utility import Utility
 
 
 class Docker:
@@ -14,12 +15,16 @@ def __init__(self, envvars, utility, output):
         self.utility = utility
         self.output = output
 
-        if self.envvars.DOCKER_HOST:
-            self.docker_client = docker.DockerClient(base_url=self.envvars.DOCKER_HOST)
-            self.docker_api = docker.APIClient(base_url=self.envvars.DOCKER_HOST)
-        else:
-            self.docker_client = docker.from_env()
-            self.docker_api = docker.APIClient()
+        try:
+            if self.envvars.DOCKER_HOST:
+                self.docker_client = docker.DockerClient(base_url=self.envvars.DOCKER_HOST)
+                self.docker_api = docker.APIClient(base_url=self.envvars.DOCKER_HOST)
+            else:
+                self.docker_client = docker.from_env(version="auto")
+                self.docker_api = docker.APIClient()
+        except Exception as ex:
+            msg = "Could not connect to Docker daemon. Please make sure Docker daemon is running and accessible"
+            raise ValueError(msg, ex)
 
     def get_os_type(self):
         return self.docker_client.info()["OSType"].lower()
@@ -127,7 +132,7 @@ def setup_registry_in_config(self, image_names):
 
         # Replace mcr.microsoft.com/ with ${CONTAINER_REGISTRY_SERVER}
         for config_file in self.utility.get_config_files():
-            config_file_contents = self.utility.get_file_contents(config_file)
+            config_file_contents = Utility.get_file_contents(config_file)
             for image_name in image_names:
                 config_file_contents = config_file_contents.replace(
                     "mcr.microsoft.com/" + image_name, "${CONTAINER_REGISTRY_SERVER}/" + image_name)

iotedgedev/edge.py

@@ -4,13 +4,13 @@ def __init__(self, envvars, output, azure_cli):
         self.output = output
         self.azure_cli = azure_cli
 
-    def deploy(self):
+    def deploy(self, manifest_file):
 
         self.output.header("DEPLOYING CONFIGURATION")
 
         self.envvars.verify_envvar_has_val("IOTHUB_CONNECTION_STRING", self.envvars.IOTHUB_CONNECTION_INFO)
         self.envvars.verify_envvar_has_val("DEVICE_CONNECTION_STRING", self.envvars.DEVICE_CONNECTION_INFO)
         self.envvars.verify_envvar_has_val("DEPLOYMENT_CONFIG_FILE", self.envvars.DEPLOYMENT_CONFIG_FILE)
 
-        if self.azure_cli.set_modules(self.envvars.DEVICE_CONNECTION_INFO.device_id, self.envvars.IOTHUB_CONNECTION_INFO, self.envvars.DEPLOYMENT_CONFIG_FILE_PATH):
+        if self.azure_cli.set_modules(self.envvars.DEVICE_CONNECTION_INFO.device_id, self.envvars.IOTHUB_CONNECTION_INFO, manifest_file):
             self.output.footer("DEPLOYMENT COMPLETE")