Add "mesh deployment create" command (#146)

.gitignore

@@ -108,5 +108,8 @@ ENV/
 # OS Files
 .DS_Store
 
-# Ignore recordings to keep tests local
-recordings
+# Ignore "recordings" folder, which is generated by tests using the vcr module
+recordings
+
+# Ignore "meshDeploy" folder as this is used to store the generated JSON files for resource deployment
+meshDeploy

CODEOWNERS

@@ -50,3 +50,14 @@ src/sfctl/helps/cluster_upgrade.py @jkochhar
 
 # Version
 src/sfctl/custom_version.py @jeffj6123
+
+# Mesh Deployment
+src/sfctl/custom_deployment.py @rapatchi @ashank
+src/sfctl/helps/deployment.py @rapatchi @ashank
+src/sfctl/tests/mesh_test.py @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_app.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_gateway.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_network.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_secret.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_secret_value.yaml @rapatchi @ashank
+src/sfctl/tests/sample_yaml/sample_volume.yaml @rapatchi @ashank

pylintrc

@@ -460,7 +460,7 @@ max-bool-expr=5
 max-branches=12
 
 # Maximum number of locals for function / method body
-max-locals=15
+max-locals=20
 
 # Maximum number of parents for a class (see R0901).
 max-parents=7

src/README.rst

@@ -25,6 +25,7 @@ Unreleased
 - Add Mesh network, gateway, code package, secret, and secretvalue commands (#141)
 - Allow any Python 3.7.x versions rather than only 3.7.0 (#142)
 - Fix missing option of "Error" health state in health reporting (#151)
+- Add the command "sfctl mesh deployment create", which takes resource description yaml files as input and deploys the corresponding mesh resources (#146)
 
 6.0.1
 -----

src/setup.py

@@ -51,7 +51,8 @@ def read(fname):
         'azure-servicefabric==6.3.0.0',
         'jsonpickle',
         'adal',
-        'future'
+        'future',
+        'sfmergeutility'
     ],
     extras_require={
         'test': [

src/sfctl/commands.py

@@ -30,6 +30,8 @@
 import sfctl.helps.chaos  # pylint: disable=unused-import
 import sfctl.helps.infrastructure  # pylint: disable=unused-import
 import sfctl.helps.secretvalue  # pylint: disable=unused-import
+import sfctl.helps.deployment # pylint: disable=unused-import
+
 EXCLUDED_PARAMS = ['self', 'raw', 'custom_headers', 'operation_config',
                    'content_version', 'kwargs', 'client']
 
@@ -400,6 +402,11 @@ def load_command_table(self, args):  # pylint: disable=too-many-statements
                           client_factory=mesh_secret_value_create) as group:
             group.command('show', 'get_secret_value')
 
+        client_func_path_mesh = 'sfctl.custom_deployment#{}'
+        with CommandGroup(self, 'mesh deployment', client_func_path_mesh,
+                          client_factory=client_create) as group:
+            group.command('create', 'mesh_deploy')
+
         return OrderedDict(self.command_table)
 
     def load_arguments(self, command):

src/sfctl/custom_deployment.py

@@ -0,0 +1,77 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+"""Custom commands for managing Service Fabric Mesh resources"""
+
+from __future__ import print_function
+
+import os
+import shutil
+from knack.util import CLIError
+from sfmergeutility import SFMergeUtility
+from sfmergeutility.utility import ResourceType, get_resource_name, get_resource_type, list_files_in_directory, load_json # pylint: disable=line-too-long
+
+def deploy_resource(client, resource):
+    """ Deploys the specified resource to the connected cluster
+    :param client: (class) Auto generated client from swagger specification
+    :param resource: (str) Relative/absolute path of the resource file
+    """
+    resource_type = get_resource_type(resource)
+    resource_name = get_resource_name(resource)
+
+    print('Creating resource: ', resource_name, 'of type: ', resource_type.name)
+
+    if resource_type == ResourceType.application:
+        application_description = load_json(resource)
+        client.mesh_application.create_or_update(resource_name, application_description.get('description')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.volume:
+        volume_description = load_json(resource)
+        client.mesh_volume.create_or_update(resource_name, volume_description.get('description')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.network:
+        network_description = load_json(resource)
+        client.mesh_network.create_or_update(resource_name, network_description.get('description').get('name'), network_description.get('description').get('properties')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.secret:
+        secret_description = load_json(resource)
+        client.mesh_secret.create_or_update(resource_name, secret_description.get('description').get('properties'), secret_description.get('description').get('name')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.secretValue:
+        secret_value_description = load_json(resource)
+        fully_qualified_resource_name = secret_value_description.get('fullyQualifiedResourceName').split('/') # pylint: disable=line-too-long
+        secret_value_resource_name = fully_qualified_resource_name[1]
+        client.mesh_secret_value.add_value(resource_name, secret_value_resource_name, secret_value_description.get('description').get('name'), secret_value_description.get('description').get('properties').get('value')) # pylint: disable=line-too-long
+    elif resource_type == ResourceType.gateway:
+        gateway_description = load_json(resource)
+        client.mesh_gateway.create_or_update(resource_name, gateway_description.get('description')) # pylint: disable=line-too-long
+
+def mesh_deploy(client, input_yaml_file_paths, parameters=None):
+    """ This function
+        1. Uses sfmergeutility to merge, convert, and
+        order the resources
+        2. Deploys the resources in the order suggested by the utility
+    :param client: (class) Auto generated client from swagger specification
+    :param input_yaml_file_paths: (str) Relative/absolute directory path or comma seperated relative/absolute file paths of the yaml resource files  # pylint: disable=line-too-long
+    """
+    file_path_list = []
+
+    if os.path.isdir(input_yaml_file_paths):
+        if not os.path.exists(input_yaml_file_paths):
+            raise CLIError('The specified directory "%s" does not exist or you do not have access to it' %(input_yaml_file_paths)) # pylint: disable=line-too-long
+        file_path_list = list_files_in_directory(input_yaml_file_paths, ".yaml")
+
+    else:
+        file_path_list = input_yaml_file_paths.split(',')
+        for file_path in file_path_list:
+            if not os.path.exists(file_path):
+                raise CLIError('The specified file "%s" does not exist or you do not have access to it' %(file_path)) # pylint: disable=line-too-long
+
+    output_dir = os.path.join(os.getcwd(), "meshDeploy")
+    if os.path.exists(output_dir):
+        shutil.rmtree(output_dir, ignore_errors=True)
+
+    SFMergeUtility.sf_merge_utility(file_path_list, "SF_SBZ_JSON", parameter_file=parameters, output_dir=output_dir, prefix="") # pylint: disable=line-too-long
+    resources = list_files_in_directory(output_dir, ".json")
+    resources.sort()
+    for resource in resources:
+        deploy_resource(client, resource)

src/sfctl/helps/deployment.py

@@ -0,0 +1,26 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+"""Help documentation for managing Service Fabric Mesh Resources."""
+
+from knack.help_files import helps
+
+helps['mesh deployment create'] = """
+    type: command
+    short-summary: Creates a deployment of Service Fabric Mesh Resources
+    parameters:
+        - name: --input-yaml-file-paths
+          type: string
+          short-summary: Comma separated relative/absolute file paths of all the yaml files or relative/absolute path of the directory (recursive) which contain yaml files
+        - name: --parameters
+          type: string
+          short-summary: A relative/absolute to json file which contains the parameters that need to be overridden
+    examples:
+        - name: Consolidates and deploys all the resources to cluster by overriding the parameters mentioned in the param.json file
+          text: sfctl mesh deployment create --input-yaml-file-paths ./app.yaml,./network.yaml --parameters ./param.json
+        - name: Consolidates and deploys all the resources in a directory to cluster by overriding the parameters mentioned in the param.json file
+          text: sfctl mesh deployment create --input-yaml-file-paths ./resources --parameters ./param.json
+"""

src/sfctl/helps/main.py

@@ -29,7 +29,7 @@
 
 helps['application'] = """
     type: group
-    short-summary: Create, delete, and manage applications and application types.
+    short-summary: Create, delete, and manage applications and application types
 """
 
 helps['chaos'] = """
@@ -104,6 +104,11 @@
     short-summary: Get and delete application resources
 """
 
+helps['mesh deployment'] = """
+    type: group
+    short-summary: Create Service Fabric Mesh resources
+"""
+
 helps['mesh service'] = """
     type: group
     short-summary: Get service details and list services of an application resource

src/sfctl/params.py

@@ -209,8 +209,3 @@ def custom_arguments(self, _):  # pylint: disable=too-many-statements
         # expect the parameter command_input in the python method as --command in commandline.
         arg_context.argument('command_input',
                              CLIArgumentType(options_list='--command'))
-
-    with ArgumentsContext(self, 'mesh secretvalue show') as arg_context:
-        arg_context.argument('secret_resource_name')
-        arg_context.argument('secret_value_resource_name')
-        arg_context.argument('show_value')

src/sfctl/tests/custom_help_text_correctness_test.py

@@ -224,7 +224,7 @@ def test_custom_help_text(self):
             print()
             print(line)
 
-        allowable_lines_not_found = 75
+        allowable_lines_not_found = 78
 
         print()
         print('The total number of lines compared is ' + str(len(custom_help_lines)))

src/sfctl/tests/help_text_test.py

@@ -312,16 +312,21 @@ def test_help_documentation(self):
             commands=('delete', 'list', 'show'))
 
         self.validate_output(
-            'sfctl mesh volume',
-            commands=('delete', 'list', 'show'))
+            'sfctl mesh deployment',
+            commands=('create',))
 
         self.validate_output(
             'sfctl mesh service',
             commands=('list', 'show'))
+
         self.validate_output(
             'sfctl mesh service-replica',
             commands=('list', 'show'))
 
+        self.validate_output(
+            'sfctl mesh volume',
+            commands=('delete', 'list', 'show'))
+
         self.validate_output(
             'sfctl node',
             commands=('disable', 'enable', 'health', 'info', 'list', 'load', 'remove-state',

src/sfctl/tests/mesh_test.py

@@ -0,0 +1,21 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+"""Custom mesh command tests"""
+
+import unittest
+from knack.util import CLIError
+import sfctl.custom_deployment as sf_resource
+
+class MeshTests(unittest.TestCase):
+    """Mesh command tests """
+
+    def test_mesh_deploy_invalid(self):
+        """ Test to check if mesh deployment command fails when invalid path is provided"""
+        with self.assertRaises(CLIError):
+            sf_resource.mesh_deploy(None, "some-dummy-file-path")
+        with self.assertRaises(CLIError):
+            sf_resource.mesh_deploy(None, "some-dummy-file-path,another-dummy-file-path")

src/sfctl/tests/request_generation_test.py

@@ -218,12 +218,13 @@ def test_paths_generation(self):
         # Call test
         self.paths_generation_helper()
 
-    def paths_generation_helper(self):  # pylint: disable=too-many-statements
+    def paths_generation_helper(self):  # pylint: disable=too-many-statements, too-many-locals
         """ Lists all the commands to be tested and their expected values.
         Expected values here refer to the expected URI that is generated
         and sent to the cluster."""
 
         sample_path_base = '@' + path.join(path.dirname(__file__), 'sample_json')
+        sample_yaml_base = '@' + path.join(path.dirname(__file__), 'sample_yaml')
 
         # The commands which don't affect or query the cluster
         # Specifically, cluster select and show-connection
@@ -1108,3 +1109,52 @@ def paths_generation_helper(self):  # pylint: disable=too-many-statements
             '/Resources/Secrets/some~secret~resource~name/values/secret~value~name/list_value',
             ['api-version=6.4-preview']
         )
+
+        sample_network = path.join(sample_yaml_base, 'sample_network.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_secret = path.join(sample_yaml_base, 'sample_secret.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_secret_value = path.join(sample_yaml_base, 'sample_secret_value.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_volume = path.join(sample_yaml_base, 'sample_volume.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_gateway = path.join(sample_yaml_base, 'sample_gateway.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+        sample_app = path.join(sample_yaml_base, 'sample_app.yaml').replace('/', '//').replace('\\', '\\\\').replace('@', '')
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_network),
+            'PUT',
+            '/Resources/Networks/someNetwork',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_secret),
+            'PUT',
+            '/Resources/Secrets/someSecret',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_secret_value),
+            'PUT',
+            '/Resources/Secrets/someSecret/values/v1',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_volume),
+            'PUT',
+            '/Resources/Volumes/someVolume',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_gateway),
+            'PUT',
+            '/Resources/Gateways/someGateway',
+            ['api-version=6.4-preview']
+        )
+
+        self.validate_command(
+            'mesh deployment create --input-yaml-file-paths {0}'.format(sample_app),
+            'PUT',
+            '/Resources/Applications/someApp',
+            ['api-version=6.4-preview']
+        )

src/sfctl/tests/sample_yaml/sample_app.yaml

@@ -0,0 +1,25 @@
+application:
+  schemaVersion: 1.0.0-preview2
+  name: someApp
+  properties:
+    description: someApp description.
+    services:
+    - name: someService
+      properties:
+        description: someService description.
+        osType: Windows
+        codePackages:
+        - name: someServiceCode
+          image: someImage
+          endpoints:
+          - name: someServiceListener
+            port: 80
+          resources:
+            requests:
+              cpu: 0.5
+              memoryInGB: 1
+        replicaCount: 1
+        networkRefs:
+        - name: someNetwork
+          endpointRefs:
+          - name: someServiceListener

src/sfctl/tests/sample_yaml/sample_gateway.yaml

@@ -0,0 +1,16 @@
+gateway:
+  schemaVersion: 1.0.0-preview2
+  name: someGateway
+  properties:
+    description: Gateway to connect to public internet.
+    sourceNetwork:
+        name: Open
+    destinationNetwork:
+        name: someNetwork
+    tcp:
+      - name: PublicConnection
+        port: 80
+        destination:
+            applicationName: someApp
+            serviceName: someService
+            endpointName: someListener

src/sfctl/tests/sample_yaml/sample_network.yaml

@@ -0,0 +1,7 @@
+network:
+  schemaVersion: 1.0.0-preview2
+  name: someNetwork
+  properties:
+    kind: Local
+    description: someNetwork description.
+    networkAddressPrefix: 8.0.0.0/16

src/sfctl/tests/sample_yaml/sample_secret.yaml

@@ -0,0 +1,7 @@
+secret:
+  schemaVersion: 1.0.0-preview2
+  name: someSecret
+  properties:
+    kind: inlinedValue
+    description: Account key for azure files
+    contentType: text/plain