From 3501653113418932a1bff38d0d008cefa5fc02aa Mon Sep 17 00:00:00 2001 From: Ravi Mukkamala Date: Mon, 30 Nov 2020 18:51:29 -0800 Subject: [PATCH 1/3] Fixes #53 Rename name to plugin_id and prettyName to name --- .../dlpx/virtualization/_internal/cli.py | 9 +-- .../_internal/commands/build.py | 18 ++--- .../_internal/delphix_client.py | 2 +- .../virtualization/_internal/plugin_util.py | 14 ++-- ...plugin_config_schema_no_id_validation.json | 67 ------------------- .../_internal/commands/test_build.py | 24 ++----- .../_internal/commands/test_delphix_client.py | 1 + .../dlpx/virtualization/_internal/conftest.py | 4 +- .../dlpx/virtualization/_internal/test_cli.py | 22 ------ 9 files changed, 18 insertions(+), 143 deletions(-) delete mode 100644 tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json diff --git a/tools/src/main/python/dlpx/virtualization/_internal/cli.py b/tools/src/main/python/dlpx/virtualization/_internal/cli.py index 227c3ff3..7d785591 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/cli.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/cli.py @@ -188,18 +188,12 @@ def init(root, ingestion_strategy, name, host_type): mutually_exclusive=['upload_artifact'], help=('Only generate the Python classes from the schema definitions. ' 'Do not do a full build or create an upload artifact.')) -@click.option('--skip-id-validation', - is_flag=True, - hidden=True, - help=('An internal flag that does not enforce the format ' - 'of the id. Use of this flag is unsupported.')) @click.option('--dev', is_flag=True, hidden=True, help=('An internal flag that installs dev builds of the ' 'wrappers. This should only be used by SDK developers.')) -def build(plugin_config, upload_artifact, generate_only, skip_id_validation, - dev): +def build(plugin_config, upload_artifact, generate_only, dev): """ Build the plugin code and generate upload artifact file using the configuration provided in the plugin config file. @@ -225,7 +219,6 @@ def build(plugin_config, upload_artifact, generate_only, skip_id_validation, build_internal.build(plugin_config, upload_artifact, generate_only, - skip_id_validation, local_vsdk_root=local_vsdk_root) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index b0dbdcc6..5a1fec2a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -31,7 +31,6 @@ def build(plugin_config, upload_artifact, generate_only, - skip_id_validation, local_vsdk_root=None): """This builds the plugin using the configurations provided in config yaml file provided as input. It reads schemas and source code from the files @@ -42,7 +41,6 @@ def build(plugin_config, plugin_config: Plugin config file used for building plugin. upload_artifact: The file to which output of build is written to. generate_only: Only generate python classes from schema definitions. - skip_id_validation: Skip validation of the plugin id. local_vsdk_root: The local path to the root of the Virtualization SDK repository. """ @@ -57,7 +55,7 @@ def build(plugin_config, logger.info('Validating plugin config file %s', plugin_config) try: result = plugin_util.validate_plugin_config_file( - plugin_config, not generate_only, skip_id_validation) + plugin_config, not generate_only) except exceptions.UserError as err: raise exceptions.BuildFailedError(err) @@ -111,8 +109,7 @@ def build(plugin_config, try: result = plugin_util.get_plugin_manifest(plugin_config, plugin_config_content, - not generate_only, - skip_id_validation) + not generate_only) except (exceptions.UserError, exceptions.SDKToolingError) as err: raise exceptions.BuildFailedError(err) @@ -164,16 +161,9 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): # Hard code the type to a set default. 'type': TYPE, - # - # Delphix Engine still accepts only name and prettyName and - # hence name is mapped to id and prettyName to name. - # Delphix Engine does not accept upper case letters for name field, - # so we convert the name to lowercase letters. - # This will be changed as part of POST GA task PYT-628 - # + 'pluginId': + plugin_config_content['id'], 'name': - plugin_config_content['id'].lower(), - 'prettyName': plugin_config_content['name'], # set default value of locale to en-us 'defaultLocale': diff --git a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py index 2ab521fa..03b5799e 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py @@ -188,7 +188,7 @@ def __get_plugin_ref_from_id(self, plugin_name, plugin_id): # and make sure the plugin hasn't been replicated. # The 'name' field will be converted to 'id' in the future. # - if p['name'] == plugin_id and p['namespace'] is None: + if p['identifier'] == plugin_id and p['namespace'] is None: return p['reference'] raise exceptions.MissingPluginError(plugin_name, self.__engine) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py index 1a210423..ca87f383 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py @@ -43,8 +43,7 @@ def validate_error_handler(plugin_file, validation_mode): def validate_plugin_config_file(plugin_config, - stop_build, - skip_id_validation=False): + stop_build): """ Reads a plugin config file and validates the contents using a pre-defined schema. If stop_build is True, will report exception @@ -55,10 +54,8 @@ def validate_plugin_config_file(plugin_config, """ validation_mode = (ValidationMode.ERROR if stop_build else ValidationMode.WARNING) - plugin_config_schema_file = (const.PLUGIN_CONFIG_SCHEMA_NO_ID_VALIDATION - if skip_id_validation else - const.PLUGIN_CONFIG_SCHEMA) - validator = PluginValidator(plugin_config, plugin_config_schema_file) + + validator = PluginValidator(plugin_config, const.PLUGIN_CONFIG_SCHEMA) with validate_error_handler(plugin_config, validation_mode): validator.validate_plugin_config() @@ -68,8 +65,7 @@ def validate_plugin_config_file(plugin_config, def get_plugin_manifest(plugin_config_file, plugin_config_content, - stop_build, - skip_id_validation=False): + stop_build): """ Validates the given plugin config content using a pre-defined schema. Plugin config file name is used to get the absolute path of plugin source @@ -115,7 +111,7 @@ def get_plugin_config_property(plugin_config_path, prop): """ Returns the value for a specific property from the plugin config file. """ - result = validate_plugin_config_file(plugin_config_path, False, False) + result = validate_plugin_config_file(plugin_config_path, False) return result.plugin_config_content[prop] diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json deleted file mode 100644 index 32eb611f..00000000 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "externalVersion": { - "type": "string" - }, - "hostTypes": { - "type": "array", - "items": { - "type": "string", - "enum": [ "UNIX", "WINDOWS" ] - }, - "maxItems": 1 - }, - "entryPoint": { - "type": "string", - "pattern": "^[^:]+:[^:]+$" - }, - "srcDir": { - "type": "string" - }, - "schemaFile": { - "type": "string" - }, - "manualDiscovery": { - "type": "boolean" - }, - "pluginType": { - "type": "string", - "enum": ["DIRECT", "STAGED"] - }, - "language": { - "type": "string", - "enum": ["PYTHON27"] - }, - "rootSquashEnabled": { - "type": "boolean" - }, - "defaultLocale": { - "type": "string", - "default": "en-us" - }, - "buildNumber": { - "type": "string", - "pattern": "^([0-9]+\\.)*[0-9]*[1-9][0-9]*(\\.[0-9]+)*$" - }, - "luaName": { - "type": "string", - "pattern": "^[a-z0-9_:-]+$" - }, - "extendedStartStopHooks" : { - "type": "boolean" - }, - "minimumLuaVersion": { - "type": "string", - "pattern": "^([0-9]+)\\.([0-9]+)$" - } - }, - "additionalProperties": false, - "required": ["id", "name", "hostTypes", "entryPoint", "srcDir", "schemaFile", "pluginType", "language", "buildNumber"] -} diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index 351d627a..fd88655d 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -377,15 +377,12 @@ def test_zip_and_encode_source_files_encode_fail(mock_encode, src_dir): @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - @pytest.mark.parametrize(('plugin_id', 'skip_id_validation'), - [('77f18ce4-4425-4cd6-b9a7-23653254d660', False), - ('77f18ce4-4425-4cd6-b9a7-23653254d660', True), - ('mongo', True)]) + @pytest.mark.parametrize('plugin_id', + ['77f18ce4-4425-4cd6-b9a7-23653254d660']) def test_id_validation_positive(mock_relative_path, mock_install_deps, mock_import_plugin, plugin_config_file, - artifact_file, skip_id_validation): - build.build(plugin_config_file, artifact_file, False, - skip_id_validation) + artifact_file): + build.build(plugin_config_file, artifact_file, False) @staticmethod @mock.patch.object(PluginImporter, @@ -666,19 +663,6 @@ def test_manual_discovery_parameter(plugin_config_content, src_dir, assert expected == upload_artifact['discoveryDefinition'][ 'manualSourceConfigDiscovery'] - @staticmethod - def test_plugin_config_schemas_diff(): - with open(const.PLUGIN_CONFIG_SCHEMA) as f: - config_schema = json.load(f) - - with open(const.PLUGIN_CONFIG_SCHEMA_NO_ID_VALIDATION) as f: - config_schema_no_id = json.load(f) - - # Only the id's pattern should be different so remove it. - config_schema['properties']['id'].pop('pattern') - - assert config_schema == config_schema_no_id - @staticmethod @pytest.mark.parametrize('build_number, expected', [ pytest.param('0.0.1', '0.0.1'), diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py index 6a9e158c..8396089a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py @@ -251,6 +251,7 @@ def httpretty_enabled(): '{"type": "Toolkit", "reference": "APPDATA_TOOLKIT-1",' '"namespace": null, "prettyName": "python_vfiles",' '"name": "16bef554-9470-11e9-b2e3-8c8590d4a42c", "language": "LUA",' + '"identifier": "16bef554-9470-11e9-b2e3-8c8590d4a42c",' '"version": "1.0.0", "buildApi": {"type": "APIVersion",' '"major": 1, "minor": 10, "micro": 5}, "hostTypes": [' '"UNIX", "WINDOWS"], "rootSquashEnabled": true, ' diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 66abfe4d..d38a2b74 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -577,8 +577,8 @@ def artifact_content(engine_api, virtual_source_definition, """ artifact = { 'type': 'Plugin', - 'name': '16bef554-9470-11e9-b2e3-8c8590d4a42c', - 'prettyName': 'python_vfiles', + 'pluginId': '16bef554-9470-11e9-b2e3-8c8590d4a42c', + 'name': 'python_vfiles', 'externalVersion': '2.0.0', 'defaultLocale': 'en-us', 'language': 'PYTHON27', diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py index 9ef87f31..4e8e215a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py @@ -268,7 +268,6 @@ def test_default_plugin_file_success(mock_build, plugin_config_filename, mock_build.assert_called_once_with(plugin_config_file, artifact_file, False, - False, local_vsdk_root=None) @staticmethod @@ -290,7 +289,6 @@ def test_generate_only_success(mock_build, plugin_config_filename, mock_build.assert_called_once_with(plugin_config_file, None, True, - False, local_vsdk_root=None) @staticmethod @@ -305,7 +303,6 @@ def test_valid_params(mock_build, plugin_config_file, artifact_file): mock_build.assert_called_once_with(plugin_config_file, artifact_file, False, - False, local_vsdk_root=None) @staticmethod @@ -322,24 +319,6 @@ def test_valid_params_new_name(mock_build, plugin_config_file, os.path.join( os.getcwd(), artifact_filename), False, - False, - local_vsdk_root=None) - - @staticmethod - @mock.patch('dlpx.virtualization._internal.commands.build.build') - def test_skip_id_validation(mock_build, plugin_config_file, artifact_file): - runner = click_testing.CliRunner() - - result = runner.invoke(cli.delphix_sdk, [ - 'build', '-c', plugin_config_file, '-a', artifact_file, - '--skip-id-validation' - ]) - - assert result.exit_code == 0, 'Output: {}'.format(result.output) - mock_build.assert_called_once_with(plugin_config_file, - artifact_file, - False, - True, local_vsdk_root=None) @staticmethod @@ -403,7 +382,6 @@ def test_with_dev(mock_build, plugin_config_file, artifact_file, mock_build.assert_called_once_with(plugin_config_file, artifact_file, False, - False, local_vsdk_root='/path/to/vsdk/dir') @staticmethod From b52c2ba3fb198c01d2d7250f801adc84f5764b23 Mon Sep 17 00:00:00 2001 From: Ravi Mukkamala Date: Mon, 30 Nov 2020 18:55:03 -0800 Subject: [PATCH 2/3] Fixes #53 Rename name to plugin_id and prettyName to name --- .bumpversion.cfg | 2 +- common/src/main/python/dlpx/virtualization/common/VERSION | 2 +- dvp/src/main/python/dlpx/virtualization/VERSION | 2 +- libs/src/main/python/dlpx/virtualization/libs/VERSION | 2 +- platform/src/main/python/dlpx/virtualization/platform/VERSION | 2 +- tools/src/main/python/dlpx/virtualization/_internal/VERSION | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bf9cf6b8..4146fcf9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0.dev2 +current_version = 3.0.0.dev3 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index 426be043..762c5b5a 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index 426be043..762c5b5a 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index 426be043..762c5b5a 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index 426be043..762c5b5a 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index 426be043..762c5b5a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file From 7a8194250b993298b510a77f87f1e81421264c4a Mon Sep 17 00:00:00 2001 From: Ravi Mukkamala Date: Mon, 30 Nov 2020 19:06:43 -0800 Subject: [PATCH 3/3] Fixes #53 Rename name to plugin_id and prettyName to name --- .../python/dlpx/virtualization/_internal/test_package_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 36e9669a..e962ea62 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.0.0.dev2' + assert package_util.get_version() == '3.0.0.dev3' @staticmethod def test_get_virtualization_api_version():