diff --git a/core/src/main/python/extract_resource.py b/core/src/main/python/extract_resource.py index 06fb5f26b9..ac81be61cf 100644 --- a/core/src/main/python/extract_resource.py +++ b/core/src/main/python/extract_resource.py @@ -15,6 +15,7 @@ # imports from local packages start here from wlsdeploy.aliases.aliases import Aliases from wlsdeploy.aliases.wlst_modes import WlstModes +from wlsdeploy.exception import exception_helper from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.tool.extract.domain_resource_extractor import DomainResourceExtractor from wlsdeploy.tool.util import model_context_helper @@ -33,21 +34,23 @@ __wlst_mode = WlstModes.OFFLINE __required_arguments = [ - CommandLineArgUtil.ORACLE_HOME_SWITCH, - CommandLineArgUtil.DOMAIN_HOME_SWITCH, - CommandLineArgUtil.DOMAIN_RESOURCE_FILE_SWITCH + CommandLineArgUtil.ORACLE_HOME_SWITCH ] __optional_arguments = [ # Used by shell script to locate WLST CommandLineArgUtil.DOMAIN_TYPE_SWITCH, + CommandLineArgUtil.DOMAIN_HOME_SWITCH, CommandLineArgUtil.ARCHIVE_FILE_SWITCH, CommandLineArgUtil.MODEL_FILE_SWITCH, + CommandLineArgUtil.TARGET_SWITCH, CommandLineArgUtil.VARIABLE_FILE_SWITCH, CommandLineArgUtil.USE_ENCRYPTION_SWITCH, CommandLineArgUtil.PASSPHRASE_SWITCH, CommandLineArgUtil.PASSPHRASE_FILE_SWITCH, CommandLineArgUtil.PASSPHRASE_ENV_SWITCH, + CommandLineArgUtil.OUTPUT_DIR_SWITCH, # move to __required_arguments once DOMAIN_RESOURCE_FILE_SWITCH is removed + CommandLineArgUtil.DOMAIN_RESOURCE_FILE_SWITCH # deprecated, only this program uses it ] @@ -57,6 +60,8 @@ def __process_args(args): :param args: the command-line arguments list :raises CLAException: if an error occurs while validating and processing the command-line arguments """ + _method_name = '__process_args' + cla_util = CommandLineArgUtil(_program_name, __required_arguments, __optional_arguments) cla_util.set_allow_multiple_models(True) argument_map = cla_util.process_args(args, TOOL_TYPE_EXTRACT) @@ -70,19 +75,37 @@ def __process_args(args): # allow unresolved tokens and archive entries argument_map[CommandLineArgUtil.VALIDATION_METHOD] = validate_configuration.LAX_METHOD + + # if no target type was specified, use wko + if CommandLineArgUtil.TARGET_SWITCH not in argument_map: + argument_map[CommandLineArgUtil.TARGET_SWITCH] = 'wko' + + # warn about deprecated -domain_resource_file argument. + # not needed once -domain_resource_file is removed and -output_dir moves to __required_arguments. + if CommandLineArgUtil.DOMAIN_RESOURCE_FILE_SWITCH in argument_map: + __logger.warning('WLSDPLY-10040', CommandLineArgUtil.DOMAIN_RESOURCE_FILE_SWITCH, + CommandLineArgUtil.OUTPUT_DIR_SWITCH, class_name=_class_name, method_name=_method_name) + elif CommandLineArgUtil.OUTPUT_DIR_SWITCH not in argument_map: + ex = exception_helper.create_cla_exception(CommandLineArgUtil.USAGE_ERROR_EXIT_CODE, 'WLSDPLY-20005', + _program_name, CommandLineArgUtil.OUTPUT_DIR_SWITCH, + class_name=_class_name, method_name=_method_name) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + return model_context_helper.create_context(_program_name, argument_map) -def __extract_resource(model, model_context): +def __extract_resource(model, model_context, aliases): """ Offline deployment orchestration :param model: the model :param model_context: the model context + :param aliases: the aliases object :raises: DeployException: if an error occurs """ _method_name = '__extract_resource' - resource_extractor = DomainResourceExtractor(model, model_context, __logger) + resource_extractor = DomainResourceExtractor(model, model_context, aliases, __logger) resource_extractor.extract() return 0 @@ -119,7 +142,7 @@ def main(args): try: model = Model(model_dictionary) - exit_code = __extract_resource(model, model_context) + exit_code = __extract_resource(model, model_context, aliases) except DeployException, ex: __logger.severe('WLSDPLY-09015', _program_name, ex.getLocalizedMessage(), error=ex, class_name=_class_name, method_name=_method_name) diff --git a/core/src/main/python/wlsdeploy/tool/extract/domain_resource_extractor.py b/core/src/main/python/wlsdeploy/tool/extract/domain_resource_extractor.py index acdd86dcfe..b6b675845f 100644 --- a/core/src/main/python/wlsdeploy/tool/extract/domain_resource_extractor.py +++ b/core/src/main/python/wlsdeploy/tool/extract/domain_resource_extractor.py @@ -4,341 +4,68 @@ """ import re -from java.io import File from oracle.weblogic.deploy.util import PyOrderedDict -from oracle.weblogic.deploy.util import PyRealBoolean -from wlsdeploy.aliases import alias_utils -from wlsdeploy.aliases.alias_constants import PASSWORD_TOKEN -from wlsdeploy.aliases.model_constants import CLUSTER -from wlsdeploy.aliases.model_constants import DEFAULT_WLS_DOMAIN_NAME -from wlsdeploy.aliases.model_constants import MODEL_LIST_DELIMITER -from wlsdeploy.aliases.model_constants import NAME -from wlsdeploy.exception import exception_helper from wlsdeploy.exception.expection_types import ExceptionType -from wlsdeploy.tool.extract import wko_schema_helper -from wlsdeploy.tool.util import k8s_helper -from wlsdeploy.util import dictionary_utils -from wlsdeploy.util.model_translator import PythonToFile +from wlsdeploy.tool.util.credential_injector import CredentialInjector +from wlsdeploy.tool.util.targets.output_file_helper import DOMAIN_HOME +from wlsdeploy.tool.util.targets.output_file_helper import SPEC +from wlsdeploy.util import target_configuration_helper -API_VERSION = 'apiVersion' -CHANNELS = 'channels' -CLUSTERS = 'clusters' -CLUSTER_NAME = 'clusterName' -CONFIGURATION = 'configuration' -DOMAIN_HOME = 'domainHome' -DOMAIN_HOME_SOURCE_TYPE = 'domainHomeSourceType' -DOMAIN_TYPE = 'domainType' -IMAGE = 'image' -IMAGE_PULL_POLICY = 'imagePullPolicy' -IMAGE_PULL_SECRETS = 'imagePullSecrets' -K_NAME = 'name' -KIND = 'kind' -METADATA = 'metadata' -MODEL = 'model' -NAMESPACE = 'namespace' -NEVER = 'Never' -REPLICAS = 'replicas' -SECRETS = 'secrets' -SPEC = 'spec' -WEBLOGIC_CREDENTIALS_SECRET = 'webLogicCredentialsSecret' - -DEFAULT_API_VERSION = 'weblogic.oracle/v8' -DEFAULT_KIND = 'Domain' -DEFAULT_WEBLOGIC_CREDENTIALS_SECRET = PASSWORD_TOKEN -DEFAULT_IMAGE = PASSWORD_TOKEN -DEFAULT_IMAGE_PULL_SECRETS = PASSWORD_TOKEN -DEFAULT_SOURCE_TYPE = 'Image' - -# specific to Verrazzano -COMPONENT = 'Component' -TEMPLATE = 'template' -VERRAZZANO_WEBLOGIC_WORKLOAD = 'VerrazzanoWebLogicWorkload' -WORKLOAD = 'workload' - -_secret_pattern = re.compile("@@SECRET:([\\w.-]+):[\\w.-]+@@") +_secret_pattern = re.compile("^@@SECRET:(.*)@@$") class DomainResourceExtractor: """ - Create a domain resource file for use with Kubernetes deployment. + Create output files for the specified target. """ _class_name = "DomainResourceExtractor" - def __init__(self, model, model_context, logger): + def __init__(self, model, model_context, aliases, logger): self._model = model self._model_context = model_context + self._aliases = aliases self._logger = logger def extract(self): - _method_name = 'extract' - - resource_file = self._model_context.get_domain_resource_file() - self._logger.info("WLSDPLY-10000", resource_file, method_name=_method_name, class_name=self._class_name) - - # create the target file directory, if needed - resource_dir = File(resource_file).getParentFile() - if (not resource_dir.isDirectory()) and (not resource_dir.mkdirs()): - mkdir_ex = exception_helper.create_deploy_exception('WLSDPLY-10001', resource_dir) - raise mkdir_ex - - # build the resource file structure from the kubernetes section of the model - resource_dict = self._create_domain_resource_dictionary() - - # revise the resource file structure with values from command line, and elsewhere in model - self._update_resource_dictionary(resource_dict) - - # write the resource file structure to the output file - writer = PythonToFile(resource_dict) - writer.write_to_file(resource_file) - - def _create_domain_resource_dictionary(self): - """ - Build the resource file structure from the kubernetes section of the model. - :return: the resource file structure - """ - kubernetes_map = self._model.get_model_kubernetes() - - resource_dict = PyOrderedDict() - - schema = wko_schema_helper.get_domain_resource_schema(ExceptionType.DEPLOY) - - self._process_object(kubernetes_map, schema, resource_dict, None) - return resource_dict - - def _process_object(self, model_dict, schema_folder, target_dict, schema_path): - """ - Transfer folders and attributes from the model dictionary to the target domain resource dictionary. - :param model_dict: the source model dictionary - :param schema_folder: the schema for this folder - :param target_dict: the target dictionary for the domain resource file. - :param schema_path: the path of schema elements, used for supported check - """ - folder_properties = schema_folder["properties"] - - for key, model_value in model_dict.items(): - properties = folder_properties[key] - - if wko_schema_helper.is_single_object(properties): - next_schema_path = wko_schema_helper.append_path(schema_path, key) - if not wko_schema_helper.is_unsupported_folder(next_schema_path): - next_target_dict = PyOrderedDict() - target_dict[key] = next_target_dict - self._process_object(model_value, properties, next_target_dict, next_schema_path) - - elif wko_schema_helper.is_object_array(properties): - next_schema_path = wko_schema_helper.append_path(schema_path, key) - if not wko_schema_helper.is_unsupported_folder(next_schema_path): - item_info = wko_schema_helper.get_array_item_info(properties) - target_dict[key] = \ - self._process_object_array(model_value, item_info, next_schema_path) - - elif wko_schema_helper.is_simple_map(properties): - # map of key / value pairs - target_dict[key] = model_value - - else: - # simple type or array of simple type, such as number, string - property_type = wko_schema_helper.get_type(properties) - target_dict[key] = _get_target_value(model_value, property_type) - - def _process_object_array(self, model_value, item_info, schema_path): - """ - Process an array of objects. - :param model_value: the model contents for a folder - :param item_info: describes the contents of the sub-folder for each element - :param schema_path: the path of schema elements, used for supported check - """ - child_list = list() - - # deprecated "named object list" format - warning was issued in validator - if isinstance(model_value, dict): - for name in model_value: - object_map = model_value[name] - next_target_dict = PyOrderedDict() - self._process_object(object_map, item_info, next_target_dict, schema_path) - - # see if the model name should become an attribute in the target dict - mapped_name = wko_schema_helper.get_object_list_key(schema_path) - properties = wko_schema_helper.get_properties(item_info) - if (mapped_name in properties.keys()) and (mapped_name not in next_target_dict.keys()): - _add_to_top(next_target_dict, mapped_name, name) - child_list.append(next_target_dict) - return child_list - # end deprecated - - for object_map in model_value: - next_target_dict = PyOrderedDict() - self._process_object(object_map, item_info, next_target_dict, schema_path) - child_list.append(next_target_dict) - return child_list - - def _update_resource_dictionary(self, resource_dict): - """ - Revise the resource file structure with values from defaults, command line, and elsewhere in model - :param resource_dict: the resource file dictionary - """ - _method_name = '_update_resource_dictionary' - - # add a metadata section if not present, since we'll at least add name - if METADATA not in resource_dict: - _add_to_top(resource_dict, METADATA, PyOrderedDict()) - metadata_section = resource_dict[METADATA] - - # add kind if not present - if KIND not in resource_dict: - _add_to_top(resource_dict, KIND, DEFAULT_KIND) - - # add API version if not present - if API_VERSION not in resource_dict: - _add_to_top(resource_dict, API_VERSION, DEFAULT_API_VERSION) + # create a credential injector containing secrets from the model + model_dict = self._model.get_model() + credential_injector = CredentialInjector(DomainResourceExtractor, model_dict, self._model_context) + _add_secrets(model_dict, credential_injector) - # if metadata name not present, use the domain name from the model, or default - if K_NAME not in metadata_section: - domain_name = dictionary_utils.get_element(self._model.get_model_topology(), NAME) - if domain_name is None: - domain_name = DEFAULT_WLS_DOMAIN_NAME - domain_name = k8s_helper.get_domain_uid(domain_name) - metadata_section[K_NAME] = domain_name - domain_uid = metadata_section[K_NAME] + # if -domain_home was specified on the command line, it should override any value in the model + domain_home_override = self._model_context.get_domain_home() + if domain_home_override: + kubernetes_dict = self._model.get_model_kubernetes() + spec_dict = get_or_create_dictionary(kubernetes_dict, SPEC) + spec_dict[DOMAIN_HOME] = domain_home_override - # add a spec section if not present, since we'll at least add domain home - if SPEC not in resource_dict: - resource_dict[SPEC] = PyOrderedDict() - spec_section = resource_dict[SPEC] + # create the output files with information from the model + target_configuration_helper.create_additional_output(self._model, self._model_context, self._aliases, + credential_injector, ExceptionType.DEPLOY) - # only set domain home if it is not present in spec section - if DOMAIN_HOME not in spec_section: - spec_section[DOMAIN_HOME] = self._model_context.get_domain_home() - # only set domain home source type if it is not present in spec section - if DOMAIN_HOME_SOURCE_TYPE not in spec_section: - spec_section[DOMAIN_HOME_SOURCE_TYPE] = DEFAULT_SOURCE_TYPE - - # only set image if it is not present in spec section - if IMAGE not in spec_section: - spec_section[IMAGE] = DEFAULT_IMAGE - - # imagePullSecrets is required unless imagePullPolicy is Never - pull_secrets_required = True - if IMAGE_PULL_POLICY in spec_section: - policy = str(spec_section[IMAGE_PULL_POLICY]) - pull_secrets_required = (policy != NEVER) - - # if imagePullSecrets required and not present, add a list with one FIX ME value - if pull_secrets_required and (IMAGE_PULL_SECRETS not in spec_section): - secrets_list = list() - secrets_list.append({'name': DEFAULT_IMAGE_PULL_SECRETS}) - spec_section[IMAGE_PULL_SECRETS] = secrets_list - - # if webLogicCredentialsSecret not present, add it using the FIX ME value - if WEBLOGIC_CREDENTIALS_SECRET not in spec_section: - spec_section[WEBLOGIC_CREDENTIALS_SECRET] = DEFAULT_WEBLOGIC_CREDENTIALS_SECRET - - # only update clusters if section is not present in spec section - if CLUSTERS not in spec_section: - topology = self._model.get_model_topology() - model_clusters = dictionary_utils.get_dictionary_element(topology, CLUSTER) - if len(model_clusters) > 0: - cluster_list = list() - spec_section[CLUSTERS] = cluster_list - for cluster_name, cluster_values in model_clusters.items(): - if REPLICAS in spec_section: - server_count = spec_section[REPLICAS] - else: - server_count = k8s_helper.get_server_count(cluster_name, cluster_values, - self._model.get_model()) - cluster_dict = PyOrderedDict() - cluster_dict[CLUSTER_NAME] = cluster_name - cluster_dict[REPLICAS] = server_count - - self._logger.info("WLSDPLY-10002", cluster_name, server_count, method_name=_method_name, - class_name=self._class_name) - cluster_list.append(cluster_dict) - - # create a configuration section in spec if needed - if CONFIGURATION not in spec_section: - spec_section[CONFIGURATION] = PyOrderedDict() - configuration_section = spec_section[CONFIGURATION] - - # create a model section in configuration if needed - if MODEL not in configuration_section: - configuration_section[MODEL] = PyOrderedDict() - model_section = configuration_section[MODEL] - - # set domainType if not specified - if DOMAIN_TYPE not in model_section: - model_section[DOMAIN_TYPE] = self._model_context.get_domain_type() - - if SECRETS in configuration_section: - # if secrets specified, convert them to a hyphen list - secrets = alias_utils.convert_to_model_type("list", configuration_section[SECRETS], MODEL_LIST_DELIMITER) - secrets_list = list() - secrets_list.extend(secrets) - - else: - # pull the secrets from the model - secrets_list = list() - _add_secrets(self._model.get_model(), secrets_list, domain_uid) - - if secrets_list: - configuration_section[SECRETS] = secrets_list - - -def _get_target_value(model_value, type_name): - """ - Return the value for the specified attribute value, to be used in the domain resource file. - :param model_value: the value to be checked - :param type_name: the schema type name of the value - :return: the formatted value - """ - if type_name == 'boolean': - # the model values can be true, false, 1, 0, etc. - # target boolean values must be 'true' or 'false' - return PyRealBoolean(alias_utils.convert_boolean(model_value)) - - if type_name == 'array': - # the model values can be 'abc,123'. - # target values must be a list object. - return alias_utils.convert_to_type('list', model_value, delimiter=MODEL_LIST_DELIMITER) - - return model_value - - -def _add_secrets(folder, secrets, domain_uid): +def _add_secrets(folder, credential_injector): """ Recursively add any secrets found in the specified folder. :param folder: the folder to be checked - :param secrets: the list to be appended + :param credential_injector: the injector to collect secrets """ for name in folder: value = folder[name] if isinstance(value, dict): - _add_secrets(value, secrets, domain_uid) + _add_secrets(value, credential_injector) else: text = str(value) - - # secrets created by discover or prepareModel use this environment variable. - # if it wasn't resolved from the environment, replace with model's domain UID. - text = text.replace("@@ENV:DOMAIN_UID@@", domain_uid) - matches = _secret_pattern.findall(text) for secret_name in matches: - if secret_name not in secrets: - secrets.append(secret_name) + # remove the domain UID variable prefix, the output helper will prepend the actual UID + secret_name = secret_name.replace('@@ENV:DOMAIN_UID@@-', '') + if secret_name not in credential_injector.get_variable_cache(): + credential_injector.add_to_cache(token_name=secret_name, token_value='') -def _add_to_top(dictionary, key, item): - """ - Add an item to the beginning of an ordered dictionary. - :param dictionary: the dictionary - :param key: the key of the item to be added - :param item: the item to be added - """ - temp_dict = PyOrderedDict() - for each_key in dictionary: - temp_dict[each_key] = dictionary[each_key] - dictionary.clear() - dictionary[key] = item - for each_key in temp_dict: - dictionary[each_key] = temp_dict[each_key] +def get_or_create_dictionary(dictionary, key): + if key not in dictionary: + dictionary[key] = PyOrderedDict() + return dictionary[key] diff --git a/core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py b/core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py index 2e0bd92f38..c97f82b323 100644 --- a/core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py @@ -4,6 +4,8 @@ Methods for creating Kubernetes resource configuration files for Verrazzano. """ +import os.path + from java.io import File from wlsdeploy.aliases.location_context import LocationContext @@ -70,11 +72,42 @@ def create_additional_output(model, model_context, aliases, credential_injector, # all current output types use this hash, and process a set of template files template_hash = _build_template_hash(model, model_context, aliases, credential_injector) template_names = model_context.get_target_configuration().get_additional_output_types() - for template_name in template_names: + for index, template_name in enumerate(template_names): + # special processing for deprecated -domain_resource_file argument + # used only by extractDomainResource + if _create_named_file(index, template_name, template_hash, model, model_context, exception_type): + continue + _create_file(template_name, template_hash, output_dir, exception_type) output_file_helper.update_from_model(output_dir, template_name, model) +# *** DELETE METHOD WHEN deprecated -domain_resource_file IS REMOVED *** +def _create_named_file(index, template_name, template_hash, model, model_context, exception_type): + """ + Special processing for deprecated -domain_resource_file argument used by extractDomainResource. + Use the directory of -domain_resource_file for all templates, + and the name of -domain_resource_file for the first (usually only) template. + """ + _method_name = '_create_named_file' + + resource_file = model_context.get_domain_resource_file() + if resource_file: + template_subdir = "targets/templates/" + template_name + template_path = path_utils.find_config_path(template_subdir) + + output_dir, output_name = os.path.split(resource_file) + if index > 0: + output_name = template_name + + output_file = File(os.path.join(output_dir, output_name)) + __logger.info('WLSDPLY-01662', output_file, class_name=__class_name, method_name=_method_name) + file_template_helper.create_file_from_file(template_path, template_hash, output_file, exception_type) + output_file_helper.update_from_model(output_dir, output_name, model) + return True + return False + + def _create_file(template_name, template_hash, output_dir, exception_type): """ Read the template from the resource stream, perform any substitutions, diff --git a/core/src/main/python/wlsdeploy/tool/util/targets/output_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/targets/output_file_helper.py index 5ec13d4017..c5d6b22d28 100644 --- a/core/src/main/python/wlsdeploy/tool/util/targets/output_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/targets/output_file_helper.py @@ -6,19 +6,16 @@ """ from java.io import File +from oracle.weblogic.deploy.util import PyOrderedDict +from oracle.weblogic.deploy.util import PyRealBoolean from oracle.weblogic.deploy.yaml import YamlException -from oracle.weblogic.deploy.util import PyOrderedDict +from wlsdeploy.aliases import alias_utils from wlsdeploy.aliases.model_constants import KUBERNETES +from wlsdeploy.aliases.model_constants import MODEL_LIST_DELIMITER +from wlsdeploy.exception.expection_types import ExceptionType from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.tool.extract import wko_schema_helper -from wlsdeploy.tool.extract.domain_resource_extractor import COMPONENT -from wlsdeploy.tool.extract.domain_resource_extractor import DEFAULT_KIND -from wlsdeploy.tool.extract.domain_resource_extractor import KIND -from wlsdeploy.tool.extract.domain_resource_extractor import SPEC -from wlsdeploy.tool.extract.domain_resource_extractor import TEMPLATE -from wlsdeploy.tool.extract.domain_resource_extractor import VERRAZZANO_WEBLOGIC_WORKLOAD -from wlsdeploy.tool.extract.domain_resource_extractor import WORKLOAD from wlsdeploy.util import dictionary_utils from wlsdeploy.yaml.yaml_translator import PythonToYaml from wlsdeploy.yaml.yaml_translator import YamlToPython @@ -26,6 +23,18 @@ __class_name = 'output_file_helper' __logger = PlatformLogger('wlsdeploy.tool.util') +KIND = 'kind' +SPEC = 'spec' + +WKO_DOMAIN_KIND = 'Domain' +DOMAIN_HOME = 'domainHome' + +# specific to Verrazzano +COMPONENT_KIND = 'Component' +TEMPLATE = 'template' +VERRAZZANO_WEBLOGIC_WORKLOAD_KIND = 'VerrazzanoWebLogicWorkload' +WORKLOAD = 'workload' + def update_from_model(output_dir, output_file_name, model): """ @@ -71,36 +80,39 @@ def _update_documents(documents, kubernetes_content, output_file_path): _method_name = '_update_documents' found = False + schema = wko_schema_helper.get_domain_resource_schema(ExceptionType.DEPLOY) + # update section(s) based on their kind, etc. for document in documents: if isinstance(document, dict): kind = dictionary_utils.get_element(document, KIND) # is this a standard WKO document? - if kind == DEFAULT_KIND: - _update_dictionary(document, kubernetes_content, None, output_file_path) + if kind == WKO_DOMAIN_KIND: + _update_dictionary(document, kubernetes_content, schema, None, output_file_path) found = True # is this a Verrazzano WebLogic workload document? - elif kind == COMPONENT: + elif kind == COMPONENT_KIND: spec = dictionary_utils.get_dictionary_element(document, SPEC) workload = dictionary_utils.get_dictionary_element(spec, WORKLOAD) component_kind = dictionary_utils.get_element(workload, KIND) - if component_kind == VERRAZZANO_WEBLOGIC_WORKLOAD: + if component_kind == VERRAZZANO_WEBLOGIC_WORKLOAD_KIND: component_spec = _get_or_create_dictionary(workload, SPEC) component_template = _get_or_create_dictionary(component_spec, TEMPLATE) - _update_dictionary(component_template, kubernetes_content, None, output_file_path) + _update_dictionary(component_template, kubernetes_content, schema, None, output_file_path) found = True if not found: __logger.warning('WLSDPLY-01676', output_file_path, class_name=__class_name, method_name=_method_name) -def _update_dictionary(output_dictionary, model_dictionary, schema_path, output_file_path): +def _update_dictionary(output_dictionary, model_dictionary, schema_folder, schema_path, output_file_path): """ Update output_dictionary with attributes from model_dictionary. :param output_dictionary: the dictionary to be updated :param model_dictionary: the dictionary to update from (type previously validated) + :param schema_folder: the schema for this folder :param schema_path: used for wko_schema_helper lookups and logging :param output_file_path: used for logging """ @@ -110,29 +122,46 @@ def _update_dictionary(output_dictionary, model_dictionary, schema_path, output_ method_name=_method_name) return + # no type checking for elements of simple (single type) map + if wko_schema_helper.is_simple_map(schema_folder): + for key, value in model_dictionary.items(): + output_dictionary[key] = value + return + + properties = wko_schema_helper.get_properties(schema_folder) + for key, value in model_dictionary.items(): + property_folder = properties[key] + element_type = wko_schema_helper.get_type(property_folder) + + # deprecated "named object list" format + value = _check_named_object_list(value, element_type, property_folder, schema_path, key) + # end deprecated + + value = _convert_value(value, element_type) + if key not in output_dictionary: output_dictionary[key] = value elif isinstance(value, dict): next_schema_path = wko_schema_helper.append_path(schema_path, key) - _update_dictionary(output_dictionary[key], value, next_schema_path, output_file_path) + _update_dictionary(output_dictionary[key], value, property_folder, next_schema_path, output_file_path) elif isinstance(value, list): if not value: # if the model has an empty list, override output value output_dictionary[key] = value else: next_schema_path = wko_schema_helper.append_path(schema_path, key) - _update_list(output_dictionary[key], value, next_schema_path, output_file_path) + _update_list(output_dictionary[key], value, property_folder, next_schema_path, output_file_path) else: output_dictionary[key] = value - pass -def _update_list(output_list, model_list, schema_path, output_file_path): +def _update_list(output_list, model_list, schema_folder, schema_path, output_file_path): """ Update output_list from model_list, overriding or merging existing values :param output_list: the list to be updated :param model_list: the list to update from (type previously validated) + :param schema_folder: the schema for members of this list :param schema_path: used for wko_schema_helper lookups and logging :param output_file_path: used for logging """ @@ -146,10 +175,13 @@ def _update_list(output_list, model_list, schema_path, output_file_path): if isinstance(item, dict): match = _find_object_match(item, output_list, schema_path) if match: - _update_dictionary(match, item, schema_path, output_file_path) + next_schema_folder = wko_schema_helper.get_array_item_info(schema_folder) + _update_dictionary(match, item, next_schema_folder, schema_path, output_file_path) else: output_list.append(item) elif item not in output_list: + element_type = wko_schema_helper.get_array_element_type(schema_folder) + item = _convert_value(item, element_type) output_list.append(item) @@ -171,6 +203,58 @@ def _find_object_match(item, match_list, schema_path): return None +def _convert_value(model_value, type_name): + """ + Convert the specified model value to match the schema type for the domain resource file. + WDT allows some model conventions that are not allowed in the domain resource file. + :param model_value: the value to be checked + :param type_name: the schema type name of the value + :return: the converted value + """ + if type_name == 'boolean': + # the model values can be true, false, 1, 0, etc. + # target boolean values must be 'true' or 'false' + return PyRealBoolean(alias_utils.convert_boolean(model_value)) + + if type_name == 'array': + # the model values can be 'abc,123'. + # target values must be a list object. + return alias_utils.convert_to_type('list', model_value, delimiter=MODEL_LIST_DELIMITER) + + return model_value + + +# *** DELETE METHOD WHEN deprecated "named object list" IS REMOVED *** +def _check_named_object_list(model_value, type_name, schema_folder, schema_path, key): + """ + Convert specified model value to an object list if it uses deprecated "named object list" format. + :param model_value: the value to be checked + :param type_name: the schema type name of the value + :param schema_folder: the schema for the value being checked + :param schema_path: used for wko_schema_helper key lookup + :param key: used for wko_schema_helper key lookup + :return: the converted value + """ + if type_name == 'array' and isinstance(model_value, dict): + object_list = list() + next_schema_path = wko_schema_helper.append_path(schema_path, key) + list_key = wko_schema_helper.get_object_list_key(next_schema_path) + item_info = wko_schema_helper.get_array_item_info(schema_folder) + properties = wko_schema_helper.get_properties(item_info) + + for model_key, model_object in model_value.items(): + new_object = model_object.copy() + + # see if the model name should become an attribute in the new object + if (list_key in properties.keys()) and (list_key not in new_object.keys()): + new_object[list_key] = model_key + + object_list.append(new_object) + return object_list + + return model_value + + def _get_or_create_dictionary(dictionary, key): if key not in dictionary: dictionary[key] = PyOrderedDict() diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 2f6ecfba98..61d53bc826 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1169,14 +1169,12 @@ WLSDPLY-09701=In online WLST, unable to target resources from extension template ############################################################################### # Extract messages (10000 - 10099) # ############################################################################### -# wlsdeploy/tool/extract/domain_resource_extractor.py -WLSDPLY-10000=Extracting to domain resource file {0} -WLSDPLY-10001=Failed to create directory {0} for domain resource file -WLSDPLY-10002=Adding cluster {0} to domain resource file with server count {1} - # wlsdeploy/tool/extract/wko_schema_helper.py WLSDPLY-10010=Failed to load WKO domain resource schema {0} +# extract_resource.py +WLSDPLY-10040=The {0} argument has been deprecated, use {1} to specify output directory + ############################################################################### # model help messages (10100 - 10199) # ############################################################################### diff --git a/core/src/test/python/wlsdeploy/tool/extract/extract_test.py b/core/src/test/python/wlsdeploy/tool/extract/extract_test.py index f0f1537b67..3ca1ad7cbe 100644 --- a/core/src/test/python/wlsdeploy/tool/extract/extract_test.py +++ b/core/src/test/python/wlsdeploy/tool/extract/extract_test.py @@ -1,10 +1,13 @@ """ -Copyright (c) 2021, Oracle and/or its affiliates. +Copyright (c) 2021, 2022, Oracle and/or its affiliates. Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. """ import os +import shutil from base_test import BaseTestCase +from wlsdeploy.aliases.aliases import Aliases +from wlsdeploy.aliases.wlst_modes import WlstModes from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.tool.extract.domain_resource_extractor import DomainResourceExtractor from wlsdeploy.util.model import Model @@ -14,21 +17,45 @@ class ExtractTest(BaseTestCase): __logger = PlatformLogger('wlsdeploy.extract') + wls_version = '12.2.1.3' def __init__(self, *args): BaseTestCase.__init__(self, *args) self.MODELS_DIR = os.path.join(self.TEST_CLASSES_DIR, 'extract') self.EXTRACT_OUTPUT_DIR = os.path.join(self.TEST_OUTPUT_DIR, 'extract') + self.TARGET_SOURCE_DIR = os.path.abspath(self.TEST_CLASSES_DIR + '/../../../core/src/main/targetconfigs') def setUp(self): BaseTestCase.setUp(self) self._suspend_logs('wlsdeploy.extract') self._establish_directory(self.EXTRACT_OUTPUT_DIR) + config_dir = os.path.join(self.TEST_OUTPUT_DIR, 'config') + targets_dir = os.path.join(config_dir, 'targets') + self._establish_directory(config_dir) + self._establish_directory(targets_dir) + + self._copy_target_file(targets_dir, 'wko', 'target.json') + self._copy_target_file(targets_dir, 'templates', 'wko-domain.yaml') + + # use WDT custom configuration to find target definition + self._set_custom_config_dir(config_dir) + def tearDown(self): BaseTestCase.tearDown(self) self._restore_logs() + # clean up temporary WDT custom configuration environment variable + self._clear_custom_config_dir() + + def _copy_target_file(self, targets_dir, target_name, target_file_name): + target_dir = os.path.join(targets_dir, target_name) + target_file = os.path.join(target_dir, target_file_name) + if not os.path.exists(target_file): + self._establish_directory(target_dir) + source_file = os.path.join(self.TARGET_SOURCE_DIR, target_name, target_file_name) + shutil.copy(source_file, target_file) + def testDefaultModel(self): """ Test that default values and information from the model @@ -99,11 +126,13 @@ def _extract_domain_resource(self, suffix): args_map = { '-domain_home': '/u01/domain', '-oracle_home': '/oracle', - '-domain_resource_file': resource_file + '-domain_resource_file': resource_file, + '-target': 'wko' } model_context = ModelContext('ExtractTest', args_map) + aliases = Aliases(model_context, WlstModes.OFFLINE, self.wls_version) - extractor = DomainResourceExtractor(model, model_context, self.__logger) + extractor = DomainResourceExtractor(model, model_context, aliases, self.__logger) extractor.extract() translator = FileToPython(resource_file, use_ordering=True) diff --git a/installer/src/main/bin/extractDomainResource.cmd b/installer/src/main/bin/extractDomainResource.cmd index 0b7b3d4f86..1c5135c4e4 100644 --- a/installer/src/main/bin/extractDomainResource.cmd +++ b/installer/src/main/bin/extractDomainResource.cmd @@ -2,7 +2,7 @@ @rem ************************************************************************** @rem extractDomainResource.cmd @rem -@rem Copyright (c) 2020, Oracle Corporation and/or its affiliates. All rights reserved. +@rem Copyright (c) 2020, 2022, Oracle Corporation and/or its affiliates. All rights reserved. @rem Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. @rem @rem NAME @@ -70,8 +70,10 @@ if "%SHOW_USAGE%" == "false" ( ECHO. ECHO Usage: %SCRIPT_NAME% [-help] [-use_encryption] ECHO [-oracle_home ^] -ECHO -domain_home ^ -ECHO -domain_resource_file ^ +ECHO [-domain_home ^] +ECHO [-output_dir ^] +ECHO [-target ^] +ECHO [-domain_resource_file ^] ECHO [-archive_file ^] ECHO [-model_file ^] ECHO [-variable_file ^] @@ -81,9 +83,15 @@ ECHO oracle_home - the existing Oracle Home directory for the domain ECHO This is required unless the ORACLE_HOME environment ECHO variable is set. ECHO. -ECHO domain_home - the domain home directory +ECHO domain_home - the domain home directory to be used in output files. +ECHO This will override any value in the model. ECHO. -ECHO domain_resource_file - the location of the extracted domain resource file +ECHO output_dir - the location for the target output files. +ECHO. +ECHO target - the target output type. The default is wko. +ECHO. +ECHO domain_resource_file - the location of the extracted domain resource file. +ECHO This is deprecated, use -output_dir to specify output location ECHO. ECHO archive_file - the path to the archive file to use. If the -model_file ECHO argument is not specified, the model file in this archive diff --git a/installer/src/main/bin/extractDomainResource.sh b/installer/src/main/bin/extractDomainResource.sh index fc438a764b..8e46b00bb3 100644 --- a/installer/src/main/bin/extractDomainResource.sh +++ b/installer/src/main/bin/extractDomainResource.sh @@ -2,7 +2,7 @@ # ***************************************************************************** # extractDomainResource.sh # -# Copyright (c) 2020, Oracle Corporation and/or its affiliates. All rights reserved. +# Copyright (c) 2020, 2022, Oracle Corporation and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # # NAME @@ -31,8 +31,10 @@ usage() { echo "" echo "Usage: $1 [-help] [-use_encryption]" echo " [-oracle_home ]" - echo " -domain_home " - echo " -domain_resource_file " + echo " [-domain_home ]" + echo " [-output_dir ]" + echo " [-target ]" + echo " [-domain_resource_file ]" echo " [-archive_file ]" echo " [-model_file ]" echo " [-variable_file ]" @@ -42,9 +44,15 @@ usage() { echo " This is required unless the ORACLE_HOME environment" echo " variable is set." echo "" - echo " domain_home - the domain home directory" + echo " domain_home - the domain home directory to be used in output files." + echo " This will override any value in the model." echo "" - echo " domain_resource_file - the location of the extracted domain resource file" + echo " output_dir - the location for the target output files." + echo "" + echo " target - the target output type. The default is wko." + echo "" + echo " domain_resource_file - the location of the extracted domain resource file." + echo " This is deprecated, use -output_dir to specify output location" echo "" echo " archive_file - the path to the archive file to use If the -model_file" echo " argument is not specified, the model file in this archive"