From df2b916e624719e5f77e29c1e893c55f88e15862 Mon Sep 17 00:00:00 2001 From: max-orlov Date: Thu, 13 Jul 2017 16:49:15 +0300 Subject: [PATCH] ARIA-174 Refactor instantiation phase The main issue this PR addressed was the consolidation of instantiation mechanism into a single package - `topology`. The code itself got additional refactoring: 1. `dump` no longer prints onto the console but rather returns a string representation to be printed. 2. The service related consumers use the new `topology` package. These additional issues were fixed as part of this PR: * Req-Cap mechanism is now trying to satisfy requirements for a specific capability type. * Issue with resuming workflows - accessing a non existing attribute. --- aria/cli/commands/executions.py | 2 +- aria/cli/commands/service_templates.py | 7 +- aria/cli/commands/services.py | 6 +- aria/core.py | 36 +- aria/modeling/functions.py | 3 +- aria/modeling/mixins.py | 42 +- aria/modeling/service_common.py | 27 - aria/modeling/service_instance.py | 535 +------------ aria/modeling/service_template.py | 740 +----------------- aria/modeling/utils.py | 85 +- .../execution_plugin/instantiation.py | 113 ++- aria/orchestrator/topology/__init__.py | 16 + aria/orchestrator/topology/common.py | 69 ++ .../orchestrator/topology/instance_handler.py | 671 ++++++++++++++++ .../orchestrator/topology/template_handler.py | 606 ++++++++++++++ aria/orchestrator/topology/topology.py | 223 ++++++ aria/orchestrator/topology/utils.py | 48 ++ aria/orchestrator/workflow_runner.py | 5 +- aria/orchestrator/workflows/api/task.py | 5 +- aria/parser/consumption/__init__.py | 3 - aria/parser/consumption/consumer.py | 7 + aria/parser/consumption/context.py | 4 +- aria/parser/consumption/modeling.py | 32 +- aria/parser/consumption/style.py | 54 -- aria/parser/modeling/context.py | 6 +- aria/parser/presentation/fields.py | 16 +- aria/parser/presentation/presentation.py | 14 +- aria/parser/reading/__init__.py | 6 +- aria/parser/reading/locator.py | 35 - aria/parser/validation/context.py | 59 +- aria/parser/validation/issue.py | 72 +- aria/utils/__init__.py | 47 ++ aria/utils/console.py | 49 +- .../aria_extension_tosca/simple_v1_0/misc.py | 2 +- .../simple_v1_0/modeling/__init__.py | 9 +- tests/parser/service_templates.py | 18 +- tests/parser/test_reqs_caps.py | 29 + .../reqs_caps/reqs_caps1.yaml | 40 + tests/storage/__init__.py | 2 - 39 files changed, 2069 insertions(+), 1674 deletions(-) create mode 100644 aria/orchestrator/topology/__init__.py create mode 100644 aria/orchestrator/topology/common.py create mode 100644 aria/orchestrator/topology/instance_handler.py create mode 100644 aria/orchestrator/topology/template_handler.py create mode 100644 aria/orchestrator/topology/topology.py create mode 100644 aria/orchestrator/topology/utils.py delete mode 100644 aria/parser/consumption/style.py create mode 100644 tests/parser/test_reqs_caps.py create mode 100644 tests/resources/service-templates/tosca-simple-1.0/reqs_caps/reqs_caps1.yaml diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py index 47834427..f130d954 100644 --- a/aria/cli/commands/executions.py +++ b/aria/cli/commands/executions.py @@ -181,7 +181,7 @@ def resume(execution_id, executor = DryExecutor() if dry else None # use WorkflowRunner's default executor execution = model_storage.execution.get(execution_id) - if execution.status != execution.status.CANCELLED: + if execution.status != execution.CANCELLED: logger.info("Can't resume execution {execution.id} - " "execution is in status {execution.status}. " "Can only resume executions in status {valid_status}" diff --git a/aria/cli/commands/service_templates.py b/aria/cli/commands/service_templates.py index f567aa8f..5a7039cc 100644 --- a/aria/cli/commands/service_templates.py +++ b/aria/cli/commands/service_templates.py @@ -28,7 +28,7 @@ from ...storage import exceptions as storage_exceptions from ...parser import consumption from ...utils import (formatting, collections, console) - +from ... orchestrator import topology DESCRIPTION_FIELD_LENGTH_LIMIT = 20 SERVICE_TEMPLATE_COLUMNS = \ @@ -73,10 +73,9 @@ def show(service_template_name, model_storage, mode_full, mode_types, format_jso elif format_yaml: console.puts(formatting.yaml_dumps(collections.prune(service_template.as_raw))) else: - service_template.dump() + console.puts(topology.Topology().dump(service_template)) elif mode_types: - consumption.ConsumptionContext() - service_template.dump_types() + console.puts(topology.Topology().dump_types(service_template=service_template)) else: logger.info('Showing service template {0}...'.format(service_template_name)) service_template_dict = service_template.to_dict() diff --git a/aria/cli/commands/services.py b/aria/cli/commands/services.py index a99f5b35..6752899b 100644 --- a/aria/cli/commands/services.py +++ b/aria/cli/commands/services.py @@ -30,6 +30,7 @@ from ...storage import exceptions as storage_exceptions from ...parser import consumption from ...utils import (formatting, collections, console) +from ...orchestrator import topology DESCRIPTION_FIELD_LENGTH_LIMIT = 20 @@ -73,10 +74,9 @@ def show(service_name, model_storage, mode_full, mode_graph, format_json, format elif format_yaml: console.puts(formatting.yaml_dumps(collections.prune(service.as_raw))) else: - service.dump() + console.puts(topology.Topology().dump(service)) elif mode_graph: - consumption.ConsumptionContext() - service.dump_graph() + console.puts(topology.Topology().dump_graph(service)) else: logger.info('Showing service {0}...'.format(service_name)) service_dict = service.to_dict() diff --git a/aria/core.py b/aria/core.py index e214b1a3..e3b3b360 100644 --- a/aria/core.py +++ b/aria/core.py @@ -20,6 +20,7 @@ from . import exceptions from .parser import consumption from .parser.loading.location import UriLocation +from .orchestrator import topology class Core(object): @@ -67,31 +68,26 @@ def delete_service_template(self, service_template_id): self.resource_storage.service_template.delete(entry_id=str(service_template.id)) def create_service(self, service_template_id, inputs, service_name=None): - service_template = self.model_storage.service_template.get(service_template_id) - # creating an empty ConsumptionContext, initiating a threadlocal context - context = consumption.ConsumptionContext() - storage_session = self.model_storage._all_api_kwargs['session'] # setting no autoflush for the duration of instantiation - this helps avoid dependency # constraints as they're being set up with storage_session.no_autoflush: - service = service_template.instantiate(None, self.model_storage, inputs=inputs) - - consumption.ConsumerChain( - context, - ( - consumption.CoerceServiceInstanceValues, - consumption.ValidateServiceInstance, - consumption.SatisfyRequirements, - consumption.CoerceServiceInstanceValues, - consumption.ValidateCapabilities, - consumption.FindHosts, - consumption.ConfigureOperations, - consumption.CoerceServiceInstanceValues - )).consume() - if context.validation.dump_issues(): + topology_ = topology.Topology() + service = topology_.instantiate( + service_template, inputs=inputs, plugins=self.model_storage.plugin.list()) + topology_.coerce(service, report_issues=True) + + topology_.validate(service) + topology_.satisfy_requirements(service) + topology_.coerce(service, report_issues=True) + + topology_.validate_capabilities(service) + topology_.assign_hosts(service) + topology_.configure_operations(service) + topology_.coerce(service, report_issues=True) + if topology_.dump_issues(): raise exceptions.InstantiationError('Failed to instantiate service template `{0}`' .format(service_template.name)) @@ -122,6 +118,8 @@ def delete_service(self, service_id, force=False): def _parse_service_template(service_template_path): context = consumption.ConsumptionContext() context.presentation.location = UriLocation(service_template_path) + # Most of the parser uses the topology package in order to manipulate the models. + # However, here we use the Consumer mechanism, but this should change in the future. consumption.ConsumerChain( context, ( diff --git a/aria/modeling/functions.py b/aria/modeling/functions.py index 6544adfa..31c3839d 100644 --- a/aria/modeling/functions.py +++ b/aria/modeling/functions.py @@ -16,9 +16,8 @@ """ Mechanism for evaluating intrinsic functions. """ - -from ..parser.consumption import ConsumptionContext from ..parser.exceptions import InvalidValueError +from ..parser.consumption import ConsumptionContext from ..utils.collections import OrderedDict from . import exceptions diff --git a/aria/modeling/mixins.py b/aria/modeling/mixins.py index 4acbe6ef..d58c25ae 100644 --- a/aria/modeling/mixins.py +++ b/aria/modeling/mixins.py @@ -25,8 +25,7 @@ PickleType ) -from ..parser.consumption import ConsumptionContext -from ..utils import console, collections, caching, formatting +from ..utils import collections, caching from ..utils.type import canonical_type_name, full_type_name from . import utils, functions @@ -132,26 +131,17 @@ class InstanceModelMixin(ModelMixin): def as_raw(self): raise NotImplementedError - def validate(self): - pass - def coerce_values(self, report_issues): pass - def dump(self): - pass - -class TemplateModelMixin(InstanceModelMixin): +class TemplateModelMixin(InstanceModelMixin): # pylint: disable=abstract-method """ Mix-in for service template models. All model models can be instantiated into service instance models. """ - def instantiate(self, container): - raise NotImplementedError - class ParameterMixin(TemplateModelMixin, caching.HasCachedMethods): #pylint: disable=abstract-method """ @@ -306,34 +296,6 @@ def as_raw(self): ('value', self.value), ('description', self.description))) - def instantiate(self, container): - return self.__class__(name=self.name, # pylint: disable=unexpected-keyword-arg - type_name=self.type_name, - _value=self._value, - description=self.description) - - def coerce_values(self, report_issues): - value = self._value - if value is not None: - evaluation = functions.evaluate(value, self, report_issues) - if (evaluation is not None) and evaluation.final: - # A final evaluation can safely replace the existing value - self._value = evaluation.value - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.type_name is not None: - console.puts('{0}: {1} ({2})'.format( - context.style.property(self.name), - context.style.literal(formatting.as_raw(self.value)), - context.style.type(self.type_name))) - else: - console.puts('{0}: {1}'.format( - context.style.property(self.name), - context.style.literal(formatting.as_raw(self.value)))) - if self.description: - console.puts(context.style.meta(self.description)) - @property def unwrapped(self): return self.name, self.value diff --git a/aria/modeling/service_common.py b/aria/modeling/service_common.py index 4ce9daef..478e5305 100644 --- a/aria/modeling/service_common.py +++ b/aria/modeling/service_common.py @@ -26,11 +26,9 @@ ) from sqlalchemy.ext.declarative import declared_attr -from ..parser.consumption import ConsumptionContext from ..utils import ( collections, formatting, - console, ) from .mixins import InstanceModelMixin, TemplateModelMixin, ParameterMixin from . import relationship @@ -563,17 +561,6 @@ def as_raw_all(self): self._append_raw_children(types) return types - def coerce_values(self, report_issues): - pass - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.name: - console.puts(context.style.type(self.name)) - with context.style.indent: - for child in self.children: - child.dump() - def _append_raw_children(self, types): for child in self.children: raw_child = formatting.as_raw(child) @@ -612,17 +599,3 @@ def as_raw(self): return collections.OrderedDict(( ('name', self.name), ('value', self.value))) - - def coerce_values(self, report_issues): - pass - - def instantiate(self, container): - from . import models - return models.Metadata(name=self.name, - value=self.value) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('{0}: {1}'.format( - context.style.property(self.name), - context.style.literal(self.value))) diff --git a/aria/modeling/service_instance.py b/aria/modeling/service_instance.py index ae6a0a06..daeb0a4f 100644 --- a/aria/modeling/service_instance.py +++ b/aria/modeling/service_instance.py @@ -30,19 +30,15 @@ from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.orderinglist import ordering_list +from . import ( + relationship, + types as modeling_types +) from .mixins import InstanceModelMixin -from ..orchestrator import execution_plugin -from ..parser import validation -from ..parser.consumption import ConsumptionContext + from ..utils import ( collections, formatting, - console -) -from . import ( - relationship, - utils, - types as modeling_types ) @@ -232,50 +228,6 @@ def service_template_fk(cls): :type: :class:`~datetime.datetime` """) - def satisfy_requirements(self): - satisfied = True - for node in self.nodes.itervalues(): - if not node.satisfy_requirements(): - satisfied = False - return satisfied - - def validate_capabilities(self): - satisfied = True - for node in self.nodes.itervalues(): - if not node.validate_capabilities(): - satisfied = False - return satisfied - - def find_hosts(self): - for node in self.nodes.itervalues(): - node.find_host() - - def configure_operations(self): - for node in self.nodes.itervalues(): - node.configure_operations() - for group in self.groups.itervalues(): - group.configure_operations() - for operation in self.workflows.itervalues(): - operation.configure() - - def is_node_a_target(self, target_node): - for node in self.nodes.itervalues(): - if self._is_node_a_target(node, target_node): - return True - return False - - def _is_node_a_target(self, source_node, target_node): - if source_node.outbound_relationships: - for relationship_model in source_node.outbound_relationships: - if relationship_model.target_node.name == target_node.name: - return True - else: - node = relationship_model.target_node - if node is not None: - if self._is_node_a_target(node, target_node): - return True - return False - @property def as_raw(self): return collections.OrderedDict(( @@ -289,70 +241,6 @@ def as_raw(self): ('outputs', formatting.as_raw_dict(self.outputs)), ('workflows', formatting.as_raw_list(self.workflows)))) - def validate(self): - utils.validate_dict_values(self.meta_data) - utils.validate_dict_values(self.nodes) - utils.validate_dict_values(self.groups) - utils.validate_dict_values(self.policies) - if self.substitution is not None: - self.substitution.validate() - utils.validate_dict_values(self.inputs) - utils.validate_dict_values(self.outputs) - utils.validate_dict_values(self.workflows) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.meta_data, report_issues) - utils.coerce_dict_values(self.nodes, report_issues) - utils.coerce_dict_values(self.groups, report_issues) - utils.coerce_dict_values(self.policies, report_issues) - if self.substitution is not None: - self.substitution.coerce_values(report_issues) - utils.coerce_dict_values(self.inputs, report_issues) - utils.coerce_dict_values(self.outputs, report_issues) - utils.coerce_dict_values(self.workflows, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.description is not None: - console.puts(context.style.meta(self.description)) - utils.dump_dict_values(self.meta_data, 'Metadata') - for node in self.nodes.itervalues(): - node.dump() - for group in self.groups.itervalues(): - group.dump() - for policy in self.policies.itervalues(): - policy.dump() - if self.substitution is not None: - self.substitution.dump() - utils.dump_dict_values(self.inputs, 'Inputs') - utils.dump_dict_values(self.outputs, 'Outputs') - utils.dump_dict_values(self.workflows, 'Workflows') - - def dump_graph(self): - for node in self.nodes.itervalues(): - if not self.is_node_a_target(node): - self._dump_graph_node(node) - - def _dump_graph_node(self, node, capability=None): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(node.name)) - if capability is not None: - console.puts('{0} ({1})'.format(context.style.property(capability.name), - context.style.type(capability.type.name))) - if node.outbound_relationships: - with context.style.indent: - for relationship_model in node.outbound_relationships: - relationship_name = context.style.property(relationship_model.name) - if relationship_model.type is not None: - console.puts('-> {0} ({1})'.format(relationship_name, - context.style.type( - relationship_model.type.name))) - else: - console.puts('-> {0}'.format(relationship_name)) - with console.indent(3): - self._dump_graph_node(relationship_model.target_node, - relationship_model.target_capability) - class NodeBase(InstanceModelMixin): """ @@ -616,118 +504,6 @@ def host_address(self): return attribute.value if attribute else None return None - def satisfy_requirements(self): - node_template = self.node_template - satisfied = True - for requirement_template in node_template.requirement_templates: - # Find target template - target_node_template, target_node_capability = \ - requirement_template.find_target(node_template) - if target_node_template is not None: - satisfied = self._satisfy_capability(target_node_capability, - target_node_template, - requirement_template) - else: - context = ConsumptionContext.get_thread_local() - context.validation.report('requirement "{0}" of node "{1}" has no target node ' - 'template'.format(requirement_template.name, self.name), - level=validation.Issue.BETWEEN_INSTANCES) - satisfied = False - return satisfied - - def _satisfy_capability(self, target_node_capability, target_node_template, - requirement_template): - from . import models - context = ConsumptionContext.get_thread_local() - # Find target nodes - target_nodes = target_node_template.nodes - if target_nodes: - target_node = None - target_capability = None - - if target_node_capability is not None: - # Relate to the first target node that has capacity - for node in target_nodes: - a_target_capability = node.capabilities.get(target_node_capability.name) - if a_target_capability.relate(): - target_node = node - target_capability = a_target_capability - break - else: - # Use first target node - target_node = target_nodes[0] - - if target_node is not None: - if requirement_template.relationship_template is not None: - relationship_model = \ - requirement_template.relationship_template.instantiate(self) - else: - relationship_model = models.Relationship() - relationship_model.name = requirement_template.name - relationship_model.requirement_template = requirement_template - relationship_model.target_node = target_node - relationship_model.target_capability = target_capability - self.outbound_relationships.append(relationship_model) - return True - else: - context.validation.report('requirement "{0}" of node "{1}" targets node ' - 'template "{2}" but its instantiated nodes do not ' - 'have enough capacity'.format( - requirement_template.name, - self.name, - target_node_template.name), - level=validation.Issue.BETWEEN_INSTANCES) - return False - else: - context.validation.report('requirement "{0}" of node "{1}" targets node template ' - '"{2}" but it has no instantiated nodes'.format( - requirement_template.name, - self.name, - target_node_template.name), - level=validation.Issue.BETWEEN_INSTANCES) - return False - - def validate_capabilities(self): - context = ConsumptionContext.get_thread_local() - satisfied = False - for capability in self.capabilities.itervalues(): - if not capability.has_enough_relationships: - context.validation.report('capability "{0}" of node "{1}" requires at least {2:d} ' - 'relationships but has {3:d}'.format( - capability.name, - self.name, - capability.min_occurrences, - capability.occurrences), - level=validation.Issue.BETWEEN_INSTANCES) - satisfied = False - return satisfied - - def find_host(self): - def _find_host(node): - if node.type.role == 'host': - return node - for the_relationship in node.outbound_relationships: - if (the_relationship.target_capability is not None) and \ - the_relationship.target_capability.type.role == 'host': - host = _find_host(the_relationship.target_node) - if host is not None: - return host - for the_relationship in node.inbound_relationships: - if (the_relationship.target_capability is not None) and \ - the_relationship.target_capability.type.role == 'feature': - host = _find_host(the_relationship.source_node) - if host is not None: - return host - return None - - self.host = _find_host(self) - - def configure_operations(self): - for interface in self.interfaces.itervalues(): - interface.configure_operations() - for the_relationship in self.outbound_relationships: - the_relationship.configure_operations() - @property def as_raw(self): return collections.OrderedDict(( @@ -740,46 +516,6 @@ def as_raw(self): ('capabilities', formatting.as_raw_list(self.capabilities)), ('relationships', formatting.as_raw_list(self.outbound_relationships)))) - def validate(self): - context = ConsumptionContext.get_thread_local() - if len(self.name) > context.modeling.id_max_length: - context.validation.report('"{0}" has an ID longer than the limit of {1:d} characters: ' - '{2:d}'.format( - self.name, - context.modeling.id_max_length, - len(self.name)), - level=validation.Issue.BETWEEN_INSTANCES) - - # TODO: validate that node template is of type? - - utils.validate_dict_values(self.properties) - utils.validate_dict_values(self.attributes) - utils.validate_dict_values(self.interfaces) - utils.validate_dict_values(self.artifacts) - utils.validate_dict_values(self.capabilities) - utils.validate_list_values(self.outbound_relationships) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - utils.coerce_dict_values(self.attributes, report_issues) - utils.coerce_dict_values(self.interfaces, report_issues) - utils.coerce_dict_values(self.artifacts, report_issues) - utils.coerce_dict_values(self.capabilities, report_issues) - utils.coerce_list_values(self.outbound_relationships, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Node: {0}'.format(context.style.node(self.name))) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - console.puts('Template: {0}'.format(context.style.node(self.node_template.name))) - utils.dump_dict_values(self.properties, 'Properties') - utils.dump_dict_values(self.attributes, 'Attributes') - utils.dump_interfaces(self.interfaces) - utils.dump_dict_values(self.artifacts, 'Artifacts') - utils.dump_dict_values(self.capabilities, 'Capabilities') - utils.dump_list_values(self.outbound_relationships, 'Relationships') - class GroupBase(InstanceModelMixin): """ @@ -885,10 +621,6 @@ def group_template_fk(cls): :type: :obj:`basestring` """) - def configure_operations(self): - for interface in self.interfaces.itervalues(): - interface.configure_operations() - @property def as_raw(self): return collections.OrderedDict(( @@ -896,27 +628,6 @@ def as_raw(self): ('properties', formatting.as_raw_dict(self.properties)), ('interfaces', formatting.as_raw_list(self.interfaces)))) - def validate(self): - utils.validate_dict_values(self.properties) - utils.validate_dict_values(self.interfaces) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - utils.coerce_dict_values(self.interfaces, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Group: {0}'.format(context.style.node(self.name))) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - utils.dump_dict_values(self.properties, 'Properties') - utils.dump_interfaces(self.interfaces) - if self.nodes: - console.puts('Member nodes:') - with context.style.indent: - for node in self.nodes: - console.puts(context.style.node(node.name)) - class PolicyBase(InstanceModelMixin): """ @@ -1030,29 +741,6 @@ def as_raw(self): ('type_name', self.type.name), ('properties', formatting.as_raw_dict(self.properties)))) - def validate(self): - utils.validate_dict_values(self.properties) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Policy: {0}'.format(context.style.node(self.name))) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - utils.dump_dict_values(self.properties, 'Properties') - if self.nodes: - console.puts('Target nodes:') - with context.style.indent: - for node in self.nodes: - console.puts(context.style.node(node.name)) - if self.groups: - console.puts('Target groups:') - with context.style.indent: - for group in self.groups: - console.puts(context.style.node(group.name)) - class SubstitutionBase(InstanceModelMixin): """ @@ -1130,19 +818,6 @@ def as_raw(self): ('node_type_name', self.node_type.name), ('mappings', formatting.as_raw_dict(self.mappings)))) - def validate(self): - utils.validate_dict_values(self.mappings) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.mappings, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Substitution:') - with context.style.indent: - console.puts('Node type: {0}'.format(context.style.type(self.node_type.name))) - utils.dump_dict_values(self.mappings, 'Mappings') - class SubstitutionMappingBase(InstanceModelMixin): """ @@ -1238,31 +913,6 @@ def as_raw(self): return collections.OrderedDict(( ('name', self.name),)) - def coerce_values(self, report_issues): - pass - - def validate(self): - context = ConsumptionContext.get_thread_local() - if (self.capability is None) and (self.requirement_template is None): - context.validation.report('mapping "{0}" refers to neither capability nor a requirement' - ' in node: {1}'.format( - self.name, - formatting.safe_repr(self.node.name)), - level=validation.Issue.BETWEEN_TYPES) - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.capability is not None: - console.puts('{0} -> {1}.{2}'.format( - context.style.node(self.name), - context.style.node(self.capability.node.name), - context.style.node(self.capability.name))) - else: - console.puts('{0} -> {1}.{2}'.format( - context.style.node(self.name), - context.style.node(self.node.name), - context.style.node(self.requirement_template.name))) - class RelationshipBase(InstanceModelMixin): """ @@ -1436,10 +1086,6 @@ def relationship_template_fk(cls): :type: :obj:`int` """) - def configure_operations(self): - for interface in self.interfaces.itervalues(): - interface.configure_operations() - @property def as_raw(self): return collections.OrderedDict(( @@ -1452,33 +1098,6 @@ def as_raw(self): ('properties', formatting.as_raw_dict(self.properties)), ('interfaces', formatting.as_raw_list(self.interfaces)))) - def validate(self): - utils.validate_dict_values(self.properties) - utils.validate_dict_values(self.interfaces) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - utils.coerce_dict_values(self.interfaces, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.name: - console.puts('{0} ->'.format(context.style.node(self.name))) - else: - console.puts('->') - with context.style.indent: - console.puts('Node: {0}'.format(context.style.node(self.target_node.name))) - if self.target_capability: - console.puts('Capability: {0}'.format(context.style.node( - self.target_capability.name))) - if self.type is not None: - console.puts('Relationship type: {0}'.format(context.style.type(self.type.name))) - if (self.relationship_template is not None) and self.relationship_template.name: - console.puts('Relationship template: {0}'.format( - context.style.node(self.relationship_template.name))) - utils.dump_dict_values(self.properties, 'Properties') - utils.dump_interfaces(self.interfaces, 'Interfaces') - class CapabilityBase(InstanceModelMixin): """ @@ -1595,25 +1214,6 @@ def as_raw(self): ('type_name', self.type.name), ('properties', formatting.as_raw_dict(self.properties)))) - def validate(self): - utils.validate_dict_values(self.properties) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - console.puts('Occurrences: {0:d} ({1:d}{2})'.format( - self.occurrences, - self.min_occurrences or 0, - ' to {0:d}'.format(self.max_occurrences) - if self.max_occurrences is not None - else ' or more')) - utils.dump_dict_values(self.properties, 'Properties') - class InterfaceBase(InstanceModelMixin): """ @@ -1738,10 +1338,6 @@ def interface_template_fk(cls): :type: :obj:`basestring` """) - def configure_operations(self): - for operation in self.operations.itervalues(): - operation.configure() - @property def as_raw(self): return collections.OrderedDict(( @@ -1751,24 +1347,6 @@ def as_raw(self): ('inputs', formatting.as_raw_dict(self.inputs)), ('operations', formatting.as_raw_list(self.operations)))) - def validate(self): - utils.validate_dict_values(self.inputs) - utils.validate_dict_values(self.operations) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.inputs, report_issues) - utils.coerce_dict_values(self.operations, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Interface type: {0}'.format(context.style.type(self.type.name))) - utils.dump_dict_values(self.inputs, 'Inputs') - utils.dump_dict_values(self.operations, 'Operations') - class OperationBase(InstanceModelMixin): """ @@ -1944,45 +1522,6 @@ def operation_template_fk(cls): :type: :obj:`float` """) - def configure(self): - if (self.implementation is None) and (self.function is None): - return - - if (self.interface is not None) and (self.plugin is None) and (self.function is None): - # ("interface" is None for workflow operations, which do not currently use "plugin") - # The default (None) plugin is the execution plugin - execution_plugin.instantiation.configure_operation(self) - else: - # In the future plugins may be able to add their own "configure_operation" hook that - # can validate the configuration and otherwise create specially derived arguments. For - # now, we just send all configuration parameters as arguments without validation. - utils.instantiate_dict(self, self.arguments, - utils.dict_as_arguments(self.configurations)) - - if self.interface is not None: - # Send all interface inputs as extra arguments - # ("interface" is None for workflow operations) - # Note that they will override existing arguments of the same names - utils.instantiate_dict(self, self.arguments, - utils.dict_as_arguments(self.interface.inputs)) - - # Send all inputs as extra arguments - # Note that they will override existing arguments of the same names - utils.instantiate_dict(self, self.arguments, utils.dict_as_arguments(self.inputs)) - - # Check for reserved arguments - from ..orchestrator.decorators import OPERATION_DECORATOR_RESERVED_ARGUMENTS - used_reserved_names = \ - OPERATION_DECORATOR_RESERVED_ARGUMENTS.intersection(self.arguments.keys()) - if used_reserved_names: - context = ConsumptionContext.get_thread_local() - context.validation.report('using reserved arguments in operation "{0}": {1}' - .format( - self.name, - formatting.string_list_as_string(used_reserved_names)), - level=validation.Issue.EXTERNAL) - - @property def as_raw(self): return collections.OrderedDict(( @@ -1992,46 +1531,6 @@ def as_raw(self): ('dependencies', self.dependencies), ('inputs', formatting.as_raw_dict(self.inputs)))) - def validate(self): - # TODO must be associated with either interface or service - utils.validate_dict_values(self.inputs) - utils.validate_dict_values(self.configurations) - utils.validate_dict_values(self.arguments) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.inputs, report_issues) - utils.coerce_dict_values(self.configurations, report_issues) - utils.coerce_dict_values(self.arguments, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - if self.implementation is not None: - console.puts('Implementation: {0}'.format( - context.style.literal(self.implementation))) - if self.dependencies: - console.puts( - 'Dependencies: {0}'.format( - ', '.join((str(context.style.literal(v)) for v in self.dependencies)))) - utils.dump_dict_values(self.inputs, 'Inputs') - if self.executor is not None: - console.puts('Executor: {0}'.format(context.style.literal(self.executor))) - if self.max_attempts is not None: - console.puts('Max attempts: {0}'.format(context.style.literal(self.max_attempts))) - if self.retry_interval is not None: - console.puts('Retry interval: {0}'.format( - context.style.literal(self.retry_interval))) - if self.plugin is not None: - console.puts('Plugin: {0}'.format( - context.style.literal(self.plugin.name))) - utils.dump_dict_values(self.configurations, 'Configuration') - if self.function is not None: - console.puts('Function: {0}'.format(context.style.literal(self.function))) - utils.dump_dict_values(self.arguments, 'Arguments') - class ArtifactBase(InstanceModelMixin): """ @@ -2150,27 +1649,3 @@ def as_raw(self): ('repository_url', self.repository_url), ('repository_credential', formatting.as_agnostic(self.repository_credential)), ('properties', formatting.as_raw_dict(self.properties)))) - - def validate(self): - utils.validate_dict_values(self.properties) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Artifact type: {0}'.format(context.style.type(self.type.name))) - console.puts('Source path: {0}'.format(context.style.literal(self.source_path))) - if self.target_path is not None: - console.puts('Target path: {0}'.format(context.style.literal(self.target_path))) - if self.repository_url is not None: - console.puts('Repository URL: {0}'.format( - context.style.literal(self.repository_url))) - if self.repository_credential: - console.puts('Repository credential: {0}'.format( - context.style.literal(self.repository_credential))) - utils.dump_dict_values(self.properties, 'Properties') diff --git a/aria/modeling/service_template.py b/aria/modeling/service_template.py index 2246340d..d988d211 100644 --- a/aria/modeling/service_template.py +++ b/aria/modeling/service_template.py @@ -21,8 +21,6 @@ from __future__ import absolute_import # so we can import standard 'types' -from datetime import datetime - from sqlalchemy import ( Column, Text, @@ -33,15 +31,10 @@ ) from sqlalchemy.ext.declarative import declared_attr -from ..parser import validation -from ..parser.consumption import ConsumptionContext -from ..parser.reading import deepcopy_with_locators -from ..utils import (collections, formatting, console) -from ..utils.versions import VersionString +from ..utils import (collections, formatting) from .mixins import TemplateModelMixin from . import ( relationship, - utils, types as modeling_types ) @@ -331,130 +324,6 @@ def types_as_raw(self): ('interface_types', formatting.as_raw(self.interface_types)), ('artifact_types', formatting.as_raw(self.artifact_types)))) - def instantiate(self, container, model_storage, inputs=None): # pylint: disable=arguments-differ - from . import models - now = datetime.now() - service = models.Service(created_at=now, - updated_at=now, - description=deepcopy_with_locators(self.description), - service_template=self) - - # TODO: we want to remove this use of the context - context = ConsumptionContext.get_thread_local() - context.modeling.instance = service - - utils.validate_no_undeclared_inputs(declared_inputs=self.inputs, - supplied_inputs=inputs or {}) - utils.validate_required_inputs_are_supplied(declared_inputs=self.inputs, - supplied_inputs=inputs or {}) - - service.inputs = utils.merge_parameter_values(inputs, self.inputs, model_cls=models.Input) - # TODO: now that we have inputs, we should scan properties and inputs and evaluate functions - - for plugin_specification in self.plugin_specifications.itervalues(): - if plugin_specification.enabled: - if plugin_specification.resolve(model_storage): - plugin = plugin_specification.plugin - service.plugins[plugin.name] = plugin - else: - context = ConsumptionContext.get_thread_local() - context.validation.report('specified plugin not found: {0}'.format( - plugin_specification.name), level=validation.Issue.EXTERNAL) - - utils.instantiate_dict(self, service.meta_data, self.meta_data) - - for node_template in self.node_templates.itervalues(): - for _ in range(node_template.scaling['default_instances']): - node = node_template.instantiate(container) - service.nodes[node.name] = node - - utils.instantiate_dict(self, service.groups, self.group_templates) - utils.instantiate_dict(self, service.policies, self.policy_templates) - utils.instantiate_dict(self, service.workflows, self.workflow_templates) - - if self.substitution_template is not None: - service.substitution = self.substitution_template.instantiate(container) - - utils.instantiate_dict(self, service.outputs, self.outputs) - - return service - - def validate(self): - utils.validate_dict_values(self.meta_data) - utils.validate_dict_values(self.node_templates) - utils.validate_dict_values(self.group_templates) - utils.validate_dict_values(self.policy_templates) - if self.substitution_template is not None: - self.substitution_template.validate() - utils.validate_dict_values(self.inputs) - utils.validate_dict_values(self.outputs) - utils.validate_dict_values(self.workflow_templates) - if self.node_types is not None: - self.node_types.validate() - if self.group_types is not None: - self.group_types.validate() - if self.policy_types is not None: - self.policy_types.validate() - if self.relationship_types is not None: - self.relationship_types.validate() - if self.capability_types is not None: - self.capability_types.validate() - if self.interface_types is not None: - self.interface_types.validate() - if self.artifact_types is not None: - self.artifact_types.validate() - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.meta_data, report_issues) - utils.coerce_dict_values(self.node_templates, report_issues) - utils.coerce_dict_values(self.group_templates, report_issues) - utils.coerce_dict_values(self.policy_templates, report_issues) - if self.substitution_template is not None: - self.substitution_template.coerce_values(report_issues) - utils.coerce_dict_values(self.inputs, report_issues) - utils.coerce_dict_values(self.outputs, report_issues) - utils.coerce_dict_values(self.workflow_templates, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.description is not None: - console.puts(context.style.meta(self.description)) - utils.dump_dict_values(self.meta_data, 'Metadata') - for node_template in self.node_templates.itervalues(): - node_template.dump() - for group_template in self.group_templates.itervalues(): - group_template.dump() - for policy_template in self.policy_templates.itervalues(): - policy_template.dump() - if self.substitution_template is not None: - self.substitution_template.dump() - utils.dump_dict_values(self.inputs, 'Inputs') - utils.dump_dict_values(self.outputs, 'Outputs') - utils.dump_dict_values(self.workflow_templates, 'Workflow templates') - - def dump_types(self): - if self.node_types.children: - console.puts('Node types:') - self.node_types.dump() - if self.group_types.children: - console.puts('Group types:') - self.group_types.dump() - if self.capability_types.children: - console.puts('Capability types:') - self.capability_types.dump() - if self.relationship_types.children: - console.puts('Relationship types:') - self.relationship_types.dump() - if self.policy_types.children: - console.puts('Policy types:') - self.policy_types.dump() - if self.artifact_types.children: - console.puts('Artifact types:') - self.artifact_types.dump() - if self.interface_types.children: - console.puts('Interface types:') - self.interface_types.dump() - class NodeTemplateBase(TemplateModelMixin): """ @@ -625,114 +494,6 @@ def as_raw(self): ('capability_templates', formatting.as_raw_list(self.capability_templates)), ('requirement_templates', formatting.as_raw_list(self.requirement_templates)))) - def instantiate(self, container): - from . import models - node = models.Node(name=self._next_name, - type=self.type, - description=deepcopy_with_locators(self.description), - state=models.Node.INITIAL, - node_template=self) - utils.instantiate_dict(node, node.properties, self.properties) - utils.instantiate_dict(node, node.attributes, self.attributes) - utils.instantiate_dict(node, node.interfaces, self.interface_templates) - utils.instantiate_dict(node, node.artifacts, self.artifact_templates) - utils.instantiate_dict(node, node.capabilities, self.capability_templates) - - # Default attributes - if ('tosca_name' in node.attributes) \ - and (node.attributes['tosca_name'].type_name == 'string'): - node.attributes['tosca_name'].value = self.name - if 'tosca_id' in node.attributes \ - and (node.attributes['tosca_id'].type_name == 'string'): - node.attributes['tosca_id'].value = node.name - - return node - - def validate(self): - utils.validate_dict_values(self.properties) - utils.validate_dict_values(self.attributes) - utils.validate_dict_values(self.interface_templates) - utils.validate_dict_values(self.artifact_templates) - utils.validate_dict_values(self.capability_templates) - utils.validate_list_values(self.requirement_templates) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - utils.coerce_dict_values(self.attributes, report_issues) - utils.coerce_dict_values(self.interface_templates, report_issues) - utils.coerce_dict_values(self.artifact_templates, report_issues) - utils.coerce_dict_values(self.capability_templates, report_issues) - utils.coerce_list_values(self.requirement_templates, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Node template: {0}'.format(context.style.node(self.name))) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - utils.dump_dict_values(self.properties, 'Properties') - utils.dump_dict_values(self.attributes, 'Attributes') - utils.dump_interfaces(self.interface_templates) - utils.dump_dict_values(self.artifact_templates, 'Artifact templates') - utils.dump_dict_values(self.capability_templates, 'Capability templates') - utils.dump_list_values(self.requirement_templates, 'Requirement templates') - - @property - def scaling(self): - scaling = {} - - def extract_property(properties, name): - if name in scaling: - return - prop = properties.get(name) - if (prop is not None) and (prop.type_name == 'integer') and (prop.value is not None): - scaling[name] = prop.value - - def extract_properties(properties): - extract_property(properties, 'min_instances') - extract_property(properties, 'max_instances') - extract_property(properties, 'default_instances') - - def default_property(name, value): - if name not in scaling: - scaling[name] = value - - # From our scaling capabilities - for capability_template in self.capability_templates.itervalues(): - if capability_template.type.role == 'scaling': - extract_properties(capability_template.properties) - - # From service scaling policies - for policy_template in self.service_template.policy_templates.itervalues(): - if policy_template.type.role == 'scaling': - if policy_template.is_for_node_template(self.name): - extract_properties(policy_template.properties) - - # Defaults - default_property('min_instances', 0) - default_property('max_instances', 1) - default_property('default_instances', 1) - - # Validate - # pylint: disable=too-many-boolean-expressions - if ((scaling['min_instances'] < 0) or - (scaling['max_instances'] < 0) or - (scaling['default_instances'] < 0) or - (scaling['max_instances'] < scaling['min_instances']) or - (scaling['default_instances'] < scaling['min_instances']) or - (scaling['default_instances'] > scaling['max_instances'])): - context = ConsumptionContext.get_thread_local() - context.validation.report('invalid scaling parameters for node template "{0}": ' - 'min={1}, max={2}, default={3}'.format( - self.name, - scaling['min_instances'], - scaling['max_instances'], - scaling['default_instances']), - level=validation.Issue.BETWEEN_TYPES) - - return scaling - def is_target_node_template_valid(self, target_node_template): """ Checks if ``target_node_template`` matches all our ``target_node_template_constraints``. @@ -769,6 +530,40 @@ def _next_name(self): return '{name}_{index}'.format(name=self.name, index=self._next_index) + @property + def scaling(self): + scaling = {} + + def extract_property(properties, name): + if name in scaling: + return + prop = properties.get(name) + if (prop is not None) and (prop.type_name == 'integer') and (prop.value is not None): + scaling[name] = prop.value + + def extract_properties(properties): + extract_property(properties, 'min_instances') + extract_property(properties, 'max_instances') + extract_property(properties, 'default_instances') + + # From our scaling capabilities + for capability_template in self.capability_templates.itervalues(): + if capability_template.type.role == 'scaling': + extract_properties(capability_template.properties) + + # From service scaling policies + for policy_template in self.service_template.policy_templates.itervalues(): + if policy_template.type.role == 'scaling': + if policy_template.is_for_node_template(self.name): + extract_properties(policy_template.properties) + + # Defaults + scaling.setdefault('min_instances', 0) + scaling.setdefault('max_instances', 1) + scaling.setdefault('default_instances', 1) + + return scaling + class GroupTemplateBase(TemplateModelMixin): """ @@ -876,40 +671,6 @@ def as_raw(self): ('properties', formatting.as_raw_dict(self.properties)), ('interface_templates', formatting.as_raw_list(self.interface_templates)))) - def instantiate(self, container): - from . import models - group = models.Group(name=self.name, - type=self.type, - description=deepcopy_with_locators(self.description), - group_template=self) - utils.instantiate_dict(self, group.properties, self.properties) - utils.instantiate_dict(self, group.interfaces, self.interface_templates) - if self.node_templates: - for node_template in self.node_templates: - group.nodes += node_template.nodes - return group - - def validate(self): - utils.validate_dict_values(self.properties) - utils.validate_dict_values(self.interface_templates) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - utils.coerce_dict_values(self.interface_templates, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Group template: {0}'.format(context.style.node(self.name))) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - utils.dump_dict_values(self.properties, 'Properties') - utils.dump_interfaces(self.interface_templates) - if self.node_templates: - console.puts('Member node templates: {0}'.format(', '.join( - (str(context.style.node(v.name)) for v in self.node_templates)))) - def contains_node_template(self, name): for node_template in self.node_templates: if node_template.name == name: @@ -1022,42 +783,6 @@ def as_raw(self): ('type_name', self.type.name), ('properties', formatting.as_raw_dict(self.properties)))) - def instantiate(self, container): - from . import models - policy = models.Policy(name=self.name, - type=self.type, - description=deepcopy_with_locators(self.description), - policy_template=self) - utils.instantiate_dict(self, policy.properties, self.properties) - if self.node_templates: - for node_template in self.node_templates: - policy.nodes += node_template.nodes - if self.group_templates: - for group_template in self.group_templates: - policy.groups += group_template.groups - return policy - - def validate(self): - utils.validate_dict_values(self.properties) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Policy template: {0}'.format(context.style.node(self.name))) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - utils.dump_dict_values(self.properties, 'Properties') - if self.node_templates: - console.puts('Target node templates: {0}'.format(', '.join( - (str(context.style.node(v.name)) for v in self.node_templates)))) - if self.group_templates: - console.puts('Target group templates: {0}'.format(', '.join( - (str(context.style.node(v.name)) for v in self.group_templates)))) - def is_for_node_template(self, name): for node_template in self.node_templates: if node_template.name == name: @@ -1134,26 +859,6 @@ def as_raw(self): ('node_type_name', self.node_type.name), ('mappings', formatting.as_raw_dict(self.mappings)))) - def instantiate(self, container): - from . import models - substitution = models.Substitution(node_type=self.node_type, - substitution_template=self) - utils.instantiate_dict(container, substitution.mappings, self.mappings) - return substitution - - def validate(self): - utils.validate_dict_values(self.mappings) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.mappings, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts('Substitution template:') - with context.style.indent: - console.puts('Node type: {0}'.format(context.style.type(self.node_type.name))) - utils.dump_dict_values(self.mappings, 'Mappings') - class SubstitutionTemplateMappingBase(TemplateModelMixin): """ @@ -1233,59 +938,6 @@ def as_raw(self): return collections.OrderedDict(( ('name', self.name),)) - def coerce_values(self, report_issues): - pass - - def instantiate(self, container): - from . import models - context = ConsumptionContext.get_thread_local() - if self.capability_template is not None: - node_template = self.capability_template.node_template - else: - node_template = self.requirement_template.node_template - nodes = node_template.nodes - if len(nodes) == 0: - context.validation.report( - 'mapping "{0}" refers to node template "{1}" but there are no ' - 'node instances'.format(self.mapped_name, self.node_template.name), - level=validation.Issue.BETWEEN_INSTANCES) - return None - # The TOSCA spec does not provide a way to choose the node, - # so we will just pick the first one - node = nodes[0] - capability = None - if self.capability_template: - for a_capability in node.capabilities.itervalues(): - if a_capability.capability_template.name == self.capability_template.name: - capability = a_capability - return models.SubstitutionMapping(name=self.name, - capability=capability, - requirement_template=self.requirement_template, - node=node) - - - def validate(self): - context = ConsumptionContext.get_thread_local() - if (self.capability_template is None) and (self.requirement_template is None): - context.validation.report('mapping "{0}" refers to neither capability nor a requirement' - ' in node template: {1}'.format( - self.name, - formatting.safe_repr(self.node_template.name)), - level=validation.Issue.BETWEEN_TYPES) - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.capability_template is not None: - node_template = self.capability_template.node_template - else: - node_template = self.requirement_template.node_template - console.puts('{0} -> {1}.{2}'.format( - context.style.node(self.name), - context.style.node(node_template.name), - context.style.node(self.capability_template.name - if self.capability_template - else self.requirement_template.name))) - class RequirementTemplateBase(TemplateModelMixin): """ @@ -1424,56 +1076,6 @@ def relationship_template_fk(cls): :type: [:class:`NodeTemplateConstraint`] """) - def find_target(self, source_node_template): - context = ConsumptionContext.get_thread_local() - - # We might already have a specific node template, so we'll just verify it - if self.target_node_template is not None: - if not source_node_template.is_target_node_template_valid(self.target_node_template): - context.validation.report('requirement "{0}" of node template "{1}" is for node ' - 'template "{2}" but it does not match constraints'.format( - self.name, - self.target_node_template.name, - source_node_template.name), - level=validation.Issue.BETWEEN_TYPES) - if (self.target_capability_type is not None) \ - or (self.target_capability_name is not None): - target_node_capability = self.find_target_capability(source_node_template, - self.target_node_template) - if target_node_capability is None: - return None, None - else: - target_node_capability = None - - return self.target_node_template, target_node_capability - - # Find first node that matches the type - elif self.target_node_type is not None: - for target_node_template in \ - self.node_template.service_template.node_templates.itervalues(): - if self.target_node_type.get_descendant(target_node_template.type.name) is None: - continue - - if not source_node_template.is_target_node_template_valid(target_node_template): - continue - - target_node_capability = self.find_target_capability(source_node_template, - target_node_template) - if target_node_capability is None: - continue - - return target_node_template, target_node_capability - - return None, None - - def find_target_capability(self, source_node_template, target_node_template): - for capability_template in target_node_template.capability_templates.itervalues(): - if capability_template.satisfies_requirement(source_node_template, - self, - target_node_template): - return capability_template - return None - @property def as_raw(self): return collections.OrderedDict(( @@ -1487,43 +1089,6 @@ def as_raw(self): ('target_capability_name', self.target_capability_name), ('relationship_template', formatting.as_raw(self.relationship_template)))) - def validate(self): - if self.relationship_template: - self.relationship_template.validate() - - def coerce_values(self, report_issues): - if self.relationship_template is not None: - self.relationship_template.coerce_values(report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.name: - console.puts(context.style.node(self.name)) - else: - console.puts('Requirement:') - with context.style.indent: - if self.target_node_type is not None: - console.puts('Target node type: {0}'.format( - context.style.type(self.target_node_type.name))) - elif self.target_node_template is not None: - console.puts('Target node template: {0}'.format( - context.style.node(self.target_node_template.name))) - if self.target_capability_type is not None: - console.puts('Target capability type: {0}'.format( - context.style.type(self.target_capability_type.name))) - elif self.target_capability_name is not None: - console.puts('Target capability name: {0}'.format( - context.style.node(self.target_capability_name))) - if self.target_node_template_constraints: - console.puts('Target node template constraints:') - with context.style.indent: - for constraint in self.target_node_template_constraints: - console.puts(context.style.literal(constraint)) - if self.relationship_template: - console.puts('Relationship:') - with context.style.indent: - self.relationship_template.dump() - class RelationshipTemplateBase(TemplateModelMixin): """ @@ -1606,37 +1171,6 @@ def as_raw(self): ('properties', formatting.as_raw_dict(self.properties)), ('interface_templates', formatting.as_raw_list(self.interface_templates)))) - def instantiate(self, container): - from . import models - relationship_model = models.Relationship(name=self.name, - type=self.type, - relationship_template=self) - utils.instantiate_dict(container, relationship_model.properties, self.properties) - utils.instantiate_dict(container, relationship_model.interfaces, self.interface_templates) - return relationship_model - - def validate(self): - # TODO: either type or name must be set - utils.validate_dict_values(self.properties) - utils.validate_dict_values(self.interface_templates) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - utils.coerce_dict_values(self.interface_templates, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - if self.type is not None: - console.puts('Relationship type: {0}'.format(context.style.type(self.type.name))) - else: - console.puts('Relationship template: {0}'.format( - context.style.node(self.name))) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - utils.dump_dict_values(self.properties, 'Properties') - utils.dump_interfaces(self.interface_templates, 'Interface templates') - class CapabilityTemplateBase(TemplateModelMixin): """ @@ -1739,29 +1273,6 @@ def node_template_fk(cls): :type: :obj:`int` """) - def satisfies_requirement(self, - source_node_template, - requirement, - target_node_template): - # Do we match the required capability type? - if requirement.target_capability_type and \ - requirement.target_capability_type.get_descendant(self.type.name) is None: - return False - - # Are we in valid_source_node_types? - if self.valid_source_node_types: - for valid_source_node_type in self.valid_source_node_types: - if valid_source_node_type.get_descendant(source_node_template.type.name) is None: - return False - - # Apply requirement constraints - if requirement.target_node_template_constraints: - for node_template_constraint in requirement.target_node_template_constraints: - if not node_template_constraint.matches(source_node_template, target_node_template): - return False - - return True - @property def as_raw(self): return collections.OrderedDict(( @@ -1773,42 +1284,6 @@ def as_raw(self): ('valid_source_node_types', [v.name for v in self.valid_source_node_types]), ('properties', formatting.as_raw_dict(self.properties)))) - def instantiate(self, container): - from . import models - capability = models.Capability(name=self.name, - type=self.type, - min_occurrences=self.min_occurrences, - max_occurrences=self.max_occurrences, - occurrences=0, - capability_template=self) - utils.instantiate_dict(container, capability.properties, self.properties) - return capability - - def validate(self): - utils.validate_dict_values(self.properties) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Type: {0}'.format(context.style.type(self.type.name))) - console.puts( - 'Occurrences: {0:d}{1}'.format( - self.min_occurrences or 0, - ' to {0:d}'.format(self.max_occurrences) - if self.max_occurrences is not None - else ' or more')) - if self.valid_source_node_types: - console.puts('Valid source node types: {0}'.format( - ', '.join((str(context.style.type(v.name)) - for v in self.valid_source_node_types)))) - utils.dump_dict_values(self.properties, 'Properties') - class InterfaceTemplateBase(TemplateModelMixin): """ @@ -1938,34 +1413,6 @@ def as_raw(self): # TODO fix self.properties reference ('operation_templates', formatting.as_raw_list(self.operation_templates)))) - def instantiate(self, container): - from . import models - interface = models.Interface(name=self.name, - type=self.type, - description=deepcopy_with_locators(self.description), - interface_template=self) - utils.instantiate_dict(container, interface.inputs, self.inputs) - utils.instantiate_dict(container, interface.operations, self.operation_templates) - return interface - - def validate(self): - utils.validate_dict_values(self.inputs) - utils.validate_dict_values(self.operation_templates) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.inputs, report_issues) - utils.coerce_dict_values(self.operation_templates, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Interface type: {0}'.format(context.style.type(self.type.name))) - utils.dump_dict_values(self.inputs, 'Inputs') - utils.dump_dict_values(self.operation_templates, 'Operation templates') - class OperationTemplateBase(TemplateModelMixin): """ @@ -2124,65 +1571,6 @@ def as_raw(self): ('dependencies', self.dependencies), ('inputs', formatting.as_raw_dict(self.inputs)))) - def instantiate(self, container): - from . import models - - plugin = self.plugin_specification.plugin \ - if (self.plugin_specification is not None) and self.plugin_specification.enabled \ - else None - - operation = models.Operation(name=self.name, - description=deepcopy_with_locators(self.description), - relationship_edge=self.relationship_edge, - implementation=self.implementation, - dependencies=self.dependencies, - executor=self.executor, - plugin=plugin, - function=self.function, - max_attempts=self.max_attempts, - retry_interval=self.retry_interval, - operation_template=self) - - utils.instantiate_dict(container, operation.inputs, self.inputs) - utils.instantiate_dict(container, operation.configurations, self.configurations) - - return operation - - def validate(self): - utils.validate_dict_values(self.inputs) - utils.validate_dict_values(self.configurations) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.inputs, report_issues) - utils.coerce_dict_values(self.configurations, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - if self.implementation is not None: - console.puts('Implementation: {0}'.format( - context.style.literal(self.implementation))) - if self.dependencies: - console.puts('Dependencies: {0}'.format( - ', '.join((str(context.style.literal(v)) for v in self.dependencies)))) - utils.dump_dict_values(self.inputs, 'Inputs') - if self.executor is not None: - console.puts('Executor: {0}'.format(context.style.literal(self.executor))) - if self.max_attempts is not None: - console.puts('Max attempts: {0}'.format(context.style.literal(self.max_attempts))) - if self.retry_interval is not None: - console.puts('Retry interval: {0}'.format( - context.style.literal(self.retry_interval))) - if self.plugin_specification is not None: - console.puts('Plugin specification: {0}'.format( - context.style.literal(self.plugin_specification.name))) - utils.dump_dict_values(self.configurations, 'Configuration') - if self.function is not None: - console.puts('Function: {0}'.format(context.style.literal(self.function))) - class ArtifactTemplateBase(TemplateModelMixin): """ @@ -2295,43 +1683,6 @@ def as_raw(self): ('repository_credential', formatting.as_agnostic(self.repository_credential)), ('properties', formatting.as_raw_dict(self.properties)))) - def instantiate(self, container): - from . import models - artifact = models.Artifact(name=self.name, - type=self.type, - description=deepcopy_with_locators(self.description), - source_path=self.source_path, - target_path=self.target_path, - repository_url=self.repository_url, - repository_credential=self.repository_credential, - artifact_template=self) - utils.instantiate_dict(container, artifact.properties, self.properties) - return artifact - - def validate(self): - utils.validate_dict_values(self.properties) - - def coerce_values(self, report_issues): - utils.coerce_dict_values(self.properties, report_issues) - - def dump(self): - context = ConsumptionContext.get_thread_local() - console.puts(context.style.node(self.name)) - if self.description: - console.puts(context.style.meta(self.description)) - with context.style.indent: - console.puts('Artifact type: {0}'.format(context.style.type(self.type.name))) - console.puts('Source path: {0}'.format(context.style.literal(self.source_path))) - if self.target_path is not None: - console.puts('Target path: {0}'.format(context.style.literal(self.target_path))) - if self.repository_url is not None: - console.puts('Repository URL: {0}'.format( - context.style.literal(self.repository_url))) - if self.repository_credential: - console.puts('Repository credential: {0}'.format( - context.style.literal(self.repository_credential))) - utils.dump_dict_values(self.properties, 'Properties') - class PluginSpecificationBase(TemplateModelMixin): """ @@ -2399,24 +1750,3 @@ def as_raw(self): ('name', self.name), ('version', self.version), ('enabled', self.enabled))) - - def coerce_values(self, report_issues): - pass - - def resolve(self, model_storage): - # TODO: we are planning a separate "instantiation" module where this will be called or - # moved to. - plugins = model_storage.plugin.list() - matching_plugins = [] - if plugins: - for plugin in plugins: - if (plugin.name == self.name) and \ - ((self.version is None) or \ - (VersionString(plugin.package_version) >= self.version)): - matching_plugins.append(plugin) - self.plugin = None - if matching_plugins: - # Return highest version of plugin - key = lambda plugin: VersionString(plugin.package_version).key - self.plugin = sorted(matching_plugins, key=key)[-1] - return self.plugin is not None diff --git a/aria/modeling/utils.py b/aria/modeling/utils.py index 274eb880..305020bb 100644 --- a/aria/modeling/utils.py +++ b/aria/modeling/utils.py @@ -22,8 +22,6 @@ from StringIO import StringIO from . import exceptions -from ..parser.consumption import ConsumptionContext -from ..utils.console import puts from ..utils.type import validate_value_type from ..utils.collections import OrderedDict from ..utils.formatting import string_list_as_string @@ -84,7 +82,7 @@ def validate_required_inputs_are_supplied(declared_inputs, supplied_inputs): .format(string_list_as_string(missing_required_inputs))) -def merge_parameter_values(provided_values, declared_parameters, model_cls): +def merge_parameter_values(provided_values, declared_parameters, model_cls=None): """ Merges parameter values according to those declared by a type. @@ -109,6 +107,7 @@ def merge_parameter_values(provided_values, declared_parameters, model_cls): provided_values = provided_values or {} provided_values_of_wrong_type = OrderedDict() model_parameters = OrderedDict() + model_cls = model_cls or _get_class_from_sql_relationship(declared_parameters) for declared_parameter_name, declared_parameter in declared_parameters.iteritems(): if declared_parameter_name in provided_values: @@ -125,14 +124,14 @@ def merge_parameter_values(provided_values, declared_parameters, model_cls): # TODO This error shouldn't be raised (or caught), but right now we lack support # for custom data_types, which will raise this error. Skipping their validation. pass - model_parameters[declared_parameter_name] = model_cls( # pylint: disable=unexpected-keyword-arg + model_parameters[declared_parameter_name] = model_cls( # pylint: disable=unexpected-keyword-arg name=declared_parameter_name, type_name=type_name, description=declared_parameter.description, value=value) else: # Copy default value from declaration - model_parameters[declared_parameter_name] = declared_parameter.instantiate(None) + model_parameters[declared_parameter_name] = model_cls(**declared_parameter.as_raw) if provided_values_of_wrong_type: error_message = StringIO() @@ -144,76 +143,6 @@ def merge_parameter_values(provided_values, declared_parameters, model_cls): return model_parameters -def coerce_dict_values(the_dict, report_issues=False): - if not the_dict: - return - coerce_list_values(the_dict.itervalues(), report_issues) - - -def coerce_list_values(the_list, report_issues=False): - if not the_list: - return - for value in the_list: - value.coerce_values(report_issues) - - -def validate_dict_values(the_dict): - if not the_dict: - return - validate_list_values(the_dict.itervalues()) - - -def validate_list_values(the_list): - if not the_list: - return - for value in the_list: - value.validate() - - -def instantiate_dict(container, the_dict, from_dict): - if not from_dict: - return - for name, value in from_dict.iteritems(): - value = value.instantiate(container) - if value is not None: - the_dict[name] = value - - -def instantiate_list(container, the_list, from_list): - if not from_list: - return - for value in from_list: - value = value.instantiate(container) - if value is not None: - the_list.append(value) - - -def dump_list_values(the_list, name): - if not the_list: - return - puts('%s:' % name) - context = ConsumptionContext.get_thread_local() - with context.style.indent: - for value in the_list: - value.dump() - - -def dump_dict_values(the_dict, name): - if not the_dict: - return - dump_list_values(the_dict.itervalues(), name) - - -def dump_interfaces(interfaces, name='Interfaces'): - if not interfaces: - return - puts('%s:' % name) - context = ConsumptionContext.get_thread_local() - with context.style.indent: - for interface in interfaces.itervalues(): - interface.dump() - - def parameters_as_values(the_dict): return dict((k, v.value) for k, v in the_dict.iteritems()) @@ -244,3 +173,9 @@ def init(*args, **kwargs): cls.__doc__ = cls.__bases__[-1].__doc__ return cls + + +def _get_class_from_sql_relationship(field): + class_ = field._sa_adapter.owner_state.class_ + prop_name = field._sa_adapter.attr.key + return getattr(class_, prop_name).property.mapper.class_ diff --git a/aria/orchestrator/execution_plugin/instantiation.py b/aria/orchestrator/execution_plugin/instantiation.py index f55aa506..8b52015e 100644 --- a/aria/orchestrator/execution_plugin/instantiation.py +++ b/aria/orchestrator/execution_plugin/instantiation.py @@ -18,16 +18,11 @@ """ # TODO: this module will eventually be moved to a new "aria.instantiation" package - -from ...utils.type import full_type_name -from ...utils.formatting import safe_repr -from ...utils.collections import OrderedDict -from ...parser import validation -from ...parser.consumption import ConsumptionContext from ...modeling.functions import Function +from ... import utils -def configure_operation(operation): +def configure_operation(operation, reporter): host = None interface = operation.interface if interface.node is not None: @@ -38,11 +33,11 @@ def configure_operation(operation): else: # either False or None (None meaning that edge was not specified) host = interface.relationship.source_node.host - _configure_common(operation) + _configure_common(operation, reporter) if host is None: _configure_local(operation) else: - _configure_remote(operation) + _configure_remote(operation, reporter) # Any remaining un-handled configuration parameters will become extra arguments, available as # kwargs in either "run_script_locally" or "run_script_with_ssh" @@ -51,7 +46,7 @@ def configure_operation(operation): operation.arguments[key] = value.instantiate(None) -def _configure_common(operation): +def _configure_common(operation, reporter): """ Local and remote operations. """ @@ -59,7 +54,7 @@ def _configure_common(operation): from ...modeling.models import Argument operation.arguments['script_path'] = Argument.wrap('script_path', operation.implementation, 'Relative path to the executable file.') - operation.arguments['process'] = Argument.wrap('process', _get_process(operation), + operation.arguments['process'] = Argument.wrap('process', _get_process(operation, reporter), 'Sub-process configuration.') @@ -73,7 +68,7 @@ def _configure_local(operation): operations.run_script_locally.__name__) -def _configure_remote(operation): +def _configure_remote(operation, reporter): """ Remote SSH operation via Fabric. """ @@ -81,7 +76,7 @@ def _configure_remote(operation): from ...modeling.models import Argument from . import operations - ssh = _get_ssh(operation) + ssh = _get_ssh(operation, reporter) # Defaults # TODO: find a way to configure these generally in the service template @@ -110,20 +105,17 @@ def _configure_remote(operation): # Make sure we have a user if fabric_env.get('user') is None: - context = ConsumptionContext.get_thread_local() - context.validation.report('must configure "ssh.user" for "{0}"' - .format(operation.implementation), - level=validation.Issue.BETWEEN_TYPES) + reporter.report('must configure "ssh.user" for "{0}"'.format(operation.implementation), + level=reporter.Issue.BETWEEN_TYPES) # Make sure we have an authentication value if (fabric_env.get('password') is None) and \ (fabric_env.get('key') is None) and \ (fabric_env.get('key_filename') is None): - context = ConsumptionContext.get_thread_local() - context.validation.report('must configure "ssh.password", "ssh.key", or "ssh.key_filename" ' - 'for "{0}"' - .format(operation.implementation), - level=validation.Issue.BETWEEN_TYPES) + reporter.report( + 'must configure "ssh.password", "ssh.key", or "ssh.key_filename" for "{0}"' + .format(operation.implementation), + level=reporter.Issue.BETWEEN_TYPES) operation.arguments['fabric_env'] = Argument.wrap('fabric_env', fabric_env, 'Fabric configuration.') @@ -132,97 +124,94 @@ def _configure_remote(operation): operations.run_script_with_ssh.__name__) -def _get_process(operation): +def _get_process(operation, reporter): value = (operation.configurations.get('process')._value if 'process' in operation.configurations else None) if value is None: return {} - _validate_type(value, dict, 'process') - value = OrderedDict(value) + _validate_type(value, dict, 'process', reporter) + value = utils.collections.OrderedDict(value) for k, v in value.iteritems(): if k == 'eval_python': - value[k] = _coerce_bool(v, 'process.eval_python') + value[k] = _coerce_bool(v, 'process.eval_python', reporter) elif k == 'cwd': - _validate_type(v, basestring, 'process.cwd') + _validate_type(v, basestring, 'process.cwd', reporter) elif k == 'command_prefix': - _validate_type(v, basestring, 'process.command_prefix') + _validate_type(v, basestring, 'process.command_prefix', reporter) elif k == 'args': - value[k] = _dict_to_list_of_strings(v, 'process.args') + value[k] = _dict_to_list_of_strings(v, 'process.args', reporter) elif k == 'env': - _validate_type(v, dict, 'process.env') + _validate_type(v, dict, 'process.env', reporter) else: - context = ConsumptionContext.get_thread_local() - context.validation.report('unsupported configuration parameter: "process.{0}"' - .format(k), - level=validation.Issue.BETWEEN_TYPES) + reporter.report('unsupported configuration parameter: "process.{0}"'.format(k), + level=reporter.Issue.BETWEEN_TYPES) return value -def _get_ssh(operation): +def _get_ssh(operation, reporter): value = (operation.configurations.get('ssh')._value if 'ssh' in operation.configurations else None) if value is None: return {} - _validate_type(value, dict, 'ssh') - value = OrderedDict(value) + _validate_type(value, dict, 'ssh', reporter) + value = utils.collections.OrderedDict(value) for k, v in value.iteritems(): if k == 'use_sudo': - value[k] = _coerce_bool(v, 'ssh.use_sudo') + value[k] = _coerce_bool(v, 'ssh.use_sudo', reporter) elif k == 'hide_output': - value[k] = _dict_to_list_of_strings(v, 'ssh.hide_output') + value[k] = _dict_to_list_of_strings(v, 'ssh.hide_output', reporter) elif k == 'warn_only': - value[k] = _coerce_bool(v, 'ssh.warn_only') + value[k] = _coerce_bool(v, 'ssh.warn_only', reporter) elif k == 'user': - _validate_type(v, basestring, 'ssh.user') + _validate_type(v, basestring, 'ssh.user', reporter) elif k == 'password': - _validate_type(v, basestring, 'ssh.password') + _validate_type(v, basestring, 'ssh.password', reporter) elif k == 'key': - _validate_type(v, basestring, 'ssh.key') + _validate_type(v, basestring, 'ssh.key', reporter) elif k == 'key_filename': - _validate_type(v, basestring, 'ssh.key_filename') + _validate_type(v, basestring, 'ssh.key_filename', reporter) elif k == 'address': - _validate_type(v, basestring, 'ssh.address') + _validate_type(v, basestring, 'ssh.address', reporter) else: - context = ConsumptionContext.get_thread_local() - context.validation.report('unsupported configuration parameter: "ssh.{0}"'.format(k), - level=validation.Issue.BETWEEN_TYPES) + reporter.report('unsupported configuration parameter: "ssh.{0}"'.format(k), + level=reporter.Issue.BETWEEN_TYPES) return value -def _validate_type(value, the_type, name): +def _validate_type(value, the_type, name, reporter): if isinstance(value, Function): return if not isinstance(value, the_type): - context = ConsumptionContext.get_thread_local() - context.validation.report('"{0}" configuration is not a {1}: {2}' - .format(name, full_type_name(the_type), safe_repr(value)), - level=validation.Issue.BETWEEN_TYPES) + reporter.report( + '"{0}" configuration is not a {1}: {2}'.format( + name, utils.type.full_type_name(the_type), utils.formatting.safe_repr(value)), + level=reporter.Issue.BETWEEN_TYPES) -def _coerce_bool(value, name): +def _coerce_bool(value, name, reporter): if value is None: return None if isinstance(value, bool): return value - _validate_type(value, basestring, name) + _validate_type(value, basestring, name, reporter) if value == 'true': return True elif value == 'false': return False else: - context = ConsumptionContext.get_thread_local() - context.validation.report('"{0}" configuration is not "true" or "false": {1}' - .format(name, safe_repr(value)), - level=validation.Issue.BETWEEN_TYPES) + reporter.report( + '"{0}" configuration is not "true" or "false": {1}'.format( + name, utils.formatting.safe_repr(value)), + level=reporter.Issue.BETWEEN_TYPES) -def _dict_to_list_of_strings(the_dict, name): - _validate_type(the_dict, dict, name) +def _dict_to_list_of_strings(the_dict, name, reporter): + _validate_type(the_dict, dict, name, reporter) value = [] for k in sorted(the_dict): v = the_dict[k] - _validate_type(v, basestring, '{0}.{1}'.format(name, k)) + _validate_type(v, basestring, '{0}.{1}'.format(name, k), reporter) value.append(v) return value diff --git a/aria/orchestrator/topology/__init__.py b/aria/orchestrator/topology/__init__.py new file mode 100644 index 00000000..099a950c --- /dev/null +++ b/aria/orchestrator/topology/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .topology import Topology diff --git a/aria/orchestrator/topology/common.py b/aria/orchestrator/topology/common.py new file mode 100644 index 00000000..51245574 --- /dev/null +++ b/aria/orchestrator/topology/common.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class HandlerBase(object): + def __init__(self, topology, model): + self._topology = topology + self._model = model + + def coerce(self, **kwargs): + raise NotImplementedError + + def _coerce(self, *models, **kwargs): + for template in models: + self._topology.coerce(template, **kwargs) + + def validate(self, **kwargs): + raise NotImplementedError + + def _validate(self, *models, **kwargs): + for template in models: + self._topology.validate(template, **kwargs) + + def dump(self, out_stream): + raise NotImplementedError + + +class TemplateHandlerBase(HandlerBase): + """ + Base handler for template based models + """ + + def instantiate(self, instance_cls, **kwargs): + raise NotImplementedError + + +class InstanceHandlerBase(HandlerBase): + """ + Base handler for instance based models + + """ + def validate(self, **kwargs): + raise NotImplementedError + + def coerce(self, **kwargs): + raise NotImplementedError + + def dump(self, out_stream): + raise NotImplementedError + + +class ActorHandlerBase(HandlerBase): + """ + Base handler for any model which has (or contains a field which references) an operation + """ + def configure_operations(self): + raise NotImplementedError diff --git a/aria/orchestrator/topology/instance_handler.py b/aria/orchestrator/topology/instance_handler.py new file mode 100644 index 00000000..51f26c63 --- /dev/null +++ b/aria/orchestrator/topology/instance_handler.py @@ -0,0 +1,671 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ... parser.modeling import context +from ... modeling import models, functions +from ... utils import formatting +from .. import execution_plugin +from .. import decorators +from . import common + + +class Artifact(common.InstanceHandlerBase): + + def coerce(self, **kwargs): + self._topology.coerce(self._model.properties, **kwargs) + + def validate(self, **kwargs): + self._topology.validate(self._model.properties, **kwargs) + + def dump(self, out_stream): + with out_stream.indent(): + out_stream.write(out_stream.node_style(self._model.name)) + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + out_stream.write('Artifact type: {0}'.format(out_stream.type_style( + self._model.type.name))) + out_stream.write('Source path: {0}'.format( + out_stream.literal_style(self._model.source_path))) + if self._model.target_path is not None: + out_stream.write('Target path: {0}'.format( + out_stream.literal_style(self._model.target_path))) + if self._model.repository_url is not None: + out_stream.write('Repository URL: {0}'.format( + out_stream.literal_style(self._model.repository_url))) + if self._model.repository_credential: + out_stream.write('Repository credential: {0}'.format( + out_stream.literal_style(self._model.repository_credential))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + + +class Capability(common.InstanceHandlerBase): + def coerce(self, **kwargs): + self._topology.coerce(self._model.properties, **kwargs) + + def validate(self, **kwargs): + self._topology.validate(self._model.properties, **kwargs) + + def dump(self, out_stream): + out_stream.write(out_stream.node_style(self._model.name)) + with out_stream.indent(): + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + out_stream.write('Occurrences: {0:d} ({1:d}{2})'.format( + self._model.occurrences, + self._model.min_occurrences or 0, + ' to {0:d}'.format(self._model.max_occurrences) + if self._model.max_occurrences is not None + else ' or more')) + self._topology.dump(self._model.properties, out_stream, title='Properties') + + +class Group(common.ActorHandlerBase): + + def coerce(self, **kwargs): + self._coerce(self._model.properties, self._model.interfaces, **kwargs) + + def validate(self, **kwargs): + self._validate(self._model.properties, + self._model.interfaces, + **kwargs) + + def dump(self, out_stream): + out_stream.write('Group: {0}'.format(out_stream.node_style(self._model.name))) + with out_stream.indent(): + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + self._topology.dump(self._model.interfaces, out_stream, title='Interfaces') + if self._model.nodes: + out_stream.write('Member nodes:') + with out_stream.indent(): + for node in self._model.nodes: + out_stream.write(out_stream.node_style(node.name)) + + def configure_operations(self): + for interface in self._model.interfaces.values(): + self._topology.configure_operations(interface) + + +class Interface(common.ActorHandlerBase): + def coerce(self, **kwargs): + self._coerce(self._model.inputs, self._model.operations, **kwargs) + + def validate(self, **kwargs): + self._validate(self._model.inputs, + self._model.operations, + **kwargs) + + def dump(self, out_stream): + out_stream.write(out_stream.node_style(self._model.name)) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + out_stream.write('Interface type: {0}'.format( + out_stream.type_style(self._model.type.name))) + self._topology.dump(self._model.inputs, out_stream, title='Inputs') + self._topology.dump(self._model.operations, out_stream, title='Operations') + + def configure_operations(self): + for operation in self._model.operations.values(): + self._topology.configure_operations(operation) + + +class Node(common.ActorHandlerBase): + def coerce(self, **kwargs): + self._coerce(self._model.properties, + self._model.attributes, + self._model.interfaces, + self._model.artifacts, + self._model.capabilities, + self._model.outbound_relationships, + **kwargs) + + def validate(self, **kwargs): + if len(self._model.name) > context.ID_MAX_LENGTH: + self._topology.report( + '"{0}" has an ID longer than the limit of {1:d} characters: {2:d}'.format( + self._model.name, context.ID_MAX_LENGTH, len(self._model.name)), + level=self._topology.Issue.BETWEEN_INSTANCES) + + self._validate(self._model.properties, + self._model.attributes, + self._model.interfaces, + self._model.artifacts, + self._model.capabilities, + self._model.outbound_relationships) + + def dump(self, out_stream): + out_stream.write('Node: {0}'.format(out_stream.node_style(self._model.name))) + with out_stream.indent(): + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + out_stream.write('Template: {0}'.format( + out_stream.node_style(self._model.node_template.name))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + self._topology.dump(self._model.attributes, out_stream, title='Attributes') + self._topology.dump(self._model.interfaces, out_stream, title='Interfaces') + self._topology.dump(self._model.artifacts, out_stream, title='Artifacts') + self._topology.dump(self._model.capabilities, out_stream, title='Capabilities') + self._topology.dump(self._model.outbound_relationships, out_stream, + title='Relationships') + + def configure_operations(self): + for interface in self._model.interfaces.values(): + self._topology.configure_operations(interface) + for relationship in self._model.outbound_relationships: + self._topology.configure_operations(relationship) + + def validate_capabilities(self): + satisfied = False + for capability in self._model.capabilities.itervalues(): + if not capability.has_enough_relationships: + self._topology.report( + 'capability "{0}" of node "{1}" requires at least {2:d} ' + 'relationships but has {3:d}'.format(capability.name, + self._model.name, + capability.min_occurrences, + capability.occurrences), + level=self._topology.Issue.BETWEEN_INSTANCES) + satisfied = False + return satisfied + + def satisfy_requirements(self): + satisfied = True + for requirement_template in self._model.node_template.requirement_templates: + + # Since we try and satisfy requirements, which are node template bound, and use that + # information in the creation of the relationship, Some requirements may have been + # satisfied by a previous run on that node template. + # The entire mechanism of satisfying requirements needs to be refactored. + if any(rel.requirement_template == requirement_template + for rel in self._model.outbound_relationships): + continue + + # Find target template + target_node_template, target_node_capability = self._find_target(requirement_template) + if target_node_template is not None: + satisfied = self._satisfy_capability( + target_node_capability, target_node_template, requirement_template) + else: + self._topology.report('requirement "{0}" of node "{1}" has no target node template'. + format(requirement_template.name, self._model.name), + level=self._topology.Issue.BETWEEN_INSTANCES) + satisfied = False + return satisfied + + def _satisfy_capability(self, target_node_capability, target_node_template, + requirement_template): + # Find target nodes + target_nodes = target_node_template.nodes + if target_nodes: + target_node = None + target_capability = None + + if target_node_capability is not None: + # Relate to the first target node that has capacity + for node in target_nodes: + a_target_capability = node.capabilities.get(target_node_capability.name) + if a_target_capability.relate(): + target_node = node + target_capability = a_target_capability + break + else: + # Use first target node + target_node = target_nodes[0] + + if target_node is not None: + if requirement_template.relationship_template is not None: + relationship_model = self._topology.instantiate( + requirement_template.relationship_template) + else: + relationship_model = models.Relationship() + relationship_model.name = requirement_template.name + relationship_model.requirement_template = requirement_template + relationship_model.target_node = target_node + relationship_model.target_capability = target_capability + self._model.outbound_relationships.append(relationship_model) + return True + else: + self._topology.report( + 'requirement "{0}" of node "{1}" targets node ' + 'template "{2}" but its instantiated nodes do not ' + 'have enough capacity'.format( + requirement_template.name, self._model.name, target_node_template.name), + level=self._topology.Issue.BETWEEN_INSTANCES) + return False + else: + self._topology.report( + 'requirement "{0}" of node "{1}" targets node template ' + '"{2}" but it has no instantiated nodes'.format( + requirement_template.name, self._model.name, target_node_template.name), + level=self._topology.Issue.BETWEEN_INSTANCES) + return False + + def _find_target(self, requirement_template): + # We might already have a specific node template from the requirement template, so + # we'll just verify it + if requirement_template.target_node_template is not None: + if not self._model.node_template.is_target_node_template_valid( + requirement_template.target_node_template): + self._topology.report( + 'requirement "{0}" of node template "{1}" is for node ' + 'template "{2}" but it does not match constraints'.format( + requirement_template.name, + requirement_template.target_node_template.name, + self._model.node_template.name), + level=self._topology.Issue.BETWEEN_TYPES) + if (requirement_template.target_capability_type is not None or + requirement_template.target_capability_name is not None): + target_node_capability = self._get_capability(requirement_template) + if target_node_capability is None: + return None, None + else: + target_node_capability = None + + return requirement_template.target_node_template, target_node_capability + + # Find first node that matches the type + elif requirement_template.target_node_type is not None: + for target_node_template in \ + self._model.node_template.service_template.node_templates.itervalues(): + if requirement_template.target_node_type.get_descendant( + target_node_template.type.name) is None: + continue + + if not self._model.node_template.is_target_node_template_valid( + target_node_template): + continue + + target_node_capability = self._get_capability(requirement_template, + target_node_template) + + if target_node_capability is None: + continue + + return target_node_template, target_node_capability + + # Find the first node which has a capability of the required type + elif requirement_template.target_capability_type is not None: + for target_node_template in \ + self._model.node_template.service_template.node_templates.itervalues(): + target_node_capability = \ + self._get_capability(requirement_template, target_node_template) + if target_node_capability: + return target_node_template, target_node_capability + + return None, None + + def _get_capability(self, requirement_template, target_node_template=None): + target_node_template = target_node_template or requirement_template.target_node_template + + for capability_template in target_node_template.capability_templates.values(): + if self._satisfies_requirement( + capability_template, requirement_template, target_node_template): + return capability_template + + return None + + def _satisfies_requirement( + self, capability_template, requirement_template, target_node_template): + # Do we match the required capability type? + if (requirement_template.target_capability_type and + requirement_template.target_capability_type.get_descendant( + capability_template.type.name) is None): + return False + + # Are we in valid_source_node_types? + if capability_template.valid_source_node_types: + for valid_source_node_type in capability_template.valid_source_node_types: + if valid_source_node_type.get_descendant( + self._model.node_template.type.name) is None: + return False + + # Apply requirement constraints + if requirement_template.target_node_template_constraints: + for node_template_constraint in requirement_template.target_node_template_constraints: + if not node_template_constraint.matches( + self._model.node_template, target_node_template): + return False + + return True + + +class Operation(common.ActorHandlerBase): + def coerce(self, **kwargs): + self._coerce(self._model.inputs, + self._model.configurations, + self._model.arguments, + **kwargs) + + def validate(self, **kwargs): + self._validate(self._model.inputs, + self._model.configurations, + self._model.arguments, + **kwargs) + + def dump(self, out_stream): + out_stream.write(out_stream.node_style(self._model.name)) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + if self._model.implementation is not None: + out_stream.write('Implementation: {0}'.format( + out_stream.literal_style(self._model.implementation))) + if self._model.dependencies: + out_stream.write( + 'Dependencies: {0}'.format(', '.join((str(out_stream.literal_style(v)) + for v in self._model.dependencies)))) + self._topology.dump(self._model.inputs, out_stream, title='Inputs') + if self._model.executor is not None: + out_stream.write('Executor: {0}'.format(out_stream.literal_style( + self._model.executor))) + if self._model.max_attempts is not None: + out_stream.write('Max attempts: {0}'.format(out_stream.literal_style( + self._model.max_attempts))) + if self._model.retry_interval is not None: + out_stream.write('Retry interval: {0}'.format( + out_stream.literal_style(self._model.retry_interval))) + if self._model.plugin is not None: + out_stream.write('Plugin: {0}'.format( + out_stream.literal_style(self._model.plugin.name))) + self._topology.dump(self._model.configurations, out_stream, title='Configuration') + if self._model.function is not None: + out_stream.write('Function: {0}'.format(out_stream.literal_style( + self._model.function))) + self._topology.dump(self._model.arguments, out_stream, title='Arguments') + + def configure_operations(self): + if self._model.implementation is None and self._model.function is None: + return + + if (self._model.interface is not None and + self._model.plugin is None and + self._model.function is None): + # ("interface" is None for workflow operations, which do not currently use "plugin") + # The default (None) plugin is the execution plugin + execution_plugin.instantiation.configure_operation(self._model, self._topology) + else: + # In the future plugins may be able to add their own "configure_operation" hook that + # can validate the configuration and otherwise create specially derived arguments. For + # now, we just send all configuration parameters as arguments without validation. + for key, conf in self._model.configurations.items(): + self._model.arguments[key] = self._topology.instantiate(conf.as_argument()) + + if self._model.interface is not None: + # Send all interface inputs as extra arguments + # ("interface" is None for workflow operations) + # Note that they will override existing arguments of the same names + for key, input in self._model.interface.inputs.items(): + self._model.arguments[key] = self._topology.instantiate(input.as_argument()) + + # Send all inputs as extra arguments + # Note that they will override existing arguments of the same names + for key, input in self._model.inputs.items(): + self._model.arguments[key] = self._topology.instantiate(input.as_argument()) + + # Check for reserved arguments + used_reserved_names = set(decorators.OPERATION_DECORATOR_RESERVED_ARGUMENTS).intersection( + self._model.arguments.keys()) + if used_reserved_names: + self._topology.report( + 'using reserved arguments in operation "{0}": {1}'.format( + self._model.name, formatting.string_list_as_string(used_reserved_names)), + level=self._topology.Issue.EXTERNAL) + + +class Policy(common.InstanceHandlerBase): + def coerce(self, **kwargs): + self._topology.coerce(self._model.properties, **kwargs) + + def validate(self, **kwargs): + self._topology.validate(self._model.properties, **kwargs) + + def dump(self, out_stream): + out_stream.write('Policy: {0}'.format(out_stream.node_style(self._model.name))) + with out_stream.indent(): + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + if self._model.nodes: + out_stream.write('Target nodes:') + with out_stream.indent(): + for node in self._model.nodes: + out_stream.write(out_stream.node_style(node.name)) + if self._model.groups: + out_stream.write('Target groups:') + with out_stream.indent(): + for group in self._model.groups: + out_stream.write(out_stream.node_style(group.name)) + + +class Relationship(common.ActorHandlerBase): + def coerce(self, **kwargs): + self._coerce(self._model.properties, + self._model.interfaces, + **kwargs) + + def validate(self, **kwargs): + self._validate(self._model.properties, + self._model.interfaces, + **kwargs) + + def dump(self, out_stream): + if self._model.name: + out_stream.write('{0} ->'.format(out_stream.node_style(self._model.name))) + else: + out_stream.write('->') + with out_stream.indent(): + out_stream.write('Node: {0}'.format(out_stream.node_style( + self._model.target_node.name))) + if self._model.target_capability: + out_stream.write('Capability: {0}'.format(out_stream.node_style( + self._model.target_capability.name))) + if self._model.type is not None: + out_stream.write('Relationship type: {0}'.format( + out_stream.type_style(self._model.type.name))) + if (self._model.relationship_template is not None and + self._model.relationship_template.name): + out_stream.write('Relationship template: {0}'.format( + out_stream.node_style(self._model.relationship_template.name))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + self._topology.dump(self._model.interfaces, out_stream, title='Interfaces') + + def configure_operations(self): + for interface in self._model.interfaces.values(): + self._topology.configure_operations(interface) + + +class Service(common.ActorHandlerBase): + def coerce(self, **kwargs): + self._coerce(self._model.meta_data, + self._model.nodes, + self._model.groups, + self._model.policies, + self._model.substitution, + self._model.inputs, + self._model.outputs, + self._model.workflows, + **kwargs) + + def validate(self, **kwargs): + self._validate(self._model.meta_data, + self._model.nodes, + self._model.groups, + self._model.policies, + self._model.substitution, + self._model.inputs, + self._model.outputs, + self._model.workflows, + **kwargs) + + def dump(self, out_stream): + if self._model.description is not None: + out_stream.write(out_stream.meta_style(self._model.description)) + self._topology.dump(self._model.meta_data, out_stream, title='Metadata') + self._topology.dump(self._model.nodes, out_stream) + self._topology.dump(self._model.groups, out_stream) + self._topology.dump(self._model.policies, out_stream) + self._topology.dump(self._model.substitution, out_stream) + self._topology.dump(self._model.inputs, out_stream, title='Inputs') + self._topology.dump(self._model.outputs, out_stream, title='Outputs') + self._topology.dump(self._model.workflows, out_stream, title='Workflows') + + def configure_operations(self): + for node in self._model.nodes.itervalues(): + self._topology.configure_operations(node) + for group in self._model.groups.itervalues(): + self._topology.configure_operations(group) + for operation in self._model.workflows.itervalues(): + self._topology.configure_operations(operation) + + def validate_capabilities(self): + satisfied = True + for node in self._model.nodes.values(): + if not self._topology.validate_capabilities(node): + satisfied = False + return satisfied + + def satisfy_requirements(self): + return all(self._topology.satisfy_requirements(node) + for node in self._model.nodes.values()) + + +class Substitution(common.InstanceHandlerBase): + def coerce(self, **kwargs): + self._topology.coerce(self._model.mappings, **kwargs) + + def validate(self, **kwargs): + self._topology.validate(self._model.mappings, **kwargs) + + def dump(self, out_stream): + out_stream.write('Substitution:') + with out_stream.indent(): + out_stream.write('Node type: {0}'.format(out_stream.type_style( + self._model.node_type.name))) + self._topology.dump(self._model.mappings, out_stream, title='Mappings') + + +class SubstitutionMapping(common.InstanceHandlerBase): + + def coerce(self, **kwargs): + pass + + def validate(self, **_): + if (self._model.capability is None) and (self._model.requirement_template is None): + self._topology.report( + 'mapping "{0}" refers to neither capability nor a requirement' + ' in node: {1}'.format( + self._model.name, formatting.safe_repr(self._model.node_style.name)), + level=self._topology.Issue.BETWEEN_TYPES) + + def dump(self, out_stream): + if self._model.capability is not None: + out_stream.write('{0} -> {1}.{2}'.format( + out_stream.node_style(self._model.name), + out_stream.node_style(self._model.capability.node.name), + out_stream.node_style(self._model.capability.name))) + else: + out_stream.write('{0} -> {1}.{2}'.format( + out_stream.node_style(self._model.name), + out_stream.node_style(self._model.node.name), + out_stream.node_style(self._model.requirement_template.name))) + + +class Metadata(common.InstanceHandlerBase): + + def dump(self, out_stream): + out_stream.write('{0}: {1}'.format( + out_stream.property_style(self._model.name), + out_stream.literal_style(self._model.value))) + + def coerce(self, **_): + pass + + def instantiate(self, instance_cls): + return instance_cls(name=self._model.name, value=self._model.value) + + def validate(self): + pass + + +class _Parameter(common.InstanceHandlerBase): + + def dump(self, out_stream): + if self._model.type_name is not None: + out_stream.write('{0}: {1} ({2})'.format( + out_stream.property_style(self._model.name), + out_stream.literal_style(formatting.as_raw(self._model.value)), + out_stream.type_style(self._model.type_name))) + else: + out_stream.write('{0}: {1}'.format( + out_stream.property_style(self._model.name), + out_stream.literal_style(formatting.as_raw(self._model.value)))) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + + def instantiate(self, instance_cls, **kwargs): + return instance_cls( + name=self._model.name, # pylint: disable=unexpected-keyword-arg + type_name=self._model.type_name, + _value=self._model._value, + description=self._model.description + ) + + def validate(self): + pass + + def coerce(self, report_issues): # pylint: disable=arguments-differ + value = self._model._value + if value is not None: + evaluation = functions.evaluate(value, self._model, report_issues) + if (evaluation is not None) and evaluation.final: + # A final evaluation can safely replace the existing value + self._model._value = evaluation.value + + +class Attribute(_Parameter): + pass + + +class Input(_Parameter): + pass + + +class Output(_Parameter): + pass + + +class Argument(_Parameter): + pass + + +class Property(_Parameter): + pass + + +class Configuration(_Parameter): + pass + + +class Type(common.InstanceHandlerBase): + def coerce(self, **_): + pass + + def dump(self, out_stream): + if self._model.name: + out_stream.write(out_stream.type_style(self._model.name)) + with out_stream.indent(): + for child in self._model.children: + self._topology.dump(child, out_stream) + + def validate(self, **kwargs): + pass diff --git a/aria/orchestrator/topology/template_handler.py b/aria/orchestrator/topology/template_handler.py new file mode 100644 index 00000000..bf0ef9fb --- /dev/null +++ b/aria/orchestrator/topology/template_handler.py @@ -0,0 +1,606 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +from ...utils import ( + formatting, + versions +) +from ...modeling import utils as modeling_utils +from . import utils, common + + +class ServiceTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + if self._model.description is not None: + out_stream.write(out_stream.meta_style(self._model.description)) + self._topology.dump(self._model.meta_data, out_stream, title='Metadata') + self._topology.dump(self._model.node_templates, out_stream) + self._topology.dump(self._model.group_templates, out_stream) + self._topology.dump(self._model.policy_templates, out_stream) + self._topology.dump(self._model.substitution_template, out_stream) + self._topology.dump(self._model.inputs, out_stream, title='Inputs') + self._topology.dump(self._model.outputs, out_stream, title='Outputs') + self._topology.dump(self._model.workflow_templates, out_stream, title='Workflow templates') + + def coerce(self, **kwargs): + self._coerce(self._model.meta_data, + self._model.node_templates, + self._model.group_templates, + self._model.policy_templates, + self._model.substitution_template, + self._model.inputs, + self._model.outputs, + self._model.workflow_templates, + **kwargs) + + def instantiate(self, instance_cls, inputs=None, plugins=None): # pylint: disable=arguments-differ + now = datetime.now() + + modeling_utils.validate_no_undeclared_inputs( + declared_inputs=self._model.inputs, supplied_inputs=inputs or {}) + modeling_utils.validate_required_inputs_are_supplied( + declared_inputs=self._model.inputs, supplied_inputs=inputs or {}) + + service = instance_cls( + created_at=now, + updated_at=now, + description=utils.deepcopy_with_locators(self._model.description), + service_template=self._model, + inputs=modeling_utils.merge_parameter_values(inputs, self._model.inputs) + ) + + for plugin_specification in self._model.plugin_specifications.itervalues(): + if plugin_specification.enabled and plugins: + if self._resolve_plugin_specification(plugin_specification, plugins): + plugin = plugin_specification.plugin + service.plugins[plugin.name] = plugin + else: + self._topology.report('specified plugin not found: {0}'.format( + plugin_specification.name), level=self._topology.Issue.EXTERNAL) + service.meta_data = self._topology.instantiate(self._model.meta_data) + + for node_template in self._model.node_templates.itervalues(): + for _ in range(self._scaling(node_template)['default_instances']): + node = self._topology.instantiate(node_template) + service.nodes[node.name] = node + + service.groups = self._topology.instantiate(self._model.group_templates) + service.policies = self._topology.instantiate(self._model.policy_templates) + service.workflows = self._topology.instantiate(self._model.workflow_templates) + service.substitution = self._topology.instantiate(self._model.substitution_template) + service.outputs = self._topology.instantiate(self._model.outputs) + + return service + + @staticmethod + def _resolve_plugin_specification(plugin_specification, plugins): + matching_plugins = [] + if plugins: + for plugin in plugins: + if (plugin.name == plugin_specification.name and + (plugin_specification.version is None or + versions.VersionString(plugin.package_version) >= + plugin_specification.version) + ): + matching_plugins.append(plugin) + plugin_specification.plugin = None + if matching_plugins: + # Return highest version of plugin + plugin_specification.plugin = \ + max(matching_plugins, + key=lambda plugin: versions.VersionString(plugin.package_version).key) + return plugin_specification.plugin is not None + + def _scaling(self, node_template): + scaling = node_template.scaling + + if any([scaling['min_instances'] < 0, + scaling['max_instances'] < scaling['min_instances'], + scaling['max_instances'] < 0, + + scaling['default_instances'] < 0, + scaling['default_instances'] < scaling['min_instances'], + scaling['default_instances'] > scaling['max_instances'] + ]): + self._topology.report( + 'invalid scaling parameters for node template "{0}": min={min_instances}, max=' + '{max_instances}, default={default_instances}'.format(self._model.name, **scaling), + level=self._topology.Issue.BETWEEN_TYPES) + + return scaling + + def validate(self, **kwargs): + self._validate( + self._model.meta_data, + self._model.node_templates, + self._model.group_templates, + self._model.policy_templates, + self._model.substitution_template, + self._model.inputs, + self._model.outputs, + self._model.workflow_templates, + self._model.node_types, + self._model.group_types, + self._model.policy_types, + self._model.relationship_types, + self._model.capability_types, + self._model.interface_types, + self._model.artifact_types, + **kwargs + ) + + +class ArtifactTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + out_stream.write(out_stream.node_style(self._model.name)) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + out_stream.write('Artifact type: {0}'.format(out_stream.type_style( + self._model.type.name))) + out_stream.write('Source path: {0}'.format(out_stream.literal_style( + self._model.source_path))) + if self._model.target_path is not None: + out_stream.write('Target path: {0}'.format(out_stream.literal_style( + self._model.target_path))) + if self._model.repository_url is not None: + out_stream.write('Repository URL: {0}'.format( + out_stream.literal_style(self._model.repository_url))) + if self._model.repository_credential: + out_stream.write('Repository credential: {0}'.format( + out_stream.literal_style(self._model.repository_credential))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + + def coerce(self, **kwargs): + self._topology.coerce(self._model.properties, **kwargs) + + def instantiate(self, instance_cls, **_): + return instance_cls( + name=self._model.name, + type=self._model.type, + description=utils.deepcopy_with_locators(self._model.description), + source_path=self._model.source_path, + target_path=self._model.target_path, + repository_url=self._model.repository_url, + repository_credential=self._model.repository_credential, + artifact_template=self._model) + + def validate(self, **kwargs): + self._topology.validate(self._model.properties, **kwargs) + + +class CapabilityTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + out_stream.write(out_stream.node_style(self._model.name)) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + out_stream.write( + 'Occurrences: {0:d}{1}'.format( + self._model.min_occurrences or 0, + ' to {0:d}'.format(self._model.max_occurrences) + if self._model.max_occurrences is not None + else ' or more')) + if self._model.valid_source_node_types: + out_stream.write('Valid source node types: {0}'.format( + ', '.join((str(out_stream.type_style(v.name)) + for v in self._model.valid_source_node_types)))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + + def coerce(self, **kwargs): + self._topology.coerce(self._model.properties, **kwargs) + + def instantiate(self, instance_cls, **_): + return instance_cls(name=self._model.name, + type=self._model.type, + min_occurrences=self._model.min_occurrences, + max_occurrences=self._model.max_occurrences, + occurrences=0, + capability_template=self._model) + + def validate(self, **kwargs): + self._topology.validate(self._model.properties, **kwargs) + + +class RequirementTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + if self._model.name: + out_stream.write(out_stream.node_style(self._model.name)) + else: + out_stream.write('Requirement:') + with out_stream.indent(): + if self._model.target_node_type is not None: + out_stream.write('Target node type: {0}'.format( + out_stream.type_style(self._model.target_node_type.name))) + elif self._model.target_node_template is not None: + out_stream.write('Target node template: {0}'.format( + out_stream.node_style(self._model.target_node_template.name))) + if self._model.target_capability_type is not None: + out_stream.write('Target capability type: {0}'.format( + out_stream.type_style(self._model.target_capability_type.name))) + elif self._model.target_capability_name is not None: + out_stream.write('Target capability name: {0}'.format( + out_stream.node_style(self._model.target_capability_name))) + if self._model.target_node_template_constraints: + out_stream.write('Target node template constraints:') + with out_stream.indent(): + for constraint in self._model.target_node_template_constraints: + out_stream.write(out_stream.literal_style(constraint)) + if self._model.relationship_template: + out_stream.write('Relationship:') + with out_stream.indent(): + self._topology.dump(self._model.relationship_template, out_stream) + + def coerce(self, **kwargs): + self._topology.coerce(self._model.relationship_template, **kwargs) + + def instantiate(self, instance_cls, **_): + pass + + def validate(self, **kwargs): + self._topology.validate(self._model.relationship_template, **kwargs) + + +class GroupTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + out_stream.write('Group template: {0}'.format(out_stream.node_style(self._model.name))) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + self._topology.dump(self._model.interface_templates, out_stream, + title='Interface Templates') + if self._model.node_templates: + out_stream.write('Member node templates: {0}'.format(', '.join( + (str(out_stream.node_style(v.name)) for v in self._model.node_templates)))) + + def coerce(self, **kwargs): + self._coerce(self._model.properties, + self._model.interface_templates, + **kwargs) + + def instantiate(self, instance_cls, **_): + group = instance_cls( + name=self._model.name, + type=self._model.type, + description=utils.deepcopy_with_locators(self._model.description), + group_template=self._model) + group.properties = self._topology.instantiate(self._model.properties) + group.interfaces = self._topology.instantiate(self._model.interface_templates) + if self._model.node_templates: + for node_template in self._model.node_templates: + group.nodes += node_template.nodes + return group + + def validate(self, **kwargs): + self._validate(self._model.properties, + self._model.interface_templates, + **kwargs) + + +class InterfaceTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + out_stream.write(out_stream.node_style(self._model.name)) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + out_stream.write('Interface type: {0}'.format(out_stream.type_style( + self._model.type.name))) + self._topology.dump(self._model.inputs, out_stream, title='Inputs') + self._topology.dump(self._model.operation_templates, out_stream, + title='Operation templates') + + def coerce(self, **kwargs): + self._coerce(self._model.inputs, + self._model.operation_templates, + **kwargs) + + def instantiate(self, instance_cls, **_): + interface = instance_cls( + name=self._model.name, + type=self._model.type, + description=utils.deepcopy_with_locators(self._model.description), + interface_template=self._model) + interface.inputs = self._topology.instantiate(self._model.inputs) + interface.operations = self._topology.instantiate(self._model.operation_templates) + return interface + + def validate(self, **kwargs): + self._validate(self._model.inputs, + self._model.operation_templates, + **kwargs) + + +class NodeTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + out_stream.write('Node template: {0}'.format(out_stream.node_style(self._model.name))) + with out_stream.indent(): + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + self._topology.dump(self._model.attributes, out_stream, title='Attributes') + self._topology.dump( + self._model.interface_templates, out_stream, title='Interface Templates') + self._topology.dump( + self._model.artifact_templates, out_stream, title='Artifact Templates') + self._topology.dump( + self._model.capability_templates, out_stream, title='Capability Templates') + self._topology.dump( + self._model.requirement_templates, out_stream, title='Requirement Templates') + + def coerce(self, **kwargs): + self._coerce(self._model.properties, + self._model.attributes, + self._model.interface_templates, + self._model.artifact_templates, + self._model.capability_templates, + self._model.requirement_templates, + **kwargs) + + def instantiate(self, instance_cls, **_): + node = instance_cls( + name=self._model._next_name, + type=self._model.type, + description=utils.deepcopy_with_locators(self._model.description), + node_template=self._model + ) + + node.properties = self._topology.instantiate(self._model.properties) + node.attributes = self._topology.instantiate(self._model.attributes) + node.interfaces = self._topology.instantiate(self._model.interface_templates) + node.artifacts = self._topology.instantiate(self._model.artifact_templates) + node.capabilities = self._topology.instantiate(self._model.capability_templates) + + # Default attributes + if 'tosca_name' in node.attributes and node.attributes['tosca_name'].type_name == 'string': + node.attributes['tosca_name'].value = self._model.name + if 'tosca_id' in node.attributes and node.attributes['tosca_id'].type_name == 'string': + node.attributes['tosca_id'].value = node.name + + return node + + def validate(self, **kwargs): + self._validate(self._model.properties, + self._model.attributes, + self._model.interface_templates, + self._model.artifact_templates, + self._model.capability_templates, + self._model.requirement_templates, + **kwargs) + + +class PolicyTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + out_stream.write('Policy template: {0}'.format(out_stream.node_style(self._model.name))) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + out_stream.write('Type: {0}'.format(out_stream.type_style(self._model.type.name))) + self._topology.dump(self._model.properties, out_stream, title='Properties') + if self._model.node_templates: + out_stream.write('Target node templates: {0}'.format(', '.join( + (str(out_stream.node_style(v.name)) for v in self._model.node_templates)))) + if self._model.group_templates: + out_stream.write('Target group templates: {0}'.format(', '.join( + (str(out_stream.node_style(v.name)) for v in self._model.group_templates)))) + + def coerce(self, **kwargs): + self._topology.coerce(self._model.properties, **kwargs) + + def instantiate(self, instance_cls, **_): + policy = instance_cls( + name=self._model.name, + type=self._model.type, + description=utils.deepcopy_with_locators(self._model.description), + policy_template=self._model) + + policy.properties = self._topology.instantiate(self._model.properties) + if self._model.node_templates: + for node_template in self._model.node_templates: + policy.nodes += node_template.nodes + if self._model.group_templates: + for group_template in self._model.group_templates: + policy.groups += group_template.groups + return policy + + def validate(self, **kwargs): + self._topology.validate(self._model.properties, **kwargs) + + +class SubstitutionTemplate(common.TemplateHandlerBase): + + def dump(self, out_stream): + out_stream.write('Substitution template:') + with out_stream.indent(): + out_stream.write('Node type: {0}'.format(out_stream.type_style( + self._model.node_type.name))) + self._topology.dump(self._model.mappings, out_stream, title='Mappings') + + def coerce(self, **kwargs): + self._topology.coerce(self._model.mappings, **kwargs) + + def instantiate(self, instance_cls, **_): + return instance_cls(node_type=self._model.node_type, substitution_template=self._model) + + def validate(self, **kwargs): + self._topology.validate(self._model.mappings, **kwargs) + + +class SubstitutionTemplateMapping(common.TemplateHandlerBase): + + def dump(self, out_stream): + if self._topology.capability_template is not None: + node_template = self._model.capability_template.node_template + else: + node_template = self._model.requirement_template.node_template + out_stream.write('{0} -> {1}.{2}'.format( + out_stream.node_style(self._model.name), + out_stream.node_style(node_template.name), + out_stream.node_style(self._model.capability_template.name + if self._model.capability_template + else self._model.requirement_template.name))) + + def coerce(self, **_): + pass + + def instantiate(self, instance_cls, **_): + substitution_mapping = instance_cls( + name=self._model.name, + requirement_template=self._model.requirement_template) + + if self._model.capability_template is not None: + node_template = self._model.capability_template.node_template + else: + node_template = self._model.requirement_template.node_template + nodes = node_template.nodes + if len(nodes) == 0: + self._topology.report( + 'mapping "{0}" refers to node template "{1}" but there are no node instances'. + format(self._model.mapped_name, self._model.node_template.name), + level=self._topology.Issue.BETWEEN_INSTANCES) + return None + # The TOSCA spec does not provide a way to choose the node, + # so we will just pick the first one + substitution_mapping.node_style = nodes[0] + if self._model.capability_template: + for a_capability in substitution_mapping.node_style.capabilities.itervalues(): + if a_capability.capability_template.name == \ + self._model.capability_template.name: + substitution_mapping.capability = a_capability + + return substitution_mapping + + def validate(self, **_): + if self._model.capability_template is None and self._model.requirement_template is None: + self._topology.report( + 'mapping "{0}" refers to neither capability nor a requirement ' + 'in node template: {1}'.format( + self._model.name, formatting.safe_repr(self._model.node_template.name)), + level=self._topology.Issue.BETWEEN_TYPES) + + +class RelationshipTemplate(common.TemplateHandlerBase): + def dump(self, out_stream): + if self._model.type is not None: + out_stream.write('Relationship type: {0}'.format(out_stream.type_style( + self._model.type.name))) + else: + out_stream.write('Relationship template: {0}'.format( + out_stream.node_style(self._model.name))) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + self._topology.dump(self._model.properties, out_stream, title='Properties') + self._topology.dump(self._model.interface_templates, out_stream, + title='Interface Templates') + + def coerce(self, **kwargs): + self._coerce(self._model.properties, self._model.interface_templates, **kwargs) + + def instantiate(self, instance_cls, **_): + relationship = instance_cls( + name=self._model.name, + type=self._model.type, + relationship_template=self._model) + + relationship.properties = self._topology.instantiate(self._model.properties) + relationship.interfaces = self._topology.instantiate(self._model.interface_templates) + return relationship + + def validate(self, **kwargs): + self._validate(self._model.properties, self._model.interface_templates, **kwargs) + + +class OperationTemplate(common.TemplateHandlerBase): + + def dump(self, out_stream): + out_stream.write(out_stream.node_style(self._model.name)) + if self._model.description: + out_stream.write(out_stream.meta_style(self._model.description)) + with out_stream.indent(): + if self._model.implementation is not None: + out_stream.write('Implementation: {0}'.format( + out_stream.literal_style(self._model.implementation))) + if self._model.dependencies: + out_stream.write('Dependencies: {0}'.format(', '.join( + (str(out_stream.literal_style(v)) for v in self._model.dependencies)))) + self._topology.dump(self._model.inputs, out_stream, title='Inputs') + if self._model.executor is not None: + out_stream.write('Executor: {0}'.format( + out_stream.literal_style(self._model.executor))) + if self._model.max_attempts is not None: + out_stream.write('Max attempts: {0}'.format(out_stream.literal_style( + self._model.max_attempts))) + if self._model.retry_interval is not None: + out_stream.write('Retry interval: {0}'.format( + out_stream.literal_style(self._model.retry_interval))) + if self._model.plugin_specification is not None: + out_stream.write('Plugin specification: {0}'.format( + out_stream.literal_style(self._model.plugin_specification.name))) + self._topology.dump(self._model.configurations, out_stream, title='Configuration') + if self._model.function is not None: + out_stream.write('Function: {0}'.format(out_stream.literal_style( + self._model.function))) + + def coerce(self, **kwargs): + self._coerce(self._model.inputs, + self._model.configurations, + **kwargs) + + def instantiate(self, instance_cls, **_): + operation = instance_cls( + name=self._model.name, + description=utils.deepcopy_with_locators(self._model.description), + relationship_edge=self._model.relationship_edge, + implementation=self._model.implementation, + dependencies=self._model.dependencies, + executor=self._model.executor, + function=self._model.function, + max_attempts=self._model.max_attempts, + retry_interval=self._model.retry_interval, + operation_template=self._model) + + if (self._model.plugin_specification is not None and + self._model.plugin_specification.enabled): + operation.plugin = self._model.plugin_specification.plugin + + operation.inputs = self._topology.instantiate(self._model.inputs) + operation.configurations = self._topology.instantiate(self._model.configurations) + + return operation + + def validate(self, **kwargs): + self._validate(self._model.inputs, + self._model.configurations, + **kwargs) + + +class PluginSpecification(common.HandlerBase): + def validate(self, **kwargs): + pass + + def coerce(self, **kwargs): + pass + + def instantiate(self, **_): + pass + + def dump(self, out_stream): + pass diff --git a/aria/orchestrator/topology/topology.py b/aria/orchestrator/topology/topology.py new file mode 100644 index 00000000..8ee33d10 --- /dev/null +++ b/aria/orchestrator/topology/topology.py @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...parser.validation import issue +from ...modeling import models +from ...utils import console +from . import ( + template_handler, + instance_handler, + common +) + + +class Topology(issue.ReporterMixin): + + _init_map = { + models.ServiceTemplate: models.Service, + models.ArtifactTemplate: models.Artifact, + models.CapabilityTemplate: models.Capability, + models.GroupTemplate: models.Group, + models.InterfaceTemplate: models.Interface, + models.NodeTemplate: models.Node, + models.PolicyTemplate: models.Policy, + models.SubstitutionTemplate: models.Substitution, + models.RelationshipTemplate: models.Relationship, + models.OperationTemplate: models.Operation, + models.SubstitutionTemplateMapping: models.SubstitutionMapping, + + # Common + models.Metadata: models.Metadata, + models.Attribute: models.Attribute, + models.Property: models.Property, + models.Input: models.Input, + models.Output: models.Output, + models.Configuration: models.Configuration, + models.Argument: models.Argument, + models.Type: models.Type + } + + def __init__(self, *args, **kwargs): + super(Topology, self).__init__(*args, **kwargs) + self._model_cls_to_handler = dict(self._init_handlers(instance_handler), + **self._init_handlers(template_handler)) + + @staticmethod + def _init_handlers(module_): + """ + Register handlers from a handler module to the models + + :param module_: The module to look for handlers + :return: a dict where the key is the models class, and the value is the handler class + associated with it from the provided module + """ + handlers = {} + for attribute_name in dir(module_): + if attribute_name.startswith('_'): + continue + attribute = getattr(module_, attribute_name) + if isinstance(attribute, type) and issubclass(attribute, common.HandlerBase): + handlers[getattr(models, attribute_name)] = attribute + return handlers + + def instantiate(self, model, **kwargs): + """ + instantiate the provided model + + :param model: + :param kwargs: + :return: + """ + if isinstance(model, dict): + return dict((name, self.instantiate(value, **kwargs)) + for name, value in model.iteritems()) + elif isinstance(model, list): + return list(self.instantiate(value, **kwargs) for value in model) + elif model is not None: + _handler = self._model_cls_to_handler[model.__class__] + model_instance_cls = self._init_map[model.__class__] + return _handler(self, model).instantiate(model_instance_cls, **kwargs) + + def validate(self, model, **kwargs): + if isinstance(model, dict): + return self.validate(model.values(), **kwargs) + elif isinstance(model, list): + return all(self.validate(value, **kwargs) for value in model) + elif model is not None: + _handler = self._model_cls_to_handler[model.__class__] + return _handler(self, model).validate(**kwargs) + + def dump(self, model, out_stream=None, title=None, **kwargs): + out_stream = out_stream or console.TopologyStylizer() + + # if model is empty, no need to print out the section name + if model and title: + out_stream.write('{0}:'.format(title)) + + if isinstance(model, dict): + if str(out_stream): + with out_stream.indent(): + return self.dump(model.values(), out_stream=out_stream, **kwargs) + else: + return self.dump(model.values(), out_stream=out_stream, **kwargs) + + elif isinstance(model, list): + for value in model: + self.dump(value, out_stream=out_stream, **kwargs) + + elif model is not None: + _handler = self._model_cls_to_handler[model.__class__] + _handler(self, model).dump(out_stream=out_stream, **kwargs) + + return out_stream + + def dump_graph(self, service): + out_stream = console.TopologyStylizer() + for node in service.nodes.itervalues(): + if not node.inbound_relationships: + self._dump_graph_node(out_stream, node) + return out_stream + + def _dump_graph_node(self, out_stream, node, capability=None): + out_stream.write(out_stream.node_style(node.name)) + if capability is not None: + out_stream.write('{0} ({1})'.format(out_stream.property_style(capability.name), + out_stream.type_style(capability.type.name))) + if node.outbound_relationships: + with out_stream.indent(): + for relationship_model in node.outbound_relationships: + styled_relationship_name = out_stream.property_style(relationship_model.name) + if relationship_model.type is not None: + out_stream.write('-> {0} ({1})'.format( + styled_relationship_name, + out_stream.type_style(relationship_model.type.name))) + else: + out_stream.write('-> {0}'.format(styled_relationship_name)) + with out_stream.indent(3): + self._dump_graph_node(out_stream, + relationship_model.target_node, + relationship_model.target_capability) + + def coerce(self, model, **kwargs): + if isinstance(model, dict): + return self.coerce(model.values(), **kwargs) + elif isinstance(model, list): + return all(self.coerce(value, **kwargs) for value in model) + elif model is not None: + _handler = self._model_cls_to_handler[model.__class__] + return _handler(self, model).coerce(**kwargs) + + def dump_types(self, service_template, out_stream=None): + out_stream = out_stream or console.TopologyStylizer() + self.dump(service_template.node_types, out_stream, 'Node types') + self.dump(service_template.group_types, out_stream, 'Group types') + self.dump(service_template.capability_types, out_stream, 'Capability types') + self.dump(service_template.relationship_types, out_stream, 'Relationship types') + self.dump(service_template.policy_types, out_stream, 'Policy types') + self.dump(service_template.artifact_types, out_stream, 'Artifact types') + self.dump(service_template.interface_types, out_stream, 'Interface types') + + return out_stream + + def satisfy_requirements(self, model, **kwargs): + if isinstance(model, dict): + return self.satisfy_requirements(model.values(), **kwargs) + elif isinstance(model, list): + return all(self.satisfy_requirements(value, **kwargs) for value in model) + elif model is not None: + _handler = self._model_cls_to_handler[model.__class__] + return _handler(self, model).satisfy_requirements(**kwargs) + + def validate_capabilities(self, model, **kwargs): + if isinstance(model, dict): + return self.validate_capabilities(model.values(), **kwargs) + elif isinstance(model, list): + return all(self.validate_capabilities(value, **kwargs) for value in model) + elif model is not None: + _handler = self._model_cls_to_handler[model.__class__] + return _handler(self, model).validate_capabilities(**kwargs) + + def _find_host(self, node): + if node.type.role == 'host': + return node + + def target_has_role(rel, role): + return (rel.target_capability is not None and + rel.target_capability.type.role == role) + + for outbound_relationship in node.outbound_relationships: + if target_has_role(outbound_relationship, 'host'): + host = self._find_host(outbound_relationship.target_node) + if host is not None: + return host + for inbound_relationship in node.inbound_relationships: + if target_has_role(inbound_relationship, 'feature'): + host = self._find_host(inbound_relationship.source_node) + if host is not None: + return host + return None + + def assign_hosts(self, service): + for node in service.nodes.values(): + node.host = self._find_host(node) + + def configure_operations(self, model, **kwargs): + if isinstance(model, dict): + return self.configure_operations(model.values(), **kwargs) + elif isinstance(model, list): + return all(self.configure_operations(value, **kwargs) for value in model) + elif model is not None: + _handler = self._model_cls_to_handler[model.__class__] + return _handler(self, model).configure_operations(**kwargs) diff --git a/aria/orchestrator/topology/utils.py b/aria/orchestrator/topology/utils.py new file mode 100644 index 00000000..ec74391c --- /dev/null +++ b/aria/orchestrator/topology/utils.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import deepcopy + + +def deepcopy_with_locators(value): + """ + Like :func:`deepcopy`, but also copies over locators. + """ + + res = deepcopy(value) + copy_locators(res, value) + return res + + +def copy_locators(target, source): + """ + Copies over ``_locator`` for all elements, recursively. + + Assumes that target and source have exactly the same list/dict structure. + """ + + locator = getattr(source, '_locator', None) + if locator is not None: + try: + setattr(target, '_locator', locator) + except AttributeError: + pass + + if isinstance(target, list) and isinstance(source, list): + for i, _ in enumerate(target): + copy_locators(target[i], source[i]) + elif isinstance(target, dict) and isinstance(source, dict): + for k, v in target.items(): + copy_locators(v, source[k]) diff --git a/aria/orchestrator/workflow_runner.py b/aria/orchestrator/workflow_runner.py index 0eac61ce..4dbf29bf 100644 --- a/aria/orchestrator/workflow_runner.py +++ b/aria/orchestrator/workflow_runner.py @@ -141,9 +141,8 @@ def _create_execution_model(self, inputs): supplied_inputs=inputs or {}) modeling_utils.validate_required_inputs_are_supplied(declared_inputs=workflow_inputs, supplied_inputs=inputs or {}) - execution.inputs = modeling_utils.merge_parameter_values(inputs, - workflow_inputs, - model_cls=models.Input) + execution.inputs = modeling_utils.merge_parameter_values( + inputs, workflow_inputs, model_cls=models.Input) # TODO: these two following calls should execute atomically self._validate_no_active_executions(execution) self._model_storage.execution.put(execution) diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py index ec96d270..6ce4a000 100644 --- a/aria/orchestrator/workflows/api/task.py +++ b/aria/orchestrator/workflows/api/task.py @@ -137,14 +137,13 @@ def __init__(self, operation = self.actor.interfaces[self.interface_name].operations[self.operation_name] self.plugin = operation.plugin self.function = operation.function - self.arguments = modeling_utils.merge_parameter_values(arguments, - operation.arguments, - model_cls=models.Argument) + self.arguments = modeling_utils.merge_parameter_values(arguments, operation.arguments) actor = self.actor if hasattr(actor, '_wrapped'): # Unwrap instrumented model actor = actor._wrapped + if isinstance(actor, models.Node): self._context_cls = context.operation.NodeOperationContext elif isinstance(actor, models.Relationship): diff --git a/aria/parser/consumption/__init__.py b/aria/parser/consumption/__init__.py index bd4b29cc..f9caf5f3 100644 --- a/aria/parser/consumption/__init__.py +++ b/aria/parser/consumption/__init__.py @@ -20,7 +20,6 @@ :nosignatures: aria.parser.consumption.ConsumptionContext - aria.parser.consumption.Style Consumers --------- @@ -47,7 +46,6 @@ from .exceptions import ConsumerException from .context import ConsumptionContext -from .style import Style from .consumer import ( Consumer, ConsumerChain @@ -70,7 +68,6 @@ __all__ = ( 'ConsumerException', 'ConsumptionContext', - 'Style', 'Consumer', 'ConsumerChain', 'Read', diff --git a/aria/parser/consumption/consumer.py b/aria/parser/consumption/consumer.py index 4f4c6142..878a1618 100644 --- a/aria/parser/consumption/consumer.py +++ b/aria/parser/consumption/consumer.py @@ -27,6 +27,9 @@ class Consumer(object): """ def __init__(self, context): + from ...orchestrator import topology + + self.topology = topology.Topology() self.context = context def consume(self): @@ -73,6 +76,10 @@ def consume(self): handle_exception(consumer, e) else: raise e + + if consumer.topology.has_issues: + self.context.validation.extend_issues(consumer.topology.issues) + if self.context.validation.has_issues: break diff --git a/aria/parser/consumption/context.py b/aria/parser/consumption/context.py index 6fa61f46..9164984f 100644 --- a/aria/parser/consumption/context.py +++ b/aria/parser/consumption/context.py @@ -16,12 +16,12 @@ import sys import threading +from ...utils import console from ..validation import ValidationContext from ..loading import LoadingContext from ..reading import ReadingContext from ..presentation import PresentationContext from ..modeling import ModelingContext -from .style import Style _thread_locals = threading.local() @@ -58,12 +58,12 @@ def get_thread_local(): def __init__(self, set_thread_local=True): self.args = [] self.out = sys.stdout - self.style = Style() self.validation = ValidationContext() self.loading = LoadingContext() self.reading = ReadingContext() self.presentation = PresentationContext() self.modeling = ModelingContext() + self.style = console.TopologyStylizer() if set_thread_local: self.set_thread_local() diff --git a/aria/parser/consumption/modeling.py b/aria/parser/consumption/modeling.py index 44027b93..221b3086 100644 --- a/aria/parser/consumption/modeling.py +++ b/aria/parser/consumption/modeling.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...utils.formatting import json_dumps, yaml_dumps from .consumer import Consumer, ConsumerChain +from ...utils.formatting import json_dumps, yaml_dumps class DeriveServiceTemplate(Consumer): @@ -42,7 +42,7 @@ class CoerceServiceTemplateValues(Consumer): """ def consume(self): - self.context.modeling.template.coerce_values(True) + self.topology.coerce(self.context.modeling.template, report_issues=True) class ValidateServiceTemplate(Consumer): @@ -51,7 +51,7 @@ class ValidateServiceTemplate(Consumer): """ def consume(self): - self.context.modeling.template.validate() + self.topology.validate(self.context.modeling.template) class ServiceTemplate(ConsumerChain): @@ -74,7 +74,7 @@ def dump(self): raw = self.context.modeling.template_as_raw self.context.write(json_dumps(raw, indent=indent)) else: - self.context.modeling.template.dump() + self.context.write(self.topology.dump(self.context.modeling.template)) class Types(Consumer): @@ -92,7 +92,7 @@ def dump(self): raw = self.context.modeling.types_as_raw self.context.write(json_dumps(raw, indent=indent)) else: - self.context.modeling.template.dump_types() + self.topology.dump_types(self.context, self.context.modeling.template) class InstantiateServiceInstance(Consumer): @@ -105,9 +105,10 @@ def consume(self): self.context.validation.report('InstantiateServiceInstance consumer: missing service ' 'template') return - - self.context.modeling.template.instantiate(None, None, - inputs=dict(self.context.modeling.inputs)) + self.context.modeling.instance = self.topology.instantiate( + self.context.modeling.template, + inputs=dict(self.context.modeling.inputs) + ) class CoerceServiceInstanceValues(Consumer): @@ -116,7 +117,7 @@ class CoerceServiceInstanceValues(Consumer): """ def consume(self): - self.context.modeling.instance.coerce_values(True) + self.topology.coerce(self.context.modeling.instance, report_issues=True) class ValidateServiceInstance(Consumer): @@ -125,7 +126,7 @@ class ValidateServiceInstance(Consumer): """ def consume(self): - self.context.modeling.instance.validate() + self.topology.validate(self.context.modeling.instance) class SatisfyRequirements(Consumer): @@ -134,7 +135,7 @@ class SatisfyRequirements(Consumer): """ def consume(self): - self.context.modeling.instance.satisfy_requirements() + self.topology.satisfy_requirements(self.context.modeling.instance) class ValidateCapabilities(Consumer): @@ -143,7 +144,7 @@ class ValidateCapabilities(Consumer): """ def consume(self): - self.context.modeling.instance.validate_capabilities() + self.topology.validate_capabilities(self.context.modeling.instance) class FindHosts(Consumer): @@ -152,7 +153,7 @@ class FindHosts(Consumer): """ def consume(self): - self.context.modeling.instance.find_hosts() + self.topology.assign_hosts(self.context.modeling.instance) class ConfigureOperations(Consumer): @@ -161,7 +162,7 @@ class ConfigureOperations(Consumer): """ def consume(self): - self.context.modeling.instance.configure_operations() + self.topology.configure_operations(self.context.modeling.instance) class ServiceInstance(ConsumerChain): @@ -193,4 +194,5 @@ def dump(self): raw = self.context.modeling.instance_as_raw self.context.write(json_dumps(raw, indent=indent)) else: - self.context.modeling.instance.dump() + str_rep = self.topology.dump(self.context.modeling.instance) + self.context.write(str_rep) diff --git a/aria/parser/consumption/style.py b/aria/parser/consumption/style.py deleted file mode 100644 index 1b52cd34..00000000 --- a/aria/parser/consumption/style.py +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ...utils.console import Colored, indent -from ...utils.formatting import safe_repr - - -class Style(object): - def __init__(self, indentation=2): - self.indentation = indentation - - @property - def indent(self): - return indent(self.indentation) - - @staticmethod - def section(value): - return Colored.cyan(value, bold=True) - - @staticmethod - def type(value): - return Colored.blue(value, bold=True) - - @staticmethod - def node(value): - return Colored.red(value, bold=True) - - @staticmethod - def property(value): - return Colored.magenta(value, bold=True) - - @staticmethod - def literal(value): - return Colored.magenta(safe_repr(value)) - - @staticmethod - def meta(value): - return Colored.green(value) - - @staticmethod - def required(value): - return Colored.white(value) diff --git a/aria/parser/modeling/context.py b/aria/parser/modeling/context.py index 3d756177..d8a1f7ad 100644 --- a/aria/parser/modeling/context.py +++ b/aria/parser/modeling/context.py @@ -19,6 +19,10 @@ from ...utils.uuid import generate_uuid +# See: http://www.faqs.org/rfcs/rfc1035.html +ID_MAX_LENGTH = 63 + + class IdType(object): LOCAL_SERIAL = 0 """ @@ -61,7 +65,7 @@ def __init__(self): #self.id_type = IdType.LOCAL_SERIAL #self.id_type = IdType.LOCAL_RANDOM self.id_type = IdType.UNIVERSAL_RANDOM - self.id_max_length = 63 # See: http://www.faqs.org/rfcs/rfc1035.html + self.id_max_length = ID_MAX_LENGTH self.inputs = StrictDict(key_class=basestring) self._serial_id_counter = itertools.count(1) diff --git a/aria/parser/presentation/fields.py b/aria/parser/presentation/fields.py index 959bad1e..5c08d4a9 100644 --- a/aria/parser/presentation/fields.py +++ b/aria/parser/presentation/fields.py @@ -571,7 +571,7 @@ def _get_primitive(self, presentation, raw, value, context): def _dump_primitive(self, context, value): if hasattr(value, 'as_raw'): value = as_raw(value) - puts('%s: %s' % (self.name, context.style.literal(value))) + puts('%s: %s' % (self.name, context.style.literal_style(value))) # primitive list @@ -606,11 +606,11 @@ def _get_primitive_list(self, presentation, raw, value, context): def _dump_primitive_list(self, context, value): puts('%s:' % self.name) - with context.style.indent: + with context.style.indent(): for primitive in value: if hasattr(primitive, 'as_raw'): primitive = as_raw(primitive) - puts(context.style.literal(primitive)) + puts(context.style.literal_style(primitive)) # primitive dict @@ -635,11 +635,11 @@ def _get_primitive_dict(self, presentation, raw, value, context): def _dump_primitive_dict(self, context, value): puts('%s:' % self.name) - with context.style.indent: + with context.style.indent(): for v in value.itervalues(): if hasattr(v, 'as_raw'): v = as_raw(v) - puts(context.style.literal(v)) + puts(context.style.literal_style(v)) # object @@ -654,7 +654,7 @@ def _get_object(self, presentation, raw, value, context): def _dump_object(self, context, value): puts('%s:' % self.name) - with context.style.indent: + with context.style.indent(): if hasattr(value, '_dump'): value._dump(context) @@ -669,7 +669,7 @@ def _get_object_list(self, presentation, raw, value, context): def _dump_object_list(self, context, value): puts('%s:' % self.name) - with context.style.indent: + with context.style.indent(): for v in value: if hasattr(v, '_dump'): v._dump(context) @@ -685,7 +685,7 @@ def _get_object_dict(self, presentation, raw, value, context): def _dump_object_dict(self, context, value): puts('%s:' % self.name) - with context.style.indent: + with context.style.indent(): for v in value.itervalues(): if hasattr(v, '_dump'): v._dump(context) diff --git a/aria/parser/presentation/presentation.py b/aria/parser/presentation/presentation.py index fb71a1e3..3f9f86d8 100644 --- a/aria/parser/presentation/presentation.py +++ b/aria/parser/presentation/presentation.py @@ -37,13 +37,13 @@ def __init__(self, type_name, value, description, required): def _dump(self, context): if self.type is not None: - puts(context.style.type(self.type)) + puts(context.style.type_style(self.type)) if self.value is not None: - puts(context.style.literal(self.value)) + puts(context.style.literal_style(self.value)) if self.description is not None: - puts(context.style.meta(self.description)) + puts(context.style.meta_style(self.description)) if self.required is not None: - puts(context.style.required(self.required)) + puts(context.style.required_style(self.required)) class PresentationBase(HasCachedMethods): @@ -142,7 +142,7 @@ def _dump(self, context): if self._name: puts(context.style.node(self._name)) - with context.style.indent: + with context.style.indent(): self._dump_content(context) else: self._dump_content(context) @@ -162,7 +162,7 @@ def _dump_content(self, context, field_names=None): for field_name in self._iter_field_names(): # pylint: disable=no-member self._dump_field(context, field_name) else: - puts(context.style.literal(self._raw)) + puts(context.style.literal_style(self._raw)) def _dump_field(self, context, field_name): """ @@ -242,7 +242,7 @@ def _validate(self, context): def _dump(self, context): if hasattr(self._raw, '_dump'): puts(context.style.node(self._name)) - with context.style.indent: + with context.style.indent(): self._raw._dump(context) else: super(AsIsPresentation, self)._dump(context) diff --git a/aria/parser/reading/__init__.py b/aria/parser/reading/__init__.py index 065ca569..c110585d 100644 --- a/aria/parser/reading/__init__.py +++ b/aria/parser/reading/__init__.py @@ -24,8 +24,6 @@ JinjaReader JsonReader Locator - deepcopy_with_locators - copy_locators RawReader Reader ReaderSource @@ -36,7 +34,7 @@ from .raw import RawReader from .reader import Reader from .yaml import YamlReader -from .locator import (Locator, deepcopy_with_locators, copy_locators) +from .locator import Locator from .json import JsonReader from .jinja import JinjaReader from .context import ReadingContext @@ -57,8 +55,6 @@ 'ReadingContext', 'RawReader', 'Locator', - 'deepcopy_with_locators', - 'copy_locators', 'YamlReader', 'JsonReader', 'JinjaReader') diff --git a/aria/parser/reading/locator.py b/aria/parser/reading/locator.py index 965164d7..57b4d502 100644 --- a/aria/parser/reading/locator.py +++ b/aria/parser/reading/locator.py @@ -10,9 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from copy import deepcopy - - from ...utils.console import puts, Colored, indent @@ -120,35 +117,3 @@ def dump(self, key=None): def __str__(self): # Should be in same format as Issue.locator_as_str return '"%s":%d:%d' % (self.location, self.line, self.column) - - -def deepcopy_with_locators(value): - """ - Like :func:`deepcopy`, but also copies over locators. - """ - - res = deepcopy(value) - copy_locators(res, value) - return res - - -def copy_locators(target, source): - """ - Copies over ``_locator`` for all elements, recursively. - - Assumes that target and source have exactly the same list/dict structure. - """ - - locator = getattr(source, '_locator', None) - if locator is not None: - try: - setattr(target, '_locator', locator) - except AttributeError: - pass - - if isinstance(target, list) and isinstance(source, list): - for i, _ in enumerate(target): - copy_locators(target[i], source[i]) - elif isinstance(target, dict) and isinstance(source, dict): - for k, v in target.items(): - copy_locators(v, source[k]) diff --git a/aria/parser/validation/context.py b/aria/parser/validation/context.py index ef641bd3..da9eef6b 100644 --- a/aria/parser/validation/context.py +++ b/aria/parser/validation/context.py @@ -13,15 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .issue import Issue -from ...utils.threading import LockedList -from ...utils.collections import FrozenList -from ...utils.exceptions import print_exception -from ...utils.console import puts, Colored, indent -from ...utils.formatting import as_raw +from . import issue -class ValidationContext(object): +class ValidationContext(issue.ReporterMixin): """ Validation context. @@ -35,53 +30,7 @@ class ValidationContext(object): :vartype max_level: int """ - def __init__(self): + def __init__(self, *args, **kwargs): + super(ValidationContext, self).__init__(*args, **kwargs) self.allow_unknown_fields = False self.allow_primitive_coersion = False - self.max_level = Issue.ALL - - self._issues = LockedList() - - def report(self, message=None, exception=None, location=None, line=None, - column=None, locator=None, snippet=None, level=Issue.PLATFORM, issue=None): - if issue is None: - issue = Issue(message, exception, location, line, column, locator, snippet, level) - - # Avoid duplicate issues - with self._issues: - for i in self._issues: - if str(i) == str(issue): - return - - self._issues.append(issue) - - @property - def has_issues(self): - return len(self._issues) > 0 - - @property - def issues(self): - issues = [i for i in self._issues if i.level <= self.max_level] - issues.sort(key=lambda i: (i.level, i.location, i.line, i.column, i.message)) - return FrozenList(issues) - - @property - def issues_as_raw(self): - return [as_raw(i) for i in self.issues] - - def dump_issues(self): - issues = self.issues - if issues: - puts(Colored.blue('Validation issues:', bold=True)) - with indent(2): - for issue in issues: - puts(Colored.blue(issue.heading_as_str)) - details = issue.details_as_str - if details: - with indent(3): - puts(details) - if issue.exception is not None: - with indent(3): - print_exception(issue.exception) - return True - return False diff --git a/aria/parser/validation/issue.py b/aria/parser/validation/issue.py index db8065d0..42fc5808 100644 --- a/aria/parser/validation/issue.py +++ b/aria/parser/validation/issue.py @@ -15,8 +15,14 @@ from __future__ import absolute_import # so we can import standard 'collections' -from ...utils.collections import OrderedDict -from ...utils.type import full_type_name +from ...utils import ( + collections, + type, + threading, + exceptions, + console, + formatting +) class Issue(object): @@ -82,14 +88,14 @@ def __init__(self, message=None, exception=None, location=None, line=None, @property def as_raw(self): - return OrderedDict(( + return collections.OrderedDict(( ('level', self.level), ('message', self.message), ('location', self.location), ('line', self.line), ('column', self.column), ('snippet', self.snippet), - ('exception', full_type_name(self.exception) if self.exception else None))) + ('exception', type.full_type_name(self.exception) if self.exception else None))) @property def locator_as_str(self): @@ -124,3 +130,61 @@ def __str__(self): if details: heading_str += ', ' + details return heading_str + + +class ReporterMixin(object): + + Issue = Issue + + def __init__(self, *args, **kwargs): + super(ReporterMixin, self).__init__(*args, **kwargs) + self._issues = threading.LockedList() + self.max_level = self.Issue.ALL + + def report(self, message=None, exception=None, location=None, line=None, + column=None, locator=None, snippet=None, level=Issue.PLATFORM, issue=None): + if issue is None: + issue = self.Issue(message, exception, location, line, column, locator, snippet, level) + + # Avoid duplicate issues + with self._issues: + for i in self._issues: + if str(i) == str(issue): + return + + self._issues.append(issue) + + @property + def has_issues(self): + return len(self._issues) > 0 + + @property + def issues(self): + issues = [i for i in self._issues if i.level <= self.max_level] + issues.sort(key=lambda i: (i.level, i.location, i.line, i.column, i.message)) + return collections.FrozenList(issues) + + @property + def issues_as_raw(self): + return [formatting.as_raw(i) for i in self.issues] + + def extend_issues(self, *issues): + with self._issues: + self._issues.extend(*issues) + + def dump_issues(self): + issues = self.issues + if issues: + console.puts(console.Colored.blue('Validation issues:', bold=True)) + with console.indent(2): + for issue in issues: + console.puts(console.Colored.blue(issue.heading_as_str)) + details = issue.details_as_str + if details: + with console.indent(3): + console.puts(details) + if issue.exception is not None: + with console.indent(3): + exceptions.print_exception(issue.exception) + return True + return False diff --git a/aria/utils/__init__.py b/aria/utils/__init__.py index 2a957a9a..43dd882d 100644 --- a/aria/utils/__init__.py +++ b/aria/utils/__init__.py @@ -16,3 +16,50 @@ """ General-purpose utilities package. """ + +from . import ( + archive, + argparse, + caching, + collections, + console, + exceptions, + file, + formatting, + http, + imports, + openclose, + plugin, + process, + specification, + threading, + type, + uris, + uuid, + validation, + versions +) + + +__all__ = ( + 'archive', + 'argparse', + 'caching', + 'collections', + 'console', + 'exceptions', + 'file', + 'formatting', + 'http', + 'imports', + 'openclose', + 'plugin', + 'process', + 'specification', + 'threading', + 'type', + 'uris', + 'uuid', + 'validation', + 'versions' +) diff --git a/aria/utils/console.py b/aria/utils/console.py index 2f6f6220..81e8cf8e 100644 --- a/aria/utils/console.py +++ b/aria/utils/console.py @@ -19,19 +19,64 @@ import os import sys +from StringIO import StringIO from contextlib import contextmanager -from .formatting import safe_str from ..cli import color +from . import formatting _indent_string = '' +class TopologyStylizer(object): + def __init__(self, indentation=0): + self._str = StringIO() + self._indentation = indentation + + def write(self, string): + self._str.write(' ' * self._indentation) + self._str.write(string) + self._str.write(os.linesep) + + @contextmanager + def indent(self, indentation=2): + self._indentation += indentation + yield + self._indentation -= indentation + + @staticmethod + def type_style(value): + return Colored.blue(value, bold=True) + + @staticmethod + def node_style(value): + return Colored.red(value, bold=True) + + @staticmethod + def property_style(value): + return Colored.magenta(value, bold=True) + + @staticmethod + def literal_style(value): + return Colored.magenta(formatting.safe_repr(value)) + + @staticmethod + def required_style(value): + return Colored.white(value) + + @staticmethod + def meta_style(value): + return Colored.green(value) + + def __str__(self): + return self._str.getvalue() + + def puts(string='', newline=True, stream=sys.stdout): stream.write(_indent_string) - stream.write(safe_str(string)) + stream.write(formatting.safe_str(string)) if newline: stream.write(os.linesep) diff --git a/extensions/aria_extension_tosca/simple_v1_0/misc.py b/extensions/aria_extension_tosca/simple_v1_0/misc.py index 23beb3cd..a65ff41b 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/misc.py +++ b/extensions/aria_extension_tosca/simple_v1_0/misc.py @@ -52,7 +52,7 @@ def __init__(self, name=None, raw=None, container=None, cls=None): # pylint: dis def _dump(self, context): value = as_raw(self.value) - puts(context.style.meta(value)) + puts(context.style.meta_style(value)) @allow_unknown_fields diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py index 2620150c..1f90d292 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py +++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py @@ -497,8 +497,11 @@ def get(name, default=None): def create_workflow_operation_template_model(context, service_template, policy): - model = OperationTemplate(name=policy._name, - service_template=service_template) + model = OperationTemplate(name=policy._name) + # since we use backpopulates, these fields are populated upon commit, we get a weird(temporary) + # behavior where in previous code service_template.workflow_templates is a dict which has None + # as key for the value of model. + service_template.workflow_templates[model.name] = model if policy.description: model.description = policy.description.value @@ -606,7 +609,7 @@ def create_parameter_model_from_value(template_parameter, template_parameter_nam def create_parameter_models_from_assignments(properties, source_properties, model_cls): if source_properties: for property_name, prop in source_properties.iteritems(): - properties[property_name] = model_cls(name=property_name, # pylint: disable=unexpected-keyword-arg + properties[property_name] = model_cls(name=property_name, # pylint: disable=unexpected-keyword-arg type_name=prop.value.type, value=prop.value.value, description=prop.value.description) diff --git a/tests/parser/service_templates.py b/tests/parser/service_templates.py index 55cd4ad1..9e8fcae1 100644 --- a/tests/parser/service_templates.py +++ b/tests/parser/service_templates.py @@ -47,6 +47,7 @@ def consume_use_case(use_case_name, consumer_class_name='instance', cache=True): assert not context.validation.has_issues return context, dumper + def consume_types_use_case(use_case_name, consumer_class_name='instance', cache=True): cachedmethod.ENABLED = cache uri = get_service_template_uri('tosca-simple-1.0', 'types', use_case_name, @@ -61,12 +62,23 @@ def consume_types_use_case(use_case_name, consumer_class_name='instance', cache= assert not context.validation.has_issues return context, dumper + def consume_node_cellar(consumer_class_name='instance', cache=True): + consume_test_case( + get_service_template_uri('tosca-simple-1.0', 'node-cellar', 'node-cellar.yaml'), + consumer_class_name=consumer_class_name, + inputs_uri=get_service_template_uri('tosca-simple-1.0', 'node-cellar', 'inputs.yaml'), + cache=cache + + ) + + +def consume_test_case(uri, inputs_uri=None, consumer_class_name='instance', cache=True): cachedmethod.ENABLED = cache - uri = get_service_template_uri('tosca-simple-1.0', 'node-cellar', 'node-cellar.yaml') + uri = get_service_template_uri(uri) context = create_context(uri) - context.args.append('--inputs=' + get_service_template_uri('tosca-simple-1.0', 'node-cellar', - 'inputs.yaml')) + if inputs_uri: + context.args.append('--inputs=' + get_service_template_uri(inputs_uri)) consumer, dumper = create_consumer(context, consumer_class_name) consumer.consume() context.validation.dump_issues() diff --git a/tests/parser/test_reqs_caps.py b/tests/parser/test_reqs_caps.py new file mode 100644 index 00000000..e92aec42 --- /dev/null +++ b/tests/parser/test_reqs_caps.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .service_templates import consume_test_case +from ..helpers import get_service_template_uri + + +def test_satisfy_capability_type(): + consume_reqs_caps_template1('instance') + + +def consume_reqs_caps_template1(consumer_class_name, cache=True): + consume_test_case( + get_service_template_uri('tosca-simple-1.0', 'reqs_caps', 'reqs_caps1.yaml'), + consumer_class_name=consumer_class_name, + cache=cache + ) diff --git a/tests/resources/service-templates/tosca-simple-1.0/reqs_caps/reqs_caps1.yaml b/tests/resources/service-templates/tosca-simple-1.0/reqs_caps/reqs_caps1.yaml new file mode 100644 index 00000000..466a78e3 --- /dev/null +++ b/tests/resources/service-templates/tosca-simple-1.0/reqs_caps/reqs_caps1.yaml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tosca_definitions_version: tosca_simple_yaml_1_0 + +capability_types: + Socket: + derived_from: tosca.capabilities.Root + +node_types: + Socket: + derived_from: tosca.nodes.Root + capabilities: + socket: Socket + + Plug: + derived_from: tosca.nodes.Root + requirements: + - plug: + capability: Socket + +topology_template: + node_templates: + socket: + type: Socket + + plug: + type: Plug \ No newline at end of file diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 8ca14806..8a4d6133 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -23,8 +23,6 @@ MetaData ) -from aria.modeling import models - class TestFileSystem(object):