From 33238199fcf6db59b11cc778d7f61609031bde8f Mon Sep 17 00:00:00 2001 From: Tal Liron Date: Wed, 16 Nov 2016 16:05:39 -0600 Subject: [PATCH] ARIA-22 Add CLI commands for parser --- README.md | 188 +++++++++++++++++- aria/__init__.py | 20 ++ aria/cli/args_parser.py | 81 +++++++- aria/cli/cli.py | 16 +- aria/cli/commands.py | 139 ++++++++++--- aria/parser/__init__.py | 28 +-- aria/parser/specification.py | 98 +++++---- aria/utils/argparse.py | 114 +++++++++++ extensions/aria_extension_tosca/__init__.py | 9 +- .../simple_v1_0/functions.py | 3 +- .../simple_v1_0/presentation/field_getters.py | 2 +- 11 files changed, 599 insertions(+), 99 deletions(-) create mode 100644 aria/utils/argparse.py diff --git a/README.md b/README.md index b945a29e..a150d09a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,188 @@ -Aria +ARIA ==== -See http://ariatosca.org/ +[ARIA](http://ariatosca.org/) is a minimal TOSCA orchestrator, as well as a platform for building +TOSCA-based products. Its features can be accessed via a well-documented Python API. + +On its own, ARIA provides built-in tools for blueprint validation and for creating ready-to-run +service instances. + +ARIA adheres strictly and meticulously to the +[TOSCA Simple Profile v1.0 cos01 specification](http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/cos01/TOSCA-Simple-Profile-YAML-v1.0-cos01.html), +providing state-of-the-art validation at seven different levels: + +
    +
  1. Platform errors. E.g. network, hardware, or even an internal bug in ARIA (let us know, + please!).
    2. +
  2. Syntax and format errors. E.g. non-compliant YAML, XML, JSON.
    3. +
  3. Field validation. E.g. assigning a string where an integer is expected, using a list instead of + a dict.
    4. +
  4. Relationships between fields within a type. This is "grammar" as it applies to rules for + setting the values of fields in relation to each other.
    5. +
  5. Relationships between types. E.g. referring to an unknown type, causing a type inheritance + loop.
    6. +
  6. Topology. These errors happen if requirements and capabilities cannot be matched in order to + assemble a valid topology.
    7. +
  7. External dependencies. These errors happen if requirement/capability matching fails due to + external resources missing, e.g. the lack of a valid virtual machine, API credentials, etc. +
    8. +
+ +Validation errors include a plain English message and when relevant the exact location (file, row, +column) of the data the caused the error. + +The ARIA API documentation always links to the relevant section of the specification, and likewise +we provide an annotated version of the specification that links back to the API documentation. + + +Quick Start +----------- + +You need Python 2.6 or 2.7. Python 3+ is not currently supported. + +To install, we recommend using [pip](https://pip.pypa.io/) and a +[virtualenv](https://virtualenv.pypa.io/en/stable/). In Debian-based systems: + + sudo apt install python-setuptools + sudo -H easy_install pip + sudo -H pip install virtualenv + +To install the latest development snapshot of ARIA: + + virtualenv env + . env/bin/activate + pip install git+http://git-wip-us.apache.org/repos/asf/incubator-ariatosca.git + +To test it, let's create a service instance from a TOSCA blueprint: + + aria parse blueprints/tosca/node-cellar/node-cellar.yaml + +You can also get it in JSON or YAML formats: + + aria parse blueprints/tosca/node-cellar/node-cellar.yaml --json + +Or get an overview of the relationship graph: + + aria parse blueprints/tosca/node-cellar/node-cellar.yaml --graph + +You can provide inputs as JSON, overriding default values provided in the blueprint + + aria parse blueprints/tosca/node-cellar/node-cellar.yaml --inputs='{"openstack_credential": {"user": "username"}}' + +Instead of providing them explicitly, you can also provide them in a file or URL, in either JSON or +YAML. If you do so, the value must end in ".json" or ".yaml": + + aria parse blueprints/tosca/node-cellar/node-cellar.yaml --inputs=blueprints/tosca/node-cellar/inputs.yaml + + +CLI +--- + +Though ARIA is fully exposed as an API, it also comes with a CLI tool to allow you to work from the +shell: + + aria parse blueprints/tosca/node-cellar/node-cellar.yaml instance + +The `parse` command supports the following directives to create variations of the default consumer +chain: + +* `presentation`: emits a colorized textual representation of the Python presentation classes + wrapping the blueprint. +* `model`: emits a colorized textual representation of the complete service model derived from the + validated blueprint. This includes all the node templates, with their requirements satisfied at + the level of relating to other node templates. +* `types`: emits a colorized textual representation of the type hierarchies. +* `instance`: **this is the default command**; emits a colorized textual representation of a + service instance instantiated from the service model. Here the node templates are each used to + create one or more nodes, with the appropriate relationships between them. Note that every time + you run this consumer, you will get a different set of node IDs. Use `--graph` to see just the + node relationship graph. + +For all these commands, you can also use `--json` or `--yaml` flags to emit in those formats. + +Additionally, The CLI tool lets you specify the complete classname of your own custom consumer to +chain at the end of the default consumer chain, after `instance`. + +Your custom consumer can be an entry point into a powerful TOSCA-based tool or application, such as +an orchestrator, a graphical modeling tool, etc. + + +Development +----------- + +Instead of installing with `pip`, it would be easier to work directly with the source files: + + pip install virtualenv + virtualenv env + . env/bin/activate + git clone http://git-wip-us.apache.org/repos/asf/incubator-ariatosca.git ariatosca + cd ariatosca + pip install -e . + +To run tests: + + pip install tox + tox + +Here's a quick example of using the API to parse YAML text into a service instance: + + from aria import install_aria_extensions + from aria.parser.consumption import ConsumptionContext, ConsumerChain, Read, Validate, Model, Instance + from aria.parser.loading import LiteralLocation + + def parse_text(payload, file_search_paths=[]): + context = ConsumptionContext() + context.presentation.location = LiteralLocation(payload) + context.loading.file_search_paths += file_search_paths + ConsumerChain(context, (Read, Validate, Model, Instance)).consume() + if not context.validation.dump_issues(): + return context.modeling.instance + return None + + install_aria_extensions() + + print parse_text(""" + tosca_definitions_version: tosca_simple_yaml_1_0 + topology_template: + node_templates: + MyNode: + type: tosca.nodes.Compute + """) + + +Parser API Architecture +----------------------- + +ARIA's parsing engine comprises individual "consumers" (in the `aria.parser.consumption` package) +that do things with blueprints. When chained together, each performs a different task, adds its own +validations, and can provide its own output. + +Parsing happens in five phases, represented in five packages: + +* `aria.parser.loading`: Loaders are used to read the TOSCA data, usually as text. For example + UriTextLoader will load text from URIs (including files). +* `aria.parser.reading`: Readers convert data from the loaders into agnostic raw data. For + example, `YamlReader` converts YAML text into Python dicts, lists, and primitives. +* `aria.parser.presentation`: Presenters wrap the agnostic raw data in a nice + Python facade (a "presentation") that makes it much easier to work with the data, including + utilities for validation, querying, etc. Note that presenters are _wrappers_: the agnostic raw + data is always maintained intact, and can always be accessed directly or written back to files. +* `aria.parser.modeling.model`: Here the topology is normalized into a coherent structure of + node templates, requirements, and capabilities. Types are inherited and properties are assigned. + The service model is a _new_ structure, which is not mapped to the YAML. In fact, it is possible + to generate the model programmatically, or from a DSL parser other than TOSCA. +* `aria.parser.modeling.instance`: The service instance is an instantiated service model. Node + templates turn into node instances (with unique IDs), and requirements are satisfied by matching + them to capabilities. This is where level 5 validation errors are detected (see above). + +The phases do not have to be used in order. Indeed, consumers do not have to be used at all: ARIA +can be used to _produce_ blueprints. For example, it is possible to fill in the +`aria.parser.presentation` classes programmatically, in Python, and then write the presentation +to a YAML file as compliant TOSCA. The same technique can be used to convert from one DSL (consume +it) to another (write it). + +The term "agnostic raw data" (ARD?) appears often in the documentation. It denotes data structures +comprising _only_ Python dicts, lists, and primitives, such that they can always be converted to and +from language-agnostic formats such as YAML, JSON, and XML. A considerable effort has been made to +conserve the agnostic raw data at all times. Thus, though ARIA makes good use of the dynamic power +of Python, you will _always_ be able to use ARIA with other systems. diff --git a/aria/__init__.py b/aria/__init__.py index eb04e393..3f81f989 100644 --- a/aria/__init__.py +++ b/aria/__init__.py @@ -17,6 +17,9 @@ Aria top level package """ +import sys +import pkgutil + from .VERSION import version as __version__ from .orchestrator.decorators import workflow, operation @@ -38,6 +41,23 @@ _resource_storage = {} +def install_aria_extensions(): + """ + Iterates all Python packages with names beginning with :code:`aria_extension_` and calls + their :code:`install_aria_extension` function if they have it. + """ + + for loader, module_name, _ in pkgutil.iter_modules(): + if module_name.startswith('aria_extension_'): + module = loader.find_module(module_name).load_module(module_name) + + if hasattr(module, 'install_aria_extension'): + module.install_aria_extension() + + # Loading the module has contaminated sys.modules, so we'll clean it up + del sys.modules[module_name] + + def application_model_storage(driver): """ Initiate model storage for the supplied storage driver diff --git a/aria/cli/args_parser.py b/aria/cli/args_parser.py index f40919ca..56fd074c 100644 --- a/aria/cli/args_parser.py +++ b/aria/cli/args_parser.py @@ -20,6 +20,8 @@ import argparse from functools import partial +from ..utils.argparse import ArgumentParser + NO_VERBOSE = 0 @@ -48,10 +50,6 @@ def _wrapper(parser): action='count', default=NO_VERBOSE, help='Set verbosity level (can be passed multiple times)') - sub_parser.add_argument( - '-d', '--deployment-id', - required=True, - help='A unique ID for the deployment') func(sub_parser) return sub_parser return _wrapper @@ -61,14 +59,16 @@ def config_parser(parser=None): """ Top level argparse configuration """ - parser = parser or argparse.ArgumentParser( - prog='Aria', - description="Aria's Command Line Interface", + parser = parser or ArgumentParser( + prog='ARIA', + description="ARIA's Command Line Interface", formatter_class=SmartFormatter) parser.add_argument('-v', '--version', action='version') sub_parser = parser.add_subparsers(title='Commands', dest='command') add_init_parser(sub_parser) add_execute_parser(sub_parser) + add_parse_parser(sub_parser) + add_spec_parser(sub_parser) return parser @@ -80,6 +80,10 @@ def add_init_parser(init): """ ``init`` command parser configuration """ + init.add_argument( + '-d', '--deployment-id', + required=True, + help='A unique ID for the deployment') init.add_argument( '-p', '--blueprint-path', dest='blueprint_path', @@ -109,6 +113,10 @@ def add_execute_parser(execute): """ ``execute`` command parser configuration """ + execute.add_argument( + '-d', '--deployment-id', + required=True, + help='A unique ID for the deployment') execute.add_argument( '-w', '--workflow', dest='workflow_id', @@ -132,3 +140,62 @@ def add_execute_parser(execute): default=1, type=int, help='How many seconds to wait before each task is retried') + + +@sub_parser_decorator( + name='parse', + help='Parse a blueprint', + formatter_class=SmartFormatter) +def add_parse_parser(parse): + """ + ``parse`` command parser configuration + """ + parse.add_argument( + 'uri', + help='URI or file path to profile') + parse.add_argument( + 'consumer', + nargs='?', + default='instance', + help='consumer class name (full class path or short name)') + parse.add_argument( + '--loader-source', + default='aria.parser.loading.DefaultLoaderSource', + help='loader source class for the parser') + parse.add_argument( + '--reader-source', + default='aria.parser.reading.DefaultReaderSource', + help='reader source class for the parser') + parse.add_argument( + '--presenter-source', + default='aria.parser.presentation.DefaultPresenterSource', + help='presenter source class for the parser') + parse.add_argument( + '--presenter', + help='force use of this presenter class in parser') + parse.add_argument( + '--prefix', nargs='*', + help='prefixes for imports') + parse.add_flag_argument( + 'debug', + help_true='print debug info', + help_false='don\'t print debug info') + parse.add_flag_argument( + 'cached-methods', + help_true='enable cached methods', + help_false='disable cached methods', + default=True) + + +@sub_parser_decorator( + name='spec', + help='Specification tool', + formatter_class=SmartFormatter) +def add_spec_parser(spec): + """ + ``spec`` command parser configuration + """ + spec.add_argument( + '--csv', + action='store_true', + help='output as CSV') diff --git a/aria/cli/cli.py b/aria/cli/cli.py index a4046ac2..ad9784ce 100644 --- a/aria/cli/cli.py +++ b/aria/cli/cli.py @@ -19,16 +19,20 @@ import logging -from aria.logger import ( +from .. import install_aria_extensions +from ..logger import ( create_logger, create_console_log_handler, create_file_log_handler, LoggerMixin, ) +from ..utils.exceptions import print_exception from .args_parser import config_parser from .commands import ( InitCommand, ExecuteCommand, + ParseCommand, + SpecCommand, ) __version__ = '0.1.0' @@ -44,6 +48,8 @@ def __init__(self, *args, **kwargs): self.commands = { 'init': InitCommand.with_logger(base_logger=self.logger), 'execute': ExecuteCommand.with_logger(base_logger=self.logger), + 'parse': ParseCommand.with_logger(base_logger=self.logger), + 'spec': SpecCommand.with_logger(base_logger=self.logger), } def __enter__(self): @@ -67,18 +73,22 @@ def run(self): Parses user arguments and run the appropriate command """ parser = config_parser() - args = parser.parse_args() + args, unknown_args = parser.parse_known_args() command_handler = self.commands[args.command] self.logger.debug('Running command: {args.command} handler: {0}'.format( command_handler, args=args)) - command_handler(args) + try: + command_handler(args, unknown_args) + except Exception as e: + print_exception(e) def main(): """ CLI entry point """ + install_aria_extensions() create_logger( handlers=[ create_console_log_handler(), diff --git a/aria/cli/commands.py b/aria/cli/commands.py index 341b13ab..57118a78 100644 --- a/aria/cli/commands.py +++ b/aria/cli/commands.py @@ -20,20 +20,34 @@ import json import os import sys +import csv from glob import glob from importlib import import_module -from dsl_parser.parser import parse_from_path -from dsl_parser.tasks import prepare_deployment_plan from yaml import safe_load, YAMLError -from aria import application_model_storage, application_resource_storage -from aria.orchestrator.context.workflow import WorkflowContext -from aria.logger import LoggerMixin -from aria.storage import FileSystemModelDriver, FileSystemResourceDriver -from aria.utils.application import StorageManager -from aria.orchestrator.workflows.core.engine import Engine -from aria.orchestrator.workflows.executor.thread import ThreadExecutor +from .. import (application_model_storage, application_resource_storage) +from ..logger import LoggerMixin +from ..storage import (FileSystemModelDriver, FileSystemResourceDriver) +from ..orchestrator.context.workflow import WorkflowContext +from ..orchestrator.workflows.core.engine import Engine +from ..orchestrator.workflows.executor.thread import ThreadExecutor +from ..parser import (DSL_SPECIFICATION_PACKAGES, iter_specifications) +from ..parser.consumption import ( + ConsumptionContext, + ConsumerChain, + Read, + Validate, + Model, + Types, + Inputs, + Instance +) +from ..parser.loading import (UriLocation, URI_LOADER_PREFIXES) +from ..utils.application import StorageManager +from ..utils.caching import cachedmethod +from ..utils.console import (puts, Colored, indent) +from ..utils.imports import (import_fullname, import_modules) from .exceptions import ( AriaCliFormatInputsError, AriaCliYAMLInputsError, @@ -49,9 +63,6 @@ ) -####################################### - - class BaseCommand(LoggerMixin): """ Base class for CLI commands @@ -60,7 +71,7 @@ class BaseCommand(LoggerMixin): def __repr__(self): return 'AriaCli({cls.__name__})'.format(cls=self.__class__) - def __call__(self, args_namespace): + def __call__(self, args_namespace, unknown_args): """ __call__ method is called when running command :param args_namespace: @@ -128,8 +139,8 @@ class InitCommand(BaseCommand): _IN_VIRTUAL_ENV = hasattr(sys, 'real_prefix') - def __call__(self, args_namespace): - super(InitCommand, self).__call__(args_namespace) + def __call__(self, args_namespace, unknown_args): + super(InitCommand, self).__call__(args_namespace, unknown_args) self._workspace_setup() inputs = self.parse_inputs(args_namespace.input) if args_namespace.input else None plan, deployment_plan = self._parse_blueprint(args_namespace.blueprint_path, inputs) @@ -164,10 +175,8 @@ def _workspace_setup(self): return local_storage() def _parse_blueprint(self, blueprint_path, inputs=None): - plan = parse_from_path(blueprint_path) - self.logger.info('blueprint parsed successfully') - deployment_plan = prepare_deployment_plan(plan=plan.copy(), inputs=inputs) - return plan, deployment_plan + # TODO + pass @staticmethod def _create_storage( @@ -206,8 +215,8 @@ class ExecuteCommand(BaseCommand): ``execute`` command implementation """ - def __call__(self, args_namespace): - super(ExecuteCommand, self).__call__(args_namespace) + def __call__(self, args_namespace, unknown_args): + super(ExecuteCommand, self).__call__(args_namespace, unknown_args) parameters = (self.parse_inputs(args_namespace.parameters) if args_namespace.parameters else {}) resource_storage = application_resource_storage( @@ -290,8 +299,92 @@ def _load_workflow_handler(handler_path): module = import_module(module_name) return getattr(module, spec_handler_name) except ImportError: - # todo: exception handler + # TODO: exception handler raise except AttributeError: - # todo: exception handler + # TODO: exception handler raise + + +class ParseCommand(BaseCommand): + def __call__(self, args_namespace, unknown_args): + super(ParseCommand, self).__call__(args_namespace, unknown_args) + + if args_namespace.prefix: + for prefix in args_namespace.prefix: + URI_LOADER_PREFIXES.append(prefix) + + cachedmethod.ENABLED = args_namespace.cached_methods + + context = ParseCommand.create_context_from_namespace(args_namespace) + context.args = unknown_args + + consumer = ConsumerChain(context, (Read, Validate)) + + consumer_class_name = args_namespace.consumer + dumper = None + if consumer_class_name == 'presentation': + dumper = consumer.consumers[0] + elif consumer_class_name == 'model': + consumer.append(Model) + elif consumer_class_name == 'types': + consumer.append(Model, Types) + elif consumer_class_name == 'instance': + consumer.append(Model, Inputs, Instance) + else: + consumer.append(Model, Inputs, Instance) + consumer.append(import_fullname(consumer_class_name)) + + if dumper is None: + # Default to last consumer + dumper = consumer.consumers[-1] + + consumer.consume() + + if not context.validation.dump_issues(): + dumper.dump() + + @staticmethod + def create_context_from_namespace(namespace, **kwargs): + args = vars(namespace).copy() + args.update(kwargs) + return ParseCommand.create_context(**args) + + @staticmethod + def create_context(uri, loader_source, reader_source, presenter_source, presenter, debug, **kwargs): + context = ConsumptionContext() + context.loading.loader_source = import_fullname(loader_source)() + context.reading.reader_source = import_fullname(reader_source)() + context.presentation.location = UriLocation(uri) if isinstance(uri, basestring) else uri + context.presentation.presenter_source = import_fullname(presenter_source)() + context.presentation.presenter_class = import_fullname(presenter) + context.presentation.print_exceptions = debug + return context + + +class SpecCommand(BaseCommand): + def __call__(self, args_namespace, unknown_args): + super(SpecCommand, self).__call__(args_namespace, unknown_args) + + # Make sure that all @dsl_specification decorators are processed + for pkg in DSL_SPECIFICATION_PACKAGES: + import_modules(pkg) + + # TODO: scan YAML documents as well + + if args_namespace.csv: + writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + writer.writerow(('Specification', 'Section', 'Code', 'URL')) + for spec, sections in iter_specifications(): + for section, details in sections: + writer.writerow((spec, section, details['code'], details['url'])) + + else: + for spec, sections in iter_specifications(): + puts(Colored.cyan(spec)) + with indent(2): + for section, details in sections: + puts(Colored.blue(section)) + with indent(2): + for k, v in details.iteritems(): + puts('%s: %s' % (Colored.magenta(k), v)) diff --git a/aria/parser/__init__.py b/aria/parser/__init__.py index c2e2fd69..2a83cd47 100644 --- a/aria/parser/__init__.py +++ b/aria/parser/__init__.py @@ -13,28 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -import pkgutil +from .specification import (DSL_SPECIFICATION_PACKAGES, DSL_SPECIFICATION_URLS, dsl_specification, + iter_specifications) -from .specification import (DSL_SPECIFICATION, DSL_SPECIFICATION_PACKAGES, dsl_specification, - iter_spec) - - -def install_aria_extensions(): - """ - Iterates all Python packages with names beginning with :code:`aria_extension_` and calls - their :code:`install_aria_extension` function if they have it. - """ - - for loader, module_name, _ in pkgutil.iter_modules(): - if module_name.startswith('aria_extension_'): - module = loader.find_module(module_name).load_module(module_name) - - if hasattr(module, 'install_aria_extension'): - module.install_aria_extension() - - # Loading the module has contaminated sys.modules, so we'll clean it up - del sys.modules[module_name] MODULES = ( 'consumption', @@ -46,8 +27,7 @@ def install_aria_extensions(): __all__ = ( 'MODULES', - 'install_aria_extensions', - 'DSL_SPECIFICATION', 'DSL_SPECIFICATION_PACKAGES', + 'DSL_SPECIFICATION_URLS', 'dsl_specification', - 'iter_spec') + 'iter_specifications') diff --git a/aria/parser/specification.py b/aria/parser/specification.py index 6e022bc8..1c7e1f21 100644 --- a/aria/parser/specification.py +++ b/aria/parser/specification.py @@ -13,16 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + from ..utils.collections import OrderedDict from ..utils.formatting import full_type_name -DSL_SPECIFICATION = {} -DSL_SPECIFICATION_PACKAGES = [] -URL = { - 'tosca-simple-profile-1.0': 'http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/' - 'csprd02/TOSCA-Simple-Profile-YAML-v1.0-csprd02.html', - 'tosca-simple-nfv-1.0': 'http://docs.oasis-open.org/tosca/tosca-nfv/v1.0/tosca-nfv-v1.0.html'} +DSL_SPECIFICATION_PACKAGES = [] +DSL_SPECIFICATION_URLS = {} +_DSL_SPECIFICATIONS = {} def dsl_specification(section, spec): @@ -31,48 +30,73 @@ def dsl_specification(section, spec): Used for documentation and standards compliance. """ - def decorator(obj): - specification = DSL_SPECIFICATION.get(spec) + specification = _DSL_SPECIFICATIONS.get(spec) + if specification is None: specification = {} - DSL_SPECIFICATION[spec] = specification + _DSL_SPECIFICATIONS[spec] = specification + if section in specification: - raise Exception('you cannot specify the same @dsl_specification twice, consider adding' - ' \'-1\', \'-2\', etc.: %s, %s' % (spec, section)) - - url = URL.get(spec) - if url: - doc = obj.__doc__ - if doc is not None: - url_start = doc.find(url) - if url_start != -1: - url_end = doc.find('>', url_start + len(url)) - if url_end != -1: - url = doc[url_start:url_end] + raise Exception('you cannot specify the same @dsl_specification twice, consider' + ' adding \'-1\', \'-2\', etc.: %s, %s' % (spec, section)) specification[section] = OrderedDict(( ('code', full_type_name(obj)), - ('url', url))) + ('doc', obj.__doc__))) + try: - setattr(obj, DSL_SPECIFICATION, {section: section, spec: spec}) + setattr(obj, '_dsl_specifications', {section: section, spec: spec}) except BaseException: pass + return obj + return decorator -def iter_spec(spec): - sections = DSL_SPECIFICATION[spec] - keys = sections.keys() - def key(value): - try: - parts = value.split('-', 1) - first = (int(v) for v in parts[0].split('.')) - second = parts[1] if len(parts) > 1 else None - return (first, second) - except ValueError: - return value - keys.sort(key=key) - for key in keys: - yield key, sections[key] +def iter_specifications(): + """ + Iterates all specification assignments in the codebase. + """ + def iter_sections(spec, sections): + for k in sorted(sections.keys(), key=_section_key): + details = OrderedDict() + details['code'] = sections[k]['code'] + yield k, _fix_details(sections[k], spec) + + for spec, sections in _DSL_SPECIFICATIONS.iteritems(): + yield spec, iter_sections(spec, sections) + + +# Utils + +def _section_key(value): + try: + parts = value.split('-', 1) + first = (int(v) for v in parts[0].split('.')) + second = parts[1] if len(parts) > 1 else None + return (first, second) + except ValueError: + return value + + +def _fix_details(details, spec): + code = details.get('code') + doc = details.get('doc') + url = DSL_SPECIFICATION_URLS.get(spec) + + if (url is not None) and (doc is not None): + # Look for a URL in ReST docstring that begins with our url + pattern = r']+)>' + match = re.search(pattern, doc) + if match: + url = re.sub(r'\s+', '', match.group(1)) + + return OrderedDict(( + ('code', code), + ('url', url))) diff --git a/aria/utils/argparse.py b/aria/utils/argparse.py new file mode 100644 index 00000000..365c1484 --- /dev/null +++ b/aria/utils/argparse.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import # so we can import standard 'argparse' + +from argparse import ArgumentParser as BaseArgumentParser + + +class ArgumentParser(BaseArgumentParser): + """ + Enhanced argument parser. + + Applied patch to fix `this issue `__. + """ + + def add_flag_argument(self, name, help_true=None, help_false=None, default=False): + """ + Adds a flag argument as two arguments: :code:`--my-flag` and :code:`--no-my-flag`. + """ + + dest = name.replace('-', '_') + + if default: + if help_true is not None: + help_true += ' (default)' + else: + help_true = '(default)' + else: + if help_false is not None: + help_false += ' (default)' + else: + help_false = '(default)' + + group = self.add_mutually_exclusive_group() + group.add_argument('--%s' % name, action='store_true', help=help_true) + group.add_argument('--no-%s' % name, dest=dest, action='store_false', help=help_false) + + self.set_defaults(**{dest: default}) + + def _parse_optional(self, arg_string): + + if self._is_positional(arg_string): + return None + + # if the option string is present in the parser, return the action + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None + + # if the option string before the "=" is present, return the action + if '=' in arg_string: + option_string, explicit_arg = arg_string.split('=', 1) + if option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, explicit_arg + + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join( + [option_string for action, option_string, explicit_arg in option_tuples]) + tup = arg_string, options + self.error('ambiguous option: %s could match %s' % tup) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple = option_tuples + return option_tuple + + # if it was not found as an option, but it looks like a negative + # number, it was meant to be positional + # unless there are negative-number-like options + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + + # it was meant to be an optional but there is no such option + # in this parser (though it might be a valid option in a subparser) + return None, arg_string, None + + def _is_positional(self, arg_string): + # if it's an empty string, it was meant to be a positional + if not arg_string: + return True + + # if it doesn't start with a prefix, it was meant to be positional + if not arg_string[0] in self.prefix_chars: + return True + + # if it's just a single character, it was meant to be positional + if len(arg_string) == 1: + return True + + # if it contains a space, it was meant to be a positional + if ' ' in arg_string and arg_string[0] not in self.prefix_chars: + return True + + return False diff --git a/extensions/aria_extension_tosca/__init__.py b/extensions/aria_extension_tosca/__init__.py index 188b80f2..54e1c846 100644 --- a/extensions/aria_extension_tosca/__init__.py +++ b/extensions/aria_extension_tosca/__init__.py @@ -15,7 +15,7 @@ import os.path -from aria.parser import DSL_SPECIFICATION_PACKAGES +from aria.parser import (DSL_SPECIFICATION_PACKAGES, DSL_SPECIFICATION_URLS) from aria.parser.presentation import PRESENTER_CLASSES from aria.parser.loading import URI_LOADER_PREFIXES @@ -32,6 +32,13 @@ def install_aria_extension(): # DSL specification DSL_SPECIFICATION_PACKAGES.append('aria_extension_tosca') + DSL_SPECIFICATION_URLS['yaml-1.1'] = \ + 'http://yaml.org' + DSL_SPECIFICATION_URLS['tosca-simple-1.0'] = \ + 'http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/cos01' \ + '/TOSCA-Simple-Profile-YAML-v1.0-cos01.html' + DSL_SPECIFICATION_URLS['tosca-simple-nfv-1.0'] = \ + 'http://docs.oasis-open.org/tosca/tosca-nfv/v1.0/tosca-nfv-v1.0.html' # Imports the_dir = os.path.dirname(__file__) diff --git a/extensions/aria_extension_tosca/simple_v1_0/functions.py b/extensions/aria_extension_tosca/simple_v1_0/functions.py index 79079dfb..d17a960f 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/functions.py +++ b/extensions/aria_extension_tosca/simple_v1_0/functions.py @@ -17,7 +17,8 @@ from aria.utils.collections import FrozenList from aria.utils.formatting import as_raw, safe_repr -from aria.parser import (dsl_specification, InvalidValueError) +from aria.parser import dsl_specification +from aria.parser.exceptions import InvalidValueError from aria.parser.modeling import (Function, CannotEvaluateFunctionException) from aria.parser.validation import Issue diff --git a/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py b/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py index a2adb1e7..f14164ab 100644 --- a/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py +++ b/extensions/aria_extension_tosca/simple_v1_0/presentation/field_getters.py @@ -14,7 +14,7 @@ # limitations under the License. from aria.utils.formatting import safe_repr -from aria.parser import InvalidValueError +from aria.parser.exceptions import InvalidValueError def data_type_class_getter(cls):