From e5b13fdbc293ebaa342c33bd93a8e484df203d14 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 1 Mar 2019 10:20:32 -0800 Subject: [PATCH 01/13] Add normalize_parameters and evaluate_paramters Signed-off-by: Shane Loretz --- launch_ros/launch_ros/actions/node.py | 81 ++------ launch_ros/launch_ros/parameters_type.py | 60 ++++++ launch_ros/launch_ros/utilities/__init__.py | 4 + .../utilities/evaluate_parameters.py | 80 ++++++++ .../utilities/normalize_parameters.py | 184 ++++++++++++++++++ .../test/test_launch_ros/actions/test_node.py | 52 ++--- .../test_normalize_parameters.py | 173 ++++++++++++++++ 7 files changed, 530 insertions(+), 104 deletions(-) create mode 100644 launch_ros/launch_ros/parameters_type.py create mode 100644 launch_ros/launch_ros/utilities/evaluate_parameters.py create mode 100644 launch_ros/launch_ros/utilities/normalize_parameters.py create mode 100644 test_launch_ros/test/test_launch_ros/test_normalize_parameters.py diff --git a/launch_ros/launch_ros/actions/node.py b/launch_ros/launch_ros/actions/node.py index 05a976d04..4b4139599 100644 --- a/launch_ros/launch_ros/actions/node.py +++ b/launch_ros/launch_ros/actions/node.py @@ -14,6 +14,7 @@ """Module for the Node action.""" +from collections.abc import Mapping import logging import os import pathlib @@ -25,12 +26,10 @@ from typing import Text # noqa: F401 from typing import Tuple -from launch import Substitution from launch.action import Action from launch.actions import ExecuteProcess from launch.launch_context import LaunchContext from launch.some_substitutions_type import SomeSubstitutionsType -from launch.some_substitutions_type import SomeSubstitutionsType_types_tuple from launch.substitutions import LocalSubstitution from launch.utilities import ensure_argument_type from launch.utilities import normalize_to_list_of_substitutions @@ -38,6 +37,8 @@ from launch_ros.remap_rule_type import SomeRemapRules from launch_ros.substitutions import ExecutableInPackage +from launch_ros.utilities import evaluate_parameters +from launch_ros.utilities import normalize_parameters from launch_ros.utilities import normalize_remap_rules from rclpy.validate_namespace import validate_namespace @@ -136,11 +137,9 @@ def __init__( ensure_argument_type(parameters, (list), 'parameters', 'Node') # All elements in the list are paths to files with parameters (or substitutions that # evaluate to paths), or dictionaries of parameters (fields can be substitutions). - parameter_types = list(SomeSubstitutionsType_types_tuple) + [pathlib.Path, dict] i = 0 for param in parameters: - ensure_argument_type(param, parameter_types, 'parameters[{}]'.format(i), 'Node') - if isinstance(param, dict) and node_name is None: + if isinstance(param, Mapping) and node_name is None: raise RuntimeError( 'If a dictionary of parameters is specified, the node name must also be ' 'specified. See https://github.com/ros2/launch/issues/139') @@ -149,6 +148,7 @@ def __init__( 'ros_specific_arguments[{}]'.format(ros_args_index), description='parameter {}'.format(i))] ros_args_index += 1 + parameters = normalize_parameters(parameters) if remappings is not None: i = 0 for remapping in normalize_remap_rules(remappings): @@ -182,65 +182,11 @@ def node_name(self): raise RuntimeError("cannot access 'node_name' before executing action") return self.__final_node_name - def _create_params_file_from_dict(self, context, params): + def _create_params_file_from_dict(self, params): with NamedTemporaryFile(mode='w', prefix='launch_params_', delete=False) as h: param_file_path = h.name # TODO(dhood): clean up generated parameter files. - - def perform_substitution_if_applicable(context, var): - if isinstance(var, (int, float, str)): - # No substitution necessary. - return var - if isinstance(var, Substitution): - return perform_substitutions(context, normalize_to_list_of_substitutions(var)) - if isinstance(var, tuple): - try: - return perform_substitutions( - context, normalize_to_list_of_substitutions(var)) - except TypeError: - raise TypeError( - 'Invalid element received in parameters dictionary ' - '(not all tuple elements are Substitutions): {}'.format(var)) - else: - raise TypeError( - 'Unsupported type received in parameters dictionary: {}' - .format(type(var))) - - def expand_dict(input_dict): - expanded_dict = {} - for k, v in input_dict.items(): - # Key (parameter/group name) can only be a string/Substitutions that evaluates - # to a string. - expanded_key = perform_substitutions( - context, normalize_to_list_of_substitutions(k)) - if isinstance(v, dict): - # Expand the nested dict. - expanded_value = expand_dict(v) - elif isinstance(v, list): - # Expand each element. - expanded_value = [] - for e in v: - if isinstance(e, list): - raise TypeError( - 'Nested lists are not supported for parameters: {} found in {}' - .format(e, v)) - expanded_value.append(perform_substitution_if_applicable(context, e)) - # Tuples are treated as Substitution(s) to be concatenated. - elif isinstance(v, tuple): - for e in v: - ensure_argument_type( - e, SomeSubstitutionsType_types_tuple, - 'parameter dictionary tuple entry', 'Node') - expanded_value = perform_substitutions( - context, normalize_to_list_of_substitutions(v)) - else: - expanded_value = perform_substitution_if_applicable(context, v) - expanded_dict[expanded_key] = expanded_value - return expanded_dict - - expanded_dict = expand_dict(params) - param_dict = { - self.__expanded_node_name: {'ros__parameters': expanded_dict}} + param_dict = {self.__expanded_node_name: {'ros__parameters': params}} if self.__expanded_node_namespace: param_dict = {self.__expanded_node_namespace: param_dict} yaml.dump(param_dict, h, default_flow_style=False) @@ -281,15 +227,14 @@ def _perform_substitutions(self, context: LaunchContext) -> None: # expand parameters too if self.__parameters is not None: self.__expanded_parameter_files = [] - for params in self.__parameters: + evaluated_parameters = evaluate_parameters(context, self.__parameters) + for params in evaluated_parameters: if isinstance(params, dict): - param_file_path = self._create_params_file_from_dict(context, params) + param_file_path = self._create_params_file_from_dict(params) + elif isinstance(params, pathlib.Path): + param_file_path = str(params) else: - if isinstance(params, pathlib.Path): - param_file_path = str(params) - else: - param_file_path = perform_substitutions( - context, normalize_to_list_of_substitutions(params)) + raise RuntimeError('invalid normalized parameters {}'.format(repr(params))) if not os.path.isfile(param_file_path): _logger.warn( 'Parameter file path is not a file: {}'.format(param_file_path)) diff --git a/launch_ros/launch_ros/parameters_type.py b/launch_ros/launch_ros/parameters_type.py new file mode 100644 index 000000000..eedd95450 --- /dev/null +++ b/launch_ros/launch_ros/parameters_type.py @@ -0,0 +1,60 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for ROS parameter types.""" + +import pathlib + +from typing import Any +from typing import Dict +from typing import Iterable +from typing import Mapping +from typing import Union + +from launch.some_substitutions_type import SomeSubstitutionsType +from launch.substitution import Substitution + + +# Parameter value types used below +_SingleValueType = Union[str, int, float, bool] +_MultiValueType = Union[ + Iterable[str], Iterable[int], Iterable[float], Iterable[bool], bytes] + +SomeParameterFile = Union[SomeSubstitutionsType, pathlib.Path] +SomeParameterName = Iterable[Substitution] +SomeParameterValue = Union[SomeSubstitutionsType, _SingleValueType, _MultiValueType] + +# TODO(sloretz) Recursive type when mypy supports them python/mypy#731 +_SomeParametersDict = Mapping[SomeParameterName, Any] +SomeParametersDict = Mapping[SomeParameterName, Union[SomeParameterValue, _SomeParametersDict]] + +# Paths to yaml files with parameters, or dictionaries of parameters, or pairs of +# parameter names and values +SomeParameters = Iterable[Union[SomeParameterFile, Mapping[SomeParameterName, SomeParameterValue]]] + +ParameterFile = Iterable[Substitution] +ParameterValue = Union[Iterable[Substitution], + Iterable[Iterable[Substitution]], + _SingleValueType, + _MultiValueType] + +# Normalized (flattened to avoid having a recursive type) parameter dict +ParametersDict = Mapping[SomeParameterName, SomeParameterValue] + +# Normalized parameters +Parameters = Iterable[Union[ParameterFile, ParametersDict]] + +EvaluatedParameterValue = Union[_SingleValueType, _MultiValueType] +# Evaluated parameters: filenames or dictionary after substitutions have been evaluated +EvaluatedParameters = Iterable[Union[pathlib.Path, Dict[str, EvaluatedParameterValue]]] diff --git a/launch_ros/launch_ros/utilities/__init__.py b/launch_ros/launch_ros/utilities/__init__.py index 861874408..814b83cb7 100644 --- a/launch_ros/launch_ros/utilities/__init__.py +++ b/launch_ros/launch_ros/utilities/__init__.py @@ -18,11 +18,15 @@ Descriptions are not executable and are immutable so they can be reused by launch entities. """ +from .evaluate_parameters import evaluate_parameters +from .normalize_parameters import normalize_parameters from .normalize_remap_rule import normalize_remap_rule from .normalize_remap_rule import normalize_remap_rules __all__ = [ + 'evaluate_parameters', + 'normalize_parameters', 'normalize_remap_rule', 'normalize_remap_rules', ] diff --git a/launch_ros/launch_ros/utilities/evaluate_parameters.py b/launch_ros/launch_ros/utilities/evaluate_parameters.py new file mode 100644 index 000000000..694f01324 --- /dev/null +++ b/launch_ros/launch_ros/utilities/evaluate_parameters.py @@ -0,0 +1,80 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module with utility for evaluating parametesr in a launch context.""" + + +from collections.abc import Mapping +from collections.abc import Sequence +import pathlib +from typing import Dict +from typing import List +from typing import Union + +from launch.launch_context import LaunchContext +from launch.substitution import Substitution +from launch.utilities import ensure_argument_type +from launch.utilities import perform_substitutions + +from ..parameters_type import EvaluatedParameters +from ..parameters_type import EvaluatedParameterValue # noqa +from ..parameters_type import Parameters + + +def evaluate_parameters(context: LaunchContext, parameters: Parameters) -> EvaluatedParameters: + """ + Evaluate substitutions to produce paths and name/value pairs. + + The parameters must have been normalized with normalize_parameters() prior to calling this. + + :param parameters: normalized parameters + :returns: values after evaluating lists of substitutions + """ + output_params = [] # type: List[Union[pathlib.Path, Dict[str, EvaluatedParameterValue]]] + for param in parameters: + # If it's a list of substitutions then evaluate them to a string and return a pathlib.Path + if isinstance(param, tuple) and len(param) and isinstance(param[0], Substitution): + # Evaluate a list of Substitution to a file path + output_params.append(pathlib.Path(perform_substitutions(context, list(param)))) + elif isinstance(param, Mapping): + # It's a list of name/value pairs + output_dict = {} # type: Dict[str, EvaluatedParameterValue] + for name, value in param.items(): + if not isinstance(name, tuple): + raise TypeError('Expecting tuple of substitutions got {}'.format(repr(name))) + name = perform_substitutions(context, list(name)) + + if isinstance(value, tuple) and len(value): + if isinstance(value[0], Substitution): + # Value is a list of substitutions, so perform them to make a string + value = perform_substitutions(context, list(value)) + elif isinstance(value[0], Sequence): + # Value is an array of a list of substitutions + output_subvalue = [] # List[str] + for subvalue in value: + output_subvalue.append(perform_substitutions(context, list(subvalue))) + value = tuple(output_subvalue) + else: + # Value is an array of the same type, so nothing to evaluate. + output_value = [] + target_type = type(value[0]) + for i, subvalue in enumerate(value): + output_value.append(target_type(subvalue)) + value = tuple(output_value) + else: + # Value is a singular type, so nothing to evaluate + ensure_argument_type(value, (float, int, str, bool, bytes), 'value') + output_dict[name] = value + output_params.append(output_dict) + return tuple(output_params) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py new file mode 100644 index 000000000..2046bc34c --- /dev/null +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -0,0 +1,184 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module with utility for normalizing parameters to a node.""" + +from collections.abc import Iterable +from collections.abc import Mapping +from collections.abc import Sequence +import pathlib +from typing import cast +from typing import List # noqa +from typing import Tuple # noqa +from typing import Union # noqa + +from launch.some_substitutions_type import SomeSubstitutionsType_types_tuple +from launch.substitution import Substitution +from launch.substitutions import TextSubstitution +from launch.utilities import ensure_argument_type +from launch.utilities import normalize_to_list_of_substitutions + +from ..parameters_type import Parameters +from ..parameters_type import ParametersDict +from ..parameters_type import ParameterValue +from ..parameters_type import SomeParameters +from ..parameters_type import SomeParametersDict +from ..parameters_type import SomeParameterValue + + +def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValue: + """Normalize substitutions while preserving the type if it's not a substitution.""" + if isinstance(value, Sequence): + # a list of values of the same type + # Figure out what type the list should be + target_type = None + for subvalue in value: + allowed_subtypes = (float, int, str, bool) + SomeSubstitutionsType_types_tuple + ensure_argument_type(subvalue, allowed_subtypes, 'subvalue') + + if isinstance(subvalue, Substitution): + subtype = Substitution + else: + subtype = type(subvalue) + + if target_type is None: + target_type = subtype + + if subtype == float and target_type == int: + # If any value is a float, convert all integers to floats + target_type = float + elif subtype == int and target_type == float: + # If any value is a float, convert all integers to floats + pass + elif subtype == str and target_type == Substitution: + # If any value is a single substitution then result is a single string + target_type = Substitution + elif subtype == str and target_type == Substitution: + # If any value is a single substitution then result is a single string + target_type = Substitution + elif subtype != target_type: + # If types don't match, assume list of strings + target_type = str + + if target_type is None: + # No clue what an empty list's type should be + return [] + elif target_type == Substitution: + # Keep the list of substitutions together to form a single string + return tuple(normalize_to_list_of_substitutions(value)) + + output_value = [] # type: List[Union[Tuple[Substitution, ...], float, int, str, bool]] + for subvalue in value: + # Make all values in a list the same type + if target_type == float: + # cast to make mypy happy + output_value.append(float(cast(float, subvalue))) + elif target_type == int: + # cast to make mypy happy + output_value.append(int(cast(int, subvalue))) + elif target_type == bool: + # cast to make mypy happy + output_value.append(bool(cast(bool, subvalue))) + else: + if not isinstance(subvalue, Iterable) and not isinstance(subvalue, Substitution): + # Convert simple types to strings + subvalue = str(subvalue) + # Make everything a substitution + output_value.append(tuple(normalize_to_list_of_substitutions(subvalue))) + return tuple(output_value) + else: + raise TypeError('Value {} must be a sequence'.format(repr(value))) + + +def normalize_parameter_dict(parameters: SomeParametersDict, *, _prefix=None) -> ParametersDict: + """ + Normalize types used to store parameters in a dictionary. + + The parameters are passed as a dictionary that specifies parameter rules. + Keys of the dictionary can be strings, a Substitution, or an iterable of Substitution. + A normalized keys will be a tuple of substitutions. + Values in the dictionary can be strings, integers, floats, substututions, lists of + the previous types, or another dictionary with the same properties. + + Normalized values that were lists will have all subvalues converted to the same type. + If all subvalues are int or float, then the normalized subvalues will all be float. + If the subvalues otherwise do not all have the same type, then the normalized subvalues + will be lists of Substitution that will result in string parameters. + + Values that are a list of strings will become a list of strings when normalized and evaluated. + Values that are a list of :class:`Substitution` will become a single string. + To make a list of strings from substitutions, each item in the list must be a list or tuple. + + Normalized values that contained nested dictionaries will be collapsed into a single + layer with parameter names concatenated with the parameter namespace separator ".". + + :param parameters: Parameters to be normalized + :param _prefix: internal parameter used for flatening recursive dictionaries + :return: Normalized parameters + """ + if not isinstance(parameters, Mapping): + raise TypeError('expected dict') + + normalized = {} # ParametersDict + for name, value in parameters.items(): + # First make name a list of substitutions + name = normalize_to_list_of_substitutions(name) + if _prefix: + # Prefix name if there is a recursive dictionary + name = _prefix + [TextSubstitution(text='.')] + name + + # Normalize the value next + if isinstance(value, Mapping): + # Flatten recursive dictionaries + sub_dict = normalize_parameter_dict(value, _prefix=name) + normalized.update(sub_dict) + elif isinstance(value, str) or isinstance(value, Substitution): + normalized[tuple(name)] = tuple(normalize_to_list_of_substitutions(value)) + elif isinstance(value, float) or isinstance(value, bool) or isinstance(value, int): + # Keep some types as is + normalized[tuple(name)] = value + elif isinstance(value, bytes): + # Keep bytes as is + normalized[tuple(name)] = value + elif isinstance(value, Sequence): + # try to make the parameter types uniform + normalized[tuple(name)] = _normalize_parameter_array_value(value) + else: + raise TypeError('Unexpected type for parameter value {}'.format(repr(value))) + return normalized + + +def normalize_parameters(parameters: SomeParameters) -> Parameters: + """ + Normalize the types used to store parameters to substitution types. + + The passed parameters must be an iterable where each element is + a path to a yaml file or a dict. + The normalized parameters will have all paths converted to a list of :class:`Substitution`, + and dictionaries normalized using :meth:`normalize_parameter_dict`. + """ + if isinstance(parameters, str) or not isinstance(parameters, Sequence): + raise TypeError('Expecting list of parameters, got {}'.format(parameters)) + + normalized_params = [] # type: Parameters + for param in parameters: + if isinstance(param, Mapping): + normalized_params.append(normalize_parameter_dict(param)) + else: + # It's a path, normalize to a list of substitutions + if isinstance(param, pathlib.Path): + param = str(param) + ensure_argument_type(param, SomeSubstitutionsType_types_tuple, 'parameters') + normalized_params.append(tuple(normalize_to_list_of_substitutions(param))) + return tuple(normalized_params) diff --git a/test_launch_ros/test/test_launch_ros/actions/test_node.py b/test_launch_ros/test/test_launch_ros/actions/test_node.py index 25af9eecd..258d6d09d 100644 --- a/test_launch_ros/test/test_launch_ros/actions/test_node.py +++ b/test_launch_ros/test/test_launch_ros/actions/test_node.py @@ -167,12 +167,8 @@ def test_launch_node_with_parameter_dict(self): 'ros__parameters': { 'param1': 'param1_value', 'param2': 'param2_value', - 'param_group1': { - 'list_params': [1.2, 3.4], - 'param_group2': { - 'param2_values': ['param2_value'], - } - } + 'param_group1.list_params': (1.2, 3.4), + 'param_group1.param_group2.param2_values': ('param2_value',), } } } @@ -199,14 +195,15 @@ def test_launch_node_with_invalid_parameter_dicts(self): """Test launching a node with invalid parameter dicts.""" # Substitutions aren't expanded until the node action is executed, at which time a type # error should be raised and cause the launch to fail. + # However, the types are checked in Node.__init__() # For each type of invalid parameter, check that they are detected at both the top-level # and at a nested level in the dictionary. # Key must be a string/Substitution evaluating to a string. - self._assert_launch_errors(actions=[ + with self.assertRaises(TypeError): self._create_node(parameters=[{5: 'asdf'}]) - ]) - self._assert_launch_errors(actions=[ + + with self.assertRaises(TypeError): self._create_node(parameters=[{ 'param_group': { 'param_subgroup': { @@ -214,41 +211,25 @@ def test_launch_node_with_invalid_parameter_dicts(self): }, }, }]) - ]) # Nested lists are not supported. - self._assert_launch_errors(actions=[ + with self.assertRaises(TypeError): self._create_node(parameters=[{'param': [1, 2, [3, 4]]}]) - ]) - self._assert_launch_errors(actions=[ - self._create_node(parameters=[{ - 'param_group': { - 'param_subgroup': { - 'param': [1, 2, [3, 4]], - }, - }, - }]) - ]) - # Tuples are only supported for Substitutions. - self._assert_launch_errors(actions=[ - self._create_node(parameters=[{'param': (1, 2)}]) - ]) - self._assert_launch_errors(actions=[ + with self.assertRaises(TypeError): self._create_node(parameters=[{ 'param_group': { 'param_subgroup': { - 'param': (1, 2), + 'param': [1, 2, [3, 4]], }, }, }]) - ]) # Other types are not supported. - self._assert_launch_errors(actions=[ + with self.assertRaises(TypeError): self._create_node(parameters=[{'param': {1, 2}}]) - ]) - self._assert_launch_errors(actions=[ + + with self.assertRaises(TypeError): self._create_node(parameters=[{ 'param_group': { 'param_subgroup': { @@ -256,11 +237,11 @@ def test_launch_node_with_invalid_parameter_dicts(self): }, }, }]) - ]) - self._assert_launch_errors(actions=[ + + with self.assertRaises(TypeError): self._create_node(parameters=[{'param': self}]) - ]) - self._assert_launch_errors(actions=[ + + with self.assertRaises(TypeError): self._create_node(parameters=[{ 'param_group': { 'param_subgroup': { @@ -268,4 +249,3 @@ def test_launch_node_with_invalid_parameter_dicts(self): }, }, }]) - ]) diff --git a/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py new file mode 100644 index 000000000..513b2571d --- /dev/null +++ b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py @@ -0,0 +1,173 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the normalizing parameters utility.""" + +import pathlib + +from launch import LaunchContext +from launch.substitutions import TextSubstitution +from launch_ros.utilities import evaluate_parameters +from launch_ros.utilities import normalize_parameters +import pytest + + +def test_not_a_list(): + with pytest.raises(TypeError): + normalize_parameters({'foo': 'bar'}) + with pytest.raises(TypeError): + normalize_parameters('foobar') + + +def test_single_str_path(): + norm = normalize_parameters(['/foo/bar']) + expected = (pathlib.Path('/foo/bar'),) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_single_pathlib_path(): + norm = normalize_parameters([pathlib.Path('/foo/bar')]) + expected = (pathlib.Path('/foo/bar'),) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_multiple_paths(): + norm = normalize_parameters([pathlib.Path('/foo/bar'), '/bar/baz']) + expected = (pathlib.Path('/foo/bar'), pathlib.Path('/bar/baz')) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_path_with_substitutions(): + orig = [(TextSubstitution(text='/foo'), TextSubstitution(text='/bar'))] + norm = normalize_parameters(orig) + expected = (pathlib.Path('/foo/bar'),) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_single_dictionary(): + orig = [{'foo': 1, 'bar': 2.0}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'bar': 2.0},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_multiple_dictionaries(): + orig = [{'foo': 1, 'bar': 2.0}, {'baz': 'asdf'}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'bar': 2.0}, {'baz': 'asdf'}) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_substitution(): + orig = [{TextSubstitution(text='bar'): TextSubstitution(text='baz')}] + norm = normalize_parameters(orig) + expected = ({'bar': 'baz'},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_substitution_list_name(): + orig = [{(TextSubstitution(text='bar'), TextSubstitution(text='foo')): 1}] + norm = normalize_parameters(orig) + expected = ({'barfoo': 1},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_substitution_list_value(): + orig = [{'foo': [TextSubstitution(text='fiz'), TextSubstitution(text='buz')]}] + norm = normalize_parameters(orig) + expected = ({'foo': 'fizbuz'},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + orig = [{'foo': [[TextSubstitution(text='fiz')], [TextSubstitution(text='buz')]]}] + norm = normalize_parameters(orig) + expected = ({'foo': ('fiz', 'buz')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_mixed_substitutions_and_strings(): + orig = [{'foo': [TextSubstitution(text='fiz'), 'bar']}] + norm = normalize_parameters(orig) + expected = ({'foo': 'fizbar'},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + orig = [{'foo': [[TextSubstitution(text='fiz')], 'bar']}] + norm = normalize_parameters(orig) + expected = ({'foo': ('fiz', 'bar')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_str(): + orig = [{'foo': 'bar', 'fiz': ['b', 'u', 'z']}] + norm = normalize_parameters(orig) + expected = ({'foo': 'bar', 'fiz': ('b', 'u', 'z')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_bool(): + orig = [{'foo': False, 'fiz': [True, False, True]}] + norm = normalize_parameters(orig) + expected = ({'foo': False, 'fiz': (True, False, True)},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_float(): + orig = [{'foo': 1.2, 'fiz': [2.3, 3.4, 4.5]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1.2, 'fiz': (2.3, 3.4, 4.5)},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_int(): + orig = [{'foo': 1, 'fiz': [2, 3, 4]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': (2, 3, 4)},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_int_and_float(): + orig = [{'foo': 1, 'fiz': [2, 3.1, 4]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': (2.0, 3.1, 4.0)},) + evaluated = evaluate_parameters(LaunchContext(), norm) + assert evaluated == expected + # pytest doesn't check int vs float type + assert tuple(map(type, evaluated[0]['fiz'])) == (float, float, float) + + +def test_dictionary_with_bytes(): + orig = [{'foo': 1, 'fiz': bytes([0xff, 0x5c, 0xaa])}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': bytes([0xff, 0x5c, 0xaa])},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_dictionary_with_dissimilar_array(): + orig = [{'foo': 1, 'fiz': [True, 2.0, 3]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': ('True', '2.0', '3')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_nested_dictionaries(): + orig = [{'foo': {'bar': 'baz'}, 'fiz': {'buz': 3}}] + norm = normalize_parameters(orig) + expected = ({'foo.bar': 'baz', 'fiz.buz': 3},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + +def test_mixed_path_dicts(): + orig = ['/foo/bar', {'fiz': {'buz': 3}}, pathlib.Path('/tmp/baz')] + norm = normalize_parameters(orig) + expected = (pathlib.Path('/foo/bar'), {'fiz.buz': 3}, pathlib.Path('/tmp/baz')) + assert evaluate_parameters(LaunchContext(), norm) == expected From 1d493eb774ba00e18f8eec6b95076d641915be16 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 1 Mar 2019 12:02:37 -0800 Subject: [PATCH 02/13] Misc mypy complaint fixes Signed-off-by: Shane Loretz --- launch_ros/launch_ros/actions/node.py | 3 +- launch_ros/launch_ros/parameters_type.py | 18 ++++----- .../utilities/evaluate_parameters.py | 13 ++++--- .../utilities/normalize_parameters.py | 39 +++++++++---------- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/launch_ros/launch_ros/actions/node.py b/launch_ros/launch_ros/actions/node.py index 4b4139599..43f9e3c3c 100644 --- a/launch_ros/launch_ros/actions/node.py +++ b/launch_ros/launch_ros/actions/node.py @@ -36,6 +36,7 @@ from launch.utilities import perform_substitutions from launch_ros.remap_rule_type import SomeRemapRules +from launch_ros.parameters_type import SomeParameters from launch_ros.substitutions import ExecutableInPackage from launch_ros.utilities import evaluate_parameters from launch_ros.utilities import normalize_parameters @@ -58,7 +59,7 @@ def __init__( node_executable: SomeSubstitutionsType, node_name: Optional[SomeSubstitutionsType] = None, node_namespace: Optional[SomeSubstitutionsType] = None, - parameters: Optional[List[SomeSubstitutionsType]] = None, + parameters: Optional[SomeParameters] = None, remappings: Optional[SomeRemapRules] = None, arguments: Optional[Iterable[SomeSubstitutionsType]] = None, **kwargs diff --git a/launch_ros/launch_ros/parameters_type.py b/launch_ros/launch_ros/parameters_type.py index eedd95450..4a840bcc6 100644 --- a/launch_ros/launch_ros/parameters_type.py +++ b/launch_ros/launch_ros/parameters_type.py @@ -18,8 +18,8 @@ from typing import Any from typing import Dict -from typing import Iterable from typing import Mapping +from typing import Sequence from typing import Union from launch.some_substitutions_type import SomeSubstitutionsType @@ -29,10 +29,10 @@ # Parameter value types used below _SingleValueType = Union[str, int, float, bool] _MultiValueType = Union[ - Iterable[str], Iterable[int], Iterable[float], Iterable[bool], bytes] + Sequence[str], Sequence[int], Sequence[float], Sequence[bool], bytes] SomeParameterFile = Union[SomeSubstitutionsType, pathlib.Path] -SomeParameterName = Iterable[Substitution] +SomeParameterName = Sequence[Substitution] SomeParameterValue = Union[SomeSubstitutionsType, _SingleValueType, _MultiValueType] # TODO(sloretz) Recursive type when mypy supports them python/mypy#731 @@ -41,11 +41,11 @@ # Paths to yaml files with parameters, or dictionaries of parameters, or pairs of # parameter names and values -SomeParameters = Iterable[Union[SomeParameterFile, Mapping[SomeParameterName, SomeParameterValue]]] +SomeParameters = Sequence[Union[SomeParameterFile, Mapping[SomeParameterName, SomeParameterValue]]] -ParameterFile = Iterable[Substitution] -ParameterValue = Union[Iterable[Substitution], - Iterable[Iterable[Substitution]], +ParameterFile = Sequence[Substitution] +ParameterValue = Union[Sequence[Substitution], + Sequence[Sequence[Substitution]], _SingleValueType, _MultiValueType] @@ -53,8 +53,8 @@ ParametersDict = Mapping[SomeParameterName, SomeParameterValue] # Normalized parameters -Parameters = Iterable[Union[ParameterFile, ParametersDict]] +Parameters = Sequence[Union[ParameterFile, ParametersDict]] EvaluatedParameterValue = Union[_SingleValueType, _MultiValueType] # Evaluated parameters: filenames or dictionary after substitutions have been evaluated -EvaluatedParameters = Iterable[Union[pathlib.Path, Dict[str, EvaluatedParameterValue]]] +EvaluatedParameters = Sequence[Union[pathlib.Path, Dict[str, EvaluatedParameterValue]]] diff --git a/launch_ros/launch_ros/utilities/evaluate_parameters.py b/launch_ros/launch_ros/utilities/evaluate_parameters.py index 694f01324..c9a4c4fa7 100644 --- a/launch_ros/launch_ros/utilities/evaluate_parameters.py +++ b/launch_ros/launch_ros/utilities/evaluate_parameters.py @@ -18,6 +18,7 @@ from collections.abc import Mapping from collections.abc import Sequence import pathlib +from typing import cast from typing import Dict from typing import List from typing import Union @@ -53,28 +54,30 @@ def evaluate_parameters(context: LaunchContext, parameters: Parameters) -> Evalu for name, value in param.items(): if not isinstance(name, tuple): raise TypeError('Expecting tuple of substitutions got {}'.format(repr(name))) - name = perform_substitutions(context, list(name)) + evaluated_name = perform_substitutions(context, list(name)) # type: str + evaluated_value = '' # type: EvaluatedParameterValue if isinstance(value, tuple) and len(value): if isinstance(value[0], Substitution): # Value is a list of substitutions, so perform them to make a string - value = perform_substitutions(context, list(value)) + evaluated_value = perform_substitutions(context, list(value)) elif isinstance(value[0], Sequence): # Value is an array of a list of substitutions output_subvalue = [] # List[str] for subvalue in value: output_subvalue.append(perform_substitutions(context, list(subvalue))) - value = tuple(output_subvalue) + evalutated_value = tuple(output_subvalue) else: # Value is an array of the same type, so nothing to evaluate. output_value = [] target_type = type(value[0]) for i, subvalue in enumerate(value): output_value.append(target_type(subvalue)) - value = tuple(output_value) + evaluated_value = tuple(output_value) else: # Value is a singular type, so nothing to evaluate ensure_argument_type(value, (float, int, str, bool, bytes), 'value') - output_dict[name] = value + evaluated_value = cast(Union[float, int, str, bool, bytes], value) + output_dict[evaluated_name] = evaluated_value output_params.append(output_dict) return tuple(output_params) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index 2046bc34c..1e3e3751a 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -20,15 +20,18 @@ import pathlib from typing import cast from typing import List # noqa -from typing import Tuple # noqa +from typing import Optional # noqa +from typing import Tuple # noqa from typing import Union # noqa +from launch.some_substitutions_type import SomeSubstitutionsType from launch.some_substitutions_type import SomeSubstitutionsType_types_tuple from launch.substitution import Substitution from launch.substitutions import TextSubstitution from launch.utilities import ensure_argument_type from launch.utilities import normalize_to_list_of_substitutions +from ..parameters_type import ParameterFile from ..parameters_type import Parameters from ..parameters_type import ParametersDict from ..parameters_type import ParameterValue @@ -42,13 +45,13 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValu if isinstance(value, Sequence): # a list of values of the same type # Figure out what type the list should be - target_type = None + target_type = None # type: Optional[type] for subvalue in value: allowed_subtypes = (float, int, str, bool) + SomeSubstitutionsType_types_tuple ensure_argument_type(subvalue, allowed_subtypes, 'subvalue') if isinstance(subvalue, Substitution): - subtype = Substitution + subtype = Substitution # type: type else: subtype = type(subvalue) @@ -76,27 +79,23 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValu return [] elif target_type == Substitution: # Keep the list of substitutions together to form a single string - return tuple(normalize_to_list_of_substitutions(value)) - - output_value = [] # type: List[Union[Tuple[Substitution, ...], float, int, str, bool]] - for subvalue in value: - # Make all values in a list the same type - if target_type == float: - # cast to make mypy happy - output_value.append(float(cast(float, subvalue))) - elif target_type == int: - # cast to make mypy happy - output_value.append(int(cast(int, subvalue))) - elif target_type == bool: - # cast to make mypy happy - output_value.append(bool(cast(bool, subvalue))) - else: + return tuple(normalize_to_list_of_substitutions(cast(SomeSubstitutionsType, value))) + + if target_type == float: + return tuple([float(s) for s in value]) + elif target_type == int: + return tuple([int(s) for s in value]) + elif target_type == bool: + return tuple([bool(s) for s in value]) + else: + output_value = [] # type: List[Tuple[Substitution, ...]] + for subvalue in value: if not isinstance(subvalue, Iterable) and not isinstance(subvalue, Substitution): # Convert simple types to strings subvalue = str(subvalue) # Make everything a substitution output_value.append(tuple(normalize_to_list_of_substitutions(subvalue))) - return tuple(output_value) + return tuple(output_value) else: raise TypeError('Value {} must be a sequence'.format(repr(value))) @@ -171,7 +170,7 @@ def normalize_parameters(parameters: SomeParameters) -> Parameters: if isinstance(parameters, str) or not isinstance(parameters, Sequence): raise TypeError('Expecting list of parameters, got {}'.format(parameters)) - normalized_params = [] # type: Parameters + normalized_params = [] # type: List[Union[ParameterFile, ParametersDict]] for param in parameters: if isinstance(param, Mapping): normalized_params.append(normalize_parameter_dict(param)) From a304468e42136807c062fb2ed9b1e0976ca14edd Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 1 Mar 2019 13:39:48 -0800 Subject: [PATCH 03/13] More mypy appeasement Signed-off-by: Shane Loretz --- launch_ros/launch_ros/actions/node.py | 4 ++-- launch_ros/launch_ros/parameters_type.py | 5 +++-- .../utilities/normalize_parameters.py | 17 +++++++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/launch_ros/launch_ros/actions/node.py b/launch_ros/launch_ros/actions/node.py index 43f9e3c3c..ce9a34cd6 100644 --- a/launch_ros/launch_ros/actions/node.py +++ b/launch_ros/launch_ros/actions/node.py @@ -149,7 +149,7 @@ def __init__( 'ros_specific_arguments[{}]'.format(ros_args_index), description='parameter {}'.format(i))] ros_args_index += 1 - parameters = normalize_parameters(parameters) + normalized_params = normalize_parameters(parameters) if remappings is not None: i = 0 for remapping in normalize_remap_rules(remappings): @@ -164,7 +164,7 @@ def __init__( self.__node_executable = node_executable self.__node_name = node_name self.__node_namespace = node_namespace - self.__parameters = [] if parameters is None else parameters + self.__parameters = [] if parameters is None else normalized_params self.__remappings = [] if remappings is None else remappings self.__arguments = arguments diff --git a/launch_ros/launch_ros/parameters_type.py b/launch_ros/launch_ros/parameters_type.py index 4a840bcc6..3cd4001b2 100644 --- a/launch_ros/launch_ros/parameters_type.py +++ b/launch_ros/launch_ros/parameters_type.py @@ -32,7 +32,7 @@ Sequence[str], Sequence[int], Sequence[float], Sequence[bool], bytes] SomeParameterFile = Union[SomeSubstitutionsType, pathlib.Path] -SomeParameterName = Sequence[Substitution] +SomeParameterName = Sequence[Union[Substitution, str]] SomeParameterValue = Union[SomeSubstitutionsType, _SingleValueType, _MultiValueType] # TODO(sloretz) Recursive type when mypy supports them python/mypy#731 @@ -44,13 +44,14 @@ SomeParameters = Sequence[Union[SomeParameterFile, Mapping[SomeParameterName, SomeParameterValue]]] ParameterFile = Sequence[Substitution] +ParameterName = Sequence[Substitution] ParameterValue = Union[Sequence[Substitution], Sequence[Sequence[Substitution]], _SingleValueType, _MultiValueType] # Normalized (flattened to avoid having a recursive type) parameter dict -ParametersDict = Mapping[SomeParameterName, SomeParameterValue] +ParametersDict = Dict[ParameterName, ParameterValue] # Normalized parameters Parameters = Sequence[Union[ParameterFile, ParametersDict]] diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index 1e3e3751a..08d9d5c33 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -20,7 +20,8 @@ import pathlib from typing import cast from typing import List # noqa -from typing import Optional # noqa +from typing import Optional +from typing import Sequence as SequenceTypeHint from typing import Tuple # noqa from typing import Union # noqa @@ -32,6 +33,7 @@ from launch.utilities import normalize_to_list_of_substitutions from ..parameters_type import ParameterFile +from ..parameters_type import ParameterName from ..parameters_type import Parameters from ..parameters_type import ParametersDict from ..parameters_type import ParameterValue @@ -100,7 +102,10 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValu raise TypeError('Value {} must be a sequence'.format(repr(value))) -def normalize_parameter_dict(parameters: SomeParametersDict, *, _prefix=None) -> ParametersDict: +def normalize_parameter_dict( + parameters: SomeParametersDict, *, + _prefix: Optional[SequenceTypeHint[Substitution]] = None +) -> ParametersDict: """ Normalize types used to store parameters in a dictionary. @@ -129,13 +134,17 @@ def normalize_parameter_dict(parameters: SomeParametersDict, *, _prefix=None) -> if not isinstance(parameters, Mapping): raise TypeError('expected dict') - normalized = {} # ParametersDict + normalized = {} # type: ParametersDict for name, value in parameters.items(): # First make name a list of substitutions name = normalize_to_list_of_substitutions(name) if _prefix: # Prefix name if there is a recursive dictionary - name = _prefix + [TextSubstitution(text='.')] + name + # weird looking logic to combine into one list to appease mypy + combined = [e for e in _prefix] + combined.append(TextSubstitution(text='.')) + combined.extend(name) + name = combined # Normalize the value next if isinstance(value, Mapping): From b0f8c777dc8a7899ee36089f445d1fd7b19ff76e Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 1 Mar 2019 13:58:25 -0800 Subject: [PATCH 04/13] fix bugs introduced whil appeasing mypy Signed-off-by: Shane Loretz --- launch_ros/launch_ros/actions/node.py | 2 +- launch_ros/launch_ros/utilities/evaluate_parameters.py | 6 ++++-- launch_ros/launch_ros/utilities/normalize_parameters.py | 7 +++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/launch_ros/launch_ros/actions/node.py b/launch_ros/launch_ros/actions/node.py index ce9a34cd6..8339fadd9 100644 --- a/launch_ros/launch_ros/actions/node.py +++ b/launch_ros/launch_ros/actions/node.py @@ -35,8 +35,8 @@ from launch.utilities import normalize_to_list_of_substitutions from launch.utilities import perform_substitutions -from launch_ros.remap_rule_type import SomeRemapRules from launch_ros.parameters_type import SomeParameters +from launch_ros.remap_rule_type import SomeRemapRules from launch_ros.substitutions import ExecutableInPackage from launch_ros.utilities import evaluate_parameters from launch_ros.utilities import normalize_parameters diff --git a/launch_ros/launch_ros/utilities/evaluate_parameters.py b/launch_ros/launch_ros/utilities/evaluate_parameters.py index c9a4c4fa7..d7dc1138a 100644 --- a/launch_ros/launch_ros/utilities/evaluate_parameters.py +++ b/launch_ros/launch_ros/utilities/evaluate_parameters.py @@ -55,7 +55,7 @@ def evaluate_parameters(context: LaunchContext, parameters: Parameters) -> Evalu if not isinstance(name, tuple): raise TypeError('Expecting tuple of substitutions got {}'.format(repr(name))) evaluated_name = perform_substitutions(context, list(name)) # type: str - evaluated_value = '' # type: EvaluatedParameterValue + evaluated_value = None # type: EvaluatedParameterValue if isinstance(value, tuple) and len(value): if isinstance(value[0], Substitution): @@ -66,7 +66,7 @@ def evaluate_parameters(context: LaunchContext, parameters: Parameters) -> Evalu output_subvalue = [] # List[str] for subvalue in value: output_subvalue.append(perform_substitutions(context, list(subvalue))) - evalutated_value = tuple(output_subvalue) + evaluated_value = tuple(output_subvalue) else: # Value is an array of the same type, so nothing to evaluate. output_value = [] @@ -78,6 +78,8 @@ def evaluate_parameters(context: LaunchContext, parameters: Parameters) -> Evalu # Value is a singular type, so nothing to evaluate ensure_argument_type(value, (float, int, str, bool, bytes), 'value') evaluated_value = cast(Union[float, int, str, bool, bytes], value) + if evaluated_value is None: + raise TypeError('given unnormalized parameters %r, %r' % (name, value)) output_dict[evaluated_name] = evaluated_value output_params.append(output_dict) return tuple(output_params) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index 08d9d5c33..e7946a292 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -33,7 +33,6 @@ from launch.utilities import normalize_to_list_of_substitutions from ..parameters_type import ParameterFile -from ..parameters_type import ParameterName from ..parameters_type import Parameters from ..parameters_type import ParametersDict from ..parameters_type import ParameterValue @@ -84,11 +83,11 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValu return tuple(normalize_to_list_of_substitutions(cast(SomeSubstitutionsType, value))) if target_type == float: - return tuple([float(s) for s in value]) + return tuple(float(s) for s in value) elif target_type == int: - return tuple([int(s) for s in value]) + return tuple(int(s) for s in value) elif target_type == bool: - return tuple([bool(s) for s in value]) + return tuple(bool(s) for s in value) else: output_value = [] # type: List[Tuple[Substitution, ...]] for subvalue in value: From a938a19df4c6715348e67987b7cb3cb3305c22a1 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Fri, 1 Mar 2019 14:01:00 -0800 Subject: [PATCH 05/13] Another mypy complaint fixed Signed-off-by: Shane Loretz --- launch_ros/launch_ros/utilities/evaluate_parameters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launch_ros/launch_ros/utilities/evaluate_parameters.py b/launch_ros/launch_ros/utilities/evaluate_parameters.py index d7dc1138a..8ed7178e0 100644 --- a/launch_ros/launch_ros/utilities/evaluate_parameters.py +++ b/launch_ros/launch_ros/utilities/evaluate_parameters.py @@ -21,6 +21,7 @@ from typing import cast from typing import Dict from typing import List +from typing import Optional # noqa from typing import Union from launch.launch_context import LaunchContext @@ -55,7 +56,7 @@ def evaluate_parameters(context: LaunchContext, parameters: Parameters) -> Evalu if not isinstance(name, tuple): raise TypeError('Expecting tuple of substitutions got {}'.format(repr(name))) evaluated_name = perform_substitutions(context, list(name)) # type: str - evaluated_value = None # type: EvaluatedParameterValue + evaluated_value = None # type: Optional[EvaluatedParameterValue] if isinstance(value, tuple) and len(value): if isinstance(value[0], Substitution): From d66bcb33570ca70161884d1ca776dbd01913fc97 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 12:34:49 -0800 Subject: [PATCH 06/13] fix checking str and substitution Signed-off-by: Shane Loretz --- launch_ros/launch_ros/utilities/normalize_parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index e7946a292..803fdd012 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -68,7 +68,7 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValu elif subtype == str and target_type == Substitution: # If any value is a single substitution then result is a single string target_type = Substitution - elif subtype == str and target_type == Substitution: + elif subtype == Substitution and target_type == str: # If any value is a single substitution then result is a single string target_type = Substitution elif subtype != target_type: From 1e5d3dcf0ebcdd5bdd714d7ebb17864bd0494d95 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 12:59:34 -0800 Subject: [PATCH 07/13] Avoid indentation Signed-off-by: Shane Loretz --- .../utilities/normalize_parameters.py | 105 +++++++++--------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index 803fdd012..2b23985f6 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -43,62 +43,61 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValue: """Normalize substitutions while preserving the type if it's not a substitution.""" - if isinstance(value, Sequence): - # a list of values of the same type - # Figure out what type the list should be - target_type = None # type: Optional[type] - for subvalue in value: - allowed_subtypes = (float, int, str, bool) + SomeSubstitutionsType_types_tuple - ensure_argument_type(subvalue, allowed_subtypes, 'subvalue') - - if isinstance(subvalue, Substitution): - subtype = Substitution # type: type - else: - subtype = type(subvalue) - - if target_type is None: - target_type = subtype - - if subtype == float and target_type == int: - # If any value is a float, convert all integers to floats - target_type = float - elif subtype == int and target_type == float: - # If any value is a float, convert all integers to floats - pass - elif subtype == str and target_type == Substitution: - # If any value is a single substitution then result is a single string - target_type = Substitution - elif subtype == Substitution and target_type == str: - # If any value is a single substitution then result is a single string - target_type = Substitution - elif subtype != target_type: - # If types don't match, assume list of strings - target_type = str + if not isinstance(value, Sequence): + raise TypeError('Value {} must be a sequence'.format(repr(value))) - if target_type is None: - # No clue what an empty list's type should be - return [] - elif target_type == Substitution: - # Keep the list of substitutions together to form a single string - return tuple(normalize_to_list_of_substitutions(cast(SomeSubstitutionsType, value))) - - if target_type == float: - return tuple(float(s) for s in value) - elif target_type == int: - return tuple(int(s) for s in value) - elif target_type == bool: - return tuple(bool(s) for s in value) + # Figure out what type the list should be + target_type = None # type: Optional[type] + for subvalue in value: + allowed_subtypes = (float, int, str, bool) + SomeSubstitutionsType_types_tuple + ensure_argument_type(subvalue, allowed_subtypes, 'subvalue') + + if isinstance(subvalue, Substitution): + subtype = Substitution # type: type else: - output_value = [] # type: List[Tuple[Substitution, ...]] - for subvalue in value: - if not isinstance(subvalue, Iterable) and not isinstance(subvalue, Substitution): - # Convert simple types to strings - subvalue = str(subvalue) - # Make everything a substitution - output_value.append(tuple(normalize_to_list_of_substitutions(subvalue))) - return tuple(output_value) + subtype = type(subvalue) + + if target_type is None: + target_type = subtype + + if subtype == float and target_type == int: + # If any value is a float, convert all integers to floats + target_type = float + elif subtype == int and target_type == float: + # If any value is a float, convert all integers to floats + pass + elif subtype == str and target_type == Substitution: + # If any value is a single substitution then result is a single string + target_type = Substitution + elif subtype == Substitution and target_type == str: + # If any value is a single substitution then result is a single string + target_type = Substitution + elif subtype != target_type: + # If types don't match, assume list of strings + target_type = str + + if target_type is None: + # No clue what an empty list's type should be + return [] + elif target_type == Substitution: + # Keep the list of substitutions together to form a single string + return tuple(normalize_to_list_of_substitutions(cast(SomeSubstitutionsType, value))) + + if target_type == float: + return tuple(float(s) for s in value) + elif target_type == int: + return tuple(int(s) for s in value) + elif target_type == bool: + return tuple(bool(s) for s in value) else: - raise TypeError('Value {} must be a sequence'.format(repr(value))) + output_value = [] # type: List[Tuple[Substitution, ...]] + for subvalue in value: + if not isinstance(subvalue, Iterable) and not isinstance(subvalue, Substitution): + # Convert simple types to strings + subvalue = str(subvalue) + # Make everything a substitution + output_value.append(tuple(normalize_to_list_of_substitutions(subvalue))) + return tuple(output_value) def normalize_parameter_dict( From 21be882ed654f0f0e9053414b87d6bbbf5746cfa Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 13:00:50 -0800 Subject: [PATCH 08/13] keys -> key Signed-off-by: Shane Loretz --- launch_ros/launch_ros/utilities/normalize_parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index 2b23985f6..63fccd62a 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -109,7 +109,7 @@ def normalize_parameter_dict( The parameters are passed as a dictionary that specifies parameter rules. Keys of the dictionary can be strings, a Substitution, or an iterable of Substitution. - A normalized keys will be a tuple of substitutions. + A normalized key will be a tuple of substitutions. Values in the dictionary can be strings, integers, floats, substututions, lists of the previous types, or another dictionary with the same properties. From d835512ae133f5dbdbe0c279ffef4a2e39459634 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 13:02:07 -0800 Subject: [PATCH 09/13] substututions -> substitutions Signed-off-by: Shane Loretz --- launch_ros/launch_ros/utilities/normalize_parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index 63fccd62a..fe42817ef 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -110,7 +110,7 @@ def normalize_parameter_dict( The parameters are passed as a dictionary that specifies parameter rules. Keys of the dictionary can be strings, a Substitution, or an iterable of Substitution. A normalized key will be a tuple of substitutions. - Values in the dictionary can be strings, integers, floats, substututions, lists of + Values in the dictionary can be strings, integers, floats, substitutions, lists of the previous types, or another dictionary with the same properties. Normalized values that were lists will have all subvalues converted to the same type. From b1f22861830e10212d8176e9a069226d78077400 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 13:07:39 -0800 Subject: [PATCH 10/13] Add test with float as first element in numeric list Signed-off-by: Shane Loretz --- .../test/test_launch_ros/test_normalize_parameters.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py index 513b2571d..fa2badb41 100644 --- a/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py +++ b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py @@ -144,6 +144,14 @@ def test_dictionary_with_int_and_float(): # pytest doesn't check int vs float type assert tuple(map(type, evaluated[0]['fiz'])) == (float, float, float) + orig = [{'foo': 1, 'fiz': [2.0, 3, 4]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': (2.0, 3.0, 4.0)},) + evaluated = evaluate_parameters(LaunchContext(), norm) + assert evaluated == expected + # pytest doesn't check int vs float type + assert tuple(map(type, evaluated[0]['fiz'])) == (float, float, float) + def test_dictionary_with_bytes(): orig = [{'foo': 1, 'fiz': bytes([0xff, 0x5c, 0xaa])}] From 46b149528115662912592690150d5af577226867 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 13:11:02 -0800 Subject: [PATCH 11/13] Add test with string before substitution Signed-off-by: Shane Loretz --- .../test/test_launch_ros/test_normalize_parameters.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py index fa2badb41..f36075458 100644 --- a/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py +++ b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py @@ -106,6 +106,16 @@ def test_dictionary_with_mixed_substitutions_and_strings(): expected = ({'foo': ('fiz', 'bar')},) assert evaluate_parameters(LaunchContext(), norm) == expected + orig = [{'foo': ['bar', TextSubstitution(text='fiz')]}] + norm = normalize_parameters(orig) + expected = ({'foo': 'barfiz'},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + orig = [{'foo': ['bar', [TextSubstitution(text='fiz')]]}] + norm = normalize_parameters(orig) + expected = ({'foo': ('bar', 'fiz')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + def test_dictionary_with_str(): orig = [{'foo': 'bar', 'fiz': ['b', 'u', 'z']}] From a4ac02b6c423f3dccd3d90be97d805b9e97239db Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 15:55:33 -0800 Subject: [PATCH 12/13] Add more dissimilar tests and fix bugs Signed-off-by: Shane Loretz --- .../utilities/normalize_parameters.py | 85 +++++++++---------- .../test_normalize_parameters.py | 20 +++++ 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index fe42817ef..55c4077af 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -14,7 +14,6 @@ """Module with utility for normalizing parameters to a node.""" -from collections.abc import Iterable from collections.abc import Mapping from collections.abc import Sequence import pathlib @@ -24,6 +23,7 @@ from typing import Sequence as SequenceTypeHint from typing import Tuple # noqa from typing import Union # noqa +from typing import Set # noqa from launch.some_substitutions_type import SomeSubstitutionsType from launch.some_substitutions_type import SomeSubstitutionsType_types_tuple @@ -47,57 +47,48 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValu raise TypeError('Value {} must be a sequence'.format(repr(value))) # Figure out what type the list should be - target_type = None # type: Optional[type] + has_types = set() # type: Set[type] for subvalue in value: allowed_subtypes = (float, int, str, bool) + SomeSubstitutionsType_types_tuple ensure_argument_type(subvalue, allowed_subtypes, 'subvalue') if isinstance(subvalue, Substitution): - subtype = Substitution # type: type + has_types.add(Substitution) + elif isinstance(subvalue, str): + has_types.add(str) + elif isinstance(subvalue, bool): + has_types.add(bool) + elif isinstance(subvalue, int): + has_types.add(int) + elif isinstance(subvalue, float): + has_types.add(float) + elif isinstance(subvalue, Sequence): + has_types.add(tuple) else: - subtype = type(subvalue) - - if target_type is None: - target_type = subtype - - if subtype == float and target_type == int: - # If any value is a float, convert all integers to floats - target_type = float - elif subtype == int and target_type == float: - # If any value is a float, convert all integers to floats - pass - elif subtype == str and target_type == Substitution: - # If any value is a single substitution then result is a single string - target_type = Substitution - elif subtype == Substitution and target_type == str: - # If any value is a single substitution then result is a single string - target_type = Substitution - elif subtype != target_type: - # If types don't match, assume list of strings - target_type = str - - if target_type is None: - # No clue what an empty list's type should be - return [] - elif target_type == Substitution: - # Keep the list of substitutions together to form a single string + raise RuntimeError('Failed to handle type {}'.format(repr(subvalue))) + + if {int} == has_types: + # all integers + return tuple(int(e) for e in value) + elif has_types in ({float}, {int, float}): + # all were floats or ints, so return floats + return tuple(float(e) for e in value) + elif Substitution in has_types and has_types.issubset({str, Substitution, tuple}): + # make a list of substitutions forming a single string return tuple(normalize_to_list_of_substitutions(cast(SomeSubstitutionsType, value))) - - if target_type == float: - return tuple(float(s) for s in value) - elif target_type == int: - return tuple(int(s) for s in value) - elif target_type == bool: - return tuple(bool(s) for s in value) + elif {bool} == has_types: + # all where bools + return tuple(bool(e) for e in value) else: - output_value = [] # type: List[Tuple[Substitution, ...]] - for subvalue in value: - if not isinstance(subvalue, Iterable) and not isinstance(subvalue, Substitution): - # Convert simple types to strings - subvalue = str(subvalue) - # Make everything a substitution - output_value.append(tuple(normalize_to_list_of_substitutions(subvalue))) - return tuple(output_value) + # Should evaluate to a list of strings + # Normalize to a list of lists of substitutions + new_value = [] # type: List[SomeSubstitutionsType] + for element in value: + if isinstance(element, float) or isinstance(element, int) or isinstance(element, bool): + new_value.append(str(element)) + else: + new_value.append(element) + return tuple(normalize_to_list_of_substitutions(e) for e in new_value) def normalize_parameter_dict( @@ -116,10 +107,12 @@ def normalize_parameter_dict( Normalized values that were lists will have all subvalues converted to the same type. If all subvalues are int or float, then the normalized subvalues will all be float. If the subvalues otherwise do not all have the same type, then the normalized subvalues - will be lists of Substitution that will result in string parameters. + will be lists of Substitution that will result in a list of strings. Values that are a list of strings will become a list of strings when normalized and evaluated. - Values that are a list of :class:`Substitution` will become a single string. + Values that are a list which has at least one :class:`Substitution` and all other elements + are either strings or a list of substitutions will become a single list of substitutions that + will evaluate to a single string. To make a list of strings from substitutions, each item in the list must be a list or tuple. Normalized values that contained nested dictionaries will be collapsed into a single diff --git a/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py index f36075458..a010de1e3 100644 --- a/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py +++ b/test_launch_ros/test/test_launch_ros/test_normalize_parameters.py @@ -176,6 +176,26 @@ def test_dictionary_with_dissimilar_array(): expected = ({'foo': 1, 'fiz': ('True', '2.0', '3')},) assert evaluate_parameters(LaunchContext(), norm) == expected + orig = [{'foo': 1, 'fiz': [True, 1, TextSubstitution(text='foo')]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': ('True', '1', 'foo')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + orig = [{'foo': 1, 'fiz': [TextSubstitution(text='foo'), True, 1]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': ('foo', 'True', '1')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + orig = [{'foo': 1, 'fiz': [True, 1, [TextSubstitution(text='foo')]]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': ('True', '1', 'foo')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + + orig = [{'foo': 1, 'fiz': [[TextSubstitution(text='foo')], True, 1]}] + norm = normalize_parameters(orig) + expected = ({'foo': 1, 'fiz': ('foo', 'True', '1')},) + assert evaluate_parameters(LaunchContext(), norm) == expected + def test_nested_dictionaries(): orig = [{'foo': {'bar': 'baz'}, 'fiz': {'buz': 3}}] From 65f23a6cb5b70e4a162af6a6da233dbb851d60c0 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 5 Mar 2019 16:21:18 -0800 Subject: [PATCH 13/13] SomeParameterValue multi value substitutions Signed-off-by: Shane Loretz --- launch_ros/launch_ros/parameters_type.py | 5 ++++- launch_ros/launch_ros/utilities/normalize_parameters.py | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/launch_ros/launch_ros/parameters_type.py b/launch_ros/launch_ros/parameters_type.py index 3cd4001b2..fb2dc495f 100644 --- a/launch_ros/launch_ros/parameters_type.py +++ b/launch_ros/launch_ros/parameters_type.py @@ -33,7 +33,10 @@ SomeParameterFile = Union[SomeSubstitutionsType, pathlib.Path] SomeParameterName = Sequence[Union[Substitution, str]] -SomeParameterValue = Union[SomeSubstitutionsType, _SingleValueType, _MultiValueType] +SomeParameterValue = Union[SomeSubstitutionsType, + Sequence[SomeSubstitutionsType], + _SingleValueType, + _MultiValueType] # TODO(sloretz) Recursive type when mypy supports them python/mypy#731 _SomeParametersDict = Mapping[SomeParameterName, Any] diff --git a/launch_ros/launch_ros/utilities/normalize_parameters.py b/launch_ros/launch_ros/utilities/normalize_parameters.py index 55c4077af..35040d835 100644 --- a/launch_ros/launch_ros/utilities/normalize_parameters.py +++ b/launch_ros/launch_ros/utilities/normalize_parameters.py @@ -68,11 +68,13 @@ def _normalize_parameter_array_value(value: SomeParameterValue) -> ParameterValu raise RuntimeError('Failed to handle type {}'.format(repr(subvalue))) if {int} == has_types: - # all integers - return tuple(int(e) for e in value) + # everything is an integer + make_mypy_happy_int = cast(List[int], value) + return tuple(int(e) for e in make_mypy_happy_int) elif has_types in ({float}, {int, float}): # all were floats or ints, so return floats - return tuple(float(e) for e in value) + make_mypy_happy_float = cast(List[Union[int, float]], value) + return tuple(float(e) for e in make_mypy_happy_float) elif Substitution in has_types and has_types.issubset({str, Substitution, tuple}): # make a list of substitutions forming a single string return tuple(normalize_to_list_of_substitutions(cast(SomeSubstitutionsType, value)))