From 07ec70dc0a9c320afffbbb4dcfc0248705d6a53f Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 10:47:52 -0400 Subject: [PATCH 01/13] outlined _load_param_tag --- src/roswire/proxy/launch/__init__.py | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index ad0758ff..632b8ac4 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -27,6 +27,12 @@ _TAG_TO_LOADER = {} +def _read_contents(tag: ET.Element) -> str: + """Reads the text contents of an XML element.""" + # FIXME add support for CDATA -- possibly via lxml or xml.dom? + return ''.join(t.text for t in tag if t.text) + + def _parse_bool(attr: str, val: str) -> bool: """Parses a boolean value from an XML attribute.""" val = val.lower() @@ -170,6 +176,43 @@ def _load_param_tag(self, return ctx, cfg + @tag('rosparam', ['command', 'ns', 'file', 'param', 'subst_value']) + def _load_rosparam_tag(self, + ctx: LaunchContext, + cfg: ROSConfig, + tag: ET.Element + ) -> Tuple[LaunchContext, ROSConfig]: + filename = self._read_optional(tag, 'file', ctx) + ns = self._read_optional(tag, 'ns', ctx) or '' + param = self._read_optional(tag, 'param', ctx) or '' + param = namespace_join(ns, param) + param = namespace_join(ctx.namespace, param) + value = _get_text(tag) + + if self._read_optional_bool(tag, 'subst_value', ctx, False): + subst_func = lambda s: self._resolve_args(s, ctx) + + cmd = self._read_optional(tag, 'command', ctx) or 'load' + if cmd not in ('load', 'delete', 'dump'): + m = f" unsupported 'command': {cmd}" + raise FailedToParseLaunchFile(m) + + if cmd == 'load' and not self.__files.isfile(filename): + m = f" file does not exist: {filename}" + raise FailedToParseLaunchFile(m) + + if cmd == 'delete' and filename is not None: + m = " command:delete does not support filename" + raise FailedToParseLaunchFile(m) + + # TODO handle load command + if cmd == 'load': + + # TODO handle dump command + # TODO handle delete command + + return ctx, cfg + @tag('remap', ['from', 'to']) def _load_remap_tag(self, ctx: LaunchContext, From 5ebef2e07c715ef36eee91739b79ccd1da305a8d Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 11:05:03 -0400 Subject: [PATCH 02/13] throw error on unsupported commands --- src/roswire/proxy/launch/__init__.py | 34 ++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index 632b8ac4..9c0952a8 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -11,6 +11,7 @@ import xml.etree.ElementTree as ET import attr +import yaml from .config import ROSConfig, NodeConfig, Parameter from .context import LaunchContext @@ -183,33 +184,46 @@ def _load_rosparam_tag(self, tag: ET.Element ) -> Tuple[LaunchContext, ROSConfig]: filename = self._read_optional(tag, 'file', ctx) + subst_value = self._read_optional_bool(tag, 'subst_value', ctx, False) ns = self._read_optional(tag, 'ns', ctx) or '' param = self._read_optional(tag, 'param', ctx) or '' param = namespace_join(ns, param) - param = namespace_join(ctx.namespace, param) + full_param = namespace_join(ctx.namespace, param) value = _get_text(tag) - if self._read_optional_bool(tag, 'subst_value', ctx, False): - subst_func = lambda s: self._resolve_args(s, ctx) - cmd = self._read_optional(tag, 'command', ctx) or 'load' if cmd not in ('load', 'delete', 'dump'): - m = f" unsupported 'command': {cmd}" + m = f" unsupported 'command': {cmd}" raise FailedToParseLaunchFile(m) if cmd == 'load' and not self.__files.isfile(filename): - m = f" file does not exist: {filename}" + m = f" file does not exist: {filename}" raise FailedToParseLaunchFile(m) if cmd == 'delete' and filename is not None: - m = " command:delete does not support filename" + m = " command:delete does not support filename" raise FailedToParseLaunchFile(m) - # TODO handle load command + # handle load command if cmd == 'load': + yml_text = self.__files.read(filename) + if subst_value: + yml_text = self._resolve_args(yml_text, ctx) + data = yaml.safe_load(yml_text) or {} + if type(data) != dict and not param: + m = " requires 'param' for non-dictionary values" + raise FailedToParseLaunchFile(m) + cfg = cfg.with_param(full_param, data) + + # handle dump command + if cmd == 'dump': + m = "'dump' command is currently not supported in " + raise NotImplementedError(m) - # TODO handle dump command - # TODO handle delete command + # handle delete command + if cmd == 'delete': + m = "'delete' command is currently not supported in " + raise NotImplementedError(m) return ctx, cfg From 6a86c0f6c6b3b54eca6349a428dff3ff466379cf Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 11:37:15 -0400 Subject: [PATCH 03/13] fixed most mypy issues --- src/roswire/proxy/launch/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index 9c0952a8..c2b0c14d 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -189,13 +189,17 @@ def _load_rosparam_tag(self, param = self._read_optional(tag, 'param', ctx) or '' param = namespace_join(ns, param) full_param = namespace_join(ctx.namespace, param) - value = _get_text(tag) + value = _read_contents(tag) - cmd = self._read_optional(tag, 'command', ctx) or 'load' + cmd: str = self._read_optional(tag, 'command', ctx) or 'load' if cmd not in ('load', 'delete', 'dump'): m = f" unsupported 'command': {cmd}" raise FailedToParseLaunchFile(m) + if cmd == 'load' and not filename: + m = f" load command requires filename" + raise FailedToParseLaunchFile(m) + if cmd == 'load' and not self.__files.isfile(filename): m = f" file does not exist: {filename}" raise FailedToParseLaunchFile(m) @@ -206,6 +210,7 @@ def _load_rosparam_tag(self, # handle load command if cmd == 'load': + assert filename # stupid mypy can't figure this out yml_text = self.__files.read(filename) if subst_value: yml_text = self._resolve_args(yml_text, ctx) From 4dd9286b911e66b56616b83b78db3e8bb2a8e571 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 15:28:48 -0400 Subject: [PATCH 04/13] workaround for mypy bug --- src/roswire/proxy/launch/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index c2b0c14d..783563d4 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -200,9 +200,11 @@ def _load_rosparam_tag(self, m = f" load command requires filename" raise FailedToParseLaunchFile(m) - if cmd == 'load' and not self.__files.isfile(filename): - m = f" file does not exist: {filename}" - raise FailedToParseLaunchFile(m) + if cmd == 'load': + assert filename is not None # mypy can't work this out + if not self.__files.isfile(filename): + m = f" file does not exist: {filename}" + raise FailedToParseLaunchFile(m) if cmd == 'delete' and filename is not None: m = " command:delete does not support filename" @@ -210,7 +212,7 @@ def _load_rosparam_tag(self, # handle load command if cmd == 'load': - assert filename # stupid mypy can't figure this out + assert filename is not None # mypy can't work this out yml_text = self.__files.read(filename) if subst_value: yml_text = self._resolve_args(yml_text, ctx) From ae1a111a0ba0995539e52d070cf4f3662f513a27 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 15:31:18 -0400 Subject: [PATCH 05/13] remove f-string --- src/roswire/proxy/launch/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index 783563d4..5e0a35bb 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -197,7 +197,7 @@ def _load_rosparam_tag(self, raise FailedToParseLaunchFile(m) if cmd == 'load' and not filename: - m = f" load command requires filename" + m = " load command requires 'filename' attribute" raise FailedToParseLaunchFile(m) if cmd == 'load': From 2de536a6eb4fdebad33d4a6d2233c1e96a5049bd Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 15:45:22 -0400 Subject: [PATCH 06/13] use YAML full_load --- src/roswire/proxy/launch/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index 5e0a35bb..2927bafc 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -216,7 +216,8 @@ def _load_rosparam_tag(self, yml_text = self.__files.read(filename) if subst_value: yml_text = self._resolve_args(yml_text, ctx) - data = yaml.safe_load(yml_text) or {} + logger.debug("parsing rosparam YAML:\n%s", yml_text) + data = yaml.full_load(yml_text) or {} if type(data) != dict and not param: m = " requires 'param' for non-dictionary values" raise FailedToParseLaunchFile(m) From 9a7c99dcb6a4ddddc30c2fb96b289a13e840eb73 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 15:48:02 -0400 Subject: [PATCH 07/13] added rosparam --- src/roswire/proxy/launch/rosparam.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/roswire/proxy/launch/rosparam.py diff --git a/src/roswire/proxy/launch/rosparam.py b/src/roswire/proxy/launch/rosparam.py new file mode 100644 index 00000000..94fc086b --- /dev/null +++ b/src/roswire/proxy/launch/rosparam.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +""" +This file provides utilities for interacting with rosparam. +""" From 11fbad01b21201f55bd3c62fe561a3e31ea9995d Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 16:02:07 -0400 Subject: [PATCH 08/13] added partial load_radians --- src/roswire/proxy/launch/rosparam.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/roswire/proxy/launch/rosparam.py b/src/roswire/proxy/launch/rosparam.py index 94fc086b..c487050a 100644 --- a/src/roswire/proxy/launch/rosparam.py +++ b/src/roswire/proxy/launch/rosparam.py @@ -2,3 +2,18 @@ """ This file provides utilities for interacting with rosparam. """ + + +def load_radians(loader, node) -> float: + """Safely converts rad(num) to a float value. + + Note + ---- + This does not support evaluation of expressions. + """ + expr_s = loader.construct_scalar(node).strip() + if expr_s.startswith('rad('): + expr_s = expr_s[4:-1] + + # TODO safely parse and evaluate expression + return float(expr_s) From c1fbeddeee2fc53efced675bd4604b83dba8f208 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 16:11:23 -0400 Subject: [PATCH 09/13] added rosparam loaders --- src/roswire/proxy/launch/rosparam.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/roswire/proxy/launch/rosparam.py b/src/roswire/proxy/launch/rosparam.py index c487050a..e73ccf10 100644 --- a/src/roswire/proxy/launch/rosparam.py +++ b/src/roswire/proxy/launch/rosparam.py @@ -2,8 +2,18 @@ """ This file provides utilities for interacting with rosparam. """ +import math +import yaml +# allow both PyYAML and LibYAML +try: + from yaml import CLoader as YAMLLoader +except ImportError: + from yaml import YAMLLoader + + +@yaml.add_constructor('!radians') def load_radians(loader, node) -> float: """Safely converts rad(num) to a float value. @@ -17,3 +27,21 @@ def load_radians(loader, node) -> float: # TODO safely parse and evaluate expression return float(expr_s) + + +@yaml.add_constructor('!degrees') +def load_degrees(loader, node) -> float: + """Safely converts deg(num) to a float value. + + Note + ---- + This does not support evaluation of expressions. + """ + expr_s = loader.construct_scalar(node).strip() + if expr_s.startswith('def('): + expr_s = expr_s[4:-1] + return float(expr_s) * math.pi / 180.0 + + +yaml.add_implicit_resolve('!degrees', r'^deg\([^\)]*\)$', first='deg(') +yaml.add_implicit_resolve('!radians', r'^rad\([^\)]*\)$', first='rad(') From aeaa8c8bbf03ec3ab17672490c8b02f193fa32f2 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 16:13:44 -0400 Subject: [PATCH 10/13] added annotations --- src/roswire/proxy/launch/rosparam.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/roswire/proxy/launch/rosparam.py b/src/roswire/proxy/launch/rosparam.py index e73ccf10..77e8b0aa 100644 --- a/src/roswire/proxy/launch/rosparam.py +++ b/src/roswire/proxy/launch/rosparam.py @@ -9,12 +9,13 @@ # allow both PyYAML and LibYAML try: from yaml import CLoader as YAMLLoader + from yaml import CYAMLObject as YAMLObject except ImportError: - from yaml import YAMLLoader + from yaml import YAMLLoader, YAMLObject @yaml.add_constructor('!radians') -def load_radians(loader, node) -> float: +def load_radians(loader: YAMLLoader, node: YAMLObject) -> float: """Safely converts rad(num) to a float value. Note @@ -30,13 +31,8 @@ def load_radians(loader, node) -> float: @yaml.add_constructor('!degrees') -def load_degrees(loader, node) -> float: - """Safely converts deg(num) to a float value. - - Note - ---- - This does not support evaluation of expressions. - """ +def load_degrees(loader: YAMLLoader, node: YAMLObject) -> float: + """Safely converts deg(num) to a float value.""" expr_s = loader.construct_scalar(node).strip() if expr_s.startswith('def('): expr_s = expr_s[4:-1] From 2121f55c9bf140612c11f12f6370964c4367953d Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 16:33:53 -0400 Subject: [PATCH 11/13] implemented rosparam loading --- src/roswire/proxy/launch/__init__.py | 4 +++- src/roswire/proxy/launch/rosparam.py | 32 +++++++++++++++++----------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index 2927bafc..8d1272fd 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -13,6 +13,7 @@ import attr import yaml +from .rosparam import load_from_yaml_string as load_rosparam_from_string from .config import ROSConfig, NodeConfig, Parameter from .context import LaunchContext from ..substitution import resolve as resolve_args @@ -217,7 +218,8 @@ def _load_rosparam_tag(self, if subst_value: yml_text = self._resolve_args(yml_text, ctx) logger.debug("parsing rosparam YAML:\n%s", yml_text) - data = yaml.full_load(yml_text) or {} + data = load_rosparam_from_string(yml_text) + logger.debug("rosparam values: %s", data) if type(data) != dict and not param: m = " requires 'param' for non-dictionary values" raise FailedToParseLaunchFile(m) diff --git a/src/roswire/proxy/launch/rosparam.py b/src/roswire/proxy/launch/rosparam.py index 77e8b0aa..82890068 100644 --- a/src/roswire/proxy/launch/rosparam.py +++ b/src/roswire/proxy/launch/rosparam.py @@ -2,20 +2,25 @@ """ This file provides utilities for interacting with rosparam. """ +__all__ = ('load_from_yaml_string',) + +from typing import Dict, Any import math +import re import yaml -# allow both PyYAML and LibYAML -try: - from yaml import CLoader as YAMLLoader - from yaml import CYAMLObject as YAMLObject -except ImportError: - from yaml import YAMLLoader, YAMLObject + +class YAMLLoader(yaml.SafeLoader): + """A custom YAML loader for rosparam files.""" + + +def load_from_yaml_string(s: str) -> Dict[str, Any]: + """Parses the contents of a rosparam file to a dictionary.""" + return yaml.load(s, Loader=YAMLLoader) or {} -@yaml.add_constructor('!radians') -def load_radians(loader: YAMLLoader, node: YAMLObject) -> float: +def __load_radians(loader: YAMLLoader, node: yaml.YAMLObject) -> float: """Safely converts rad(num) to a float value. Note @@ -30,8 +35,7 @@ def load_radians(loader: YAMLLoader, node: YAMLObject) -> float: return float(expr_s) -@yaml.add_constructor('!degrees') -def load_degrees(loader: YAMLLoader, node: YAMLObject) -> float: +def __load_degrees(loader: YAMLLoader, node: yaml.YAMLObject) -> float: """Safely converts deg(num) to a float value.""" expr_s = loader.construct_scalar(node).strip() if expr_s.startswith('def('): @@ -39,5 +43,9 @@ def load_degrees(loader: YAMLLoader, node: YAMLObject) -> float: return float(expr_s) * math.pi / 180.0 -yaml.add_implicit_resolve('!degrees', r'^deg\([^\)]*\)$', first='deg(') -yaml.add_implicit_resolve('!radians', r'^rad\([^\)]*\)$', first='rad(') +YAMLLoader.add_constructor('!degrees', __load_degrees) +YAMLLoader.add_implicit_resolver( + '!degrees', re.compile('^deg\([^\)]*\)$'), first='deg(') +YAMLLoader.add_constructor('!radians', __load_radians) +YAMLLoader.add_implicit_resolver( + '!radians', re.compile('^rad\([^\)]*\)$'), first='rad(') From cb8e426ec267293af9c699afeeb4f06c8bd260db Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 16:37:48 -0400 Subject: [PATCH 12/13] fixed dict checking --- src/roswire/proxy/launch/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roswire/proxy/launch/__init__.py b/src/roswire/proxy/launch/__init__.py index 8d1272fd..0870df73 100644 --- a/src/roswire/proxy/launch/__init__.py +++ b/src/roswire/proxy/launch/__init__.py @@ -220,7 +220,7 @@ def _load_rosparam_tag(self, logger.debug("parsing rosparam YAML:\n%s", yml_text) data = load_rosparam_from_string(yml_text) logger.debug("rosparam values: %s", data) - if type(data) != dict and not param: + if not isinstance(data, dict) and not param: m = " requires 'param' for non-dictionary values" raise FailedToParseLaunchFile(m) cfg = cfg.with_param(full_param, data) From df30db97c040c329b70b7960728b0bbad4bcac4e Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Fri, 14 Jun 2019 16:38:24 -0400 Subject: [PATCH 13/13] excess whitespace --- src/roswire/proxy/launch/rosparam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roswire/proxy/launch/rosparam.py b/src/roswire/proxy/launch/rosparam.py index 82890068..09b59325 100644 --- a/src/roswire/proxy/launch/rosparam.py +++ b/src/roswire/proxy/launch/rosparam.py @@ -22,7 +22,7 @@ def load_from_yaml_string(s: str) -> Dict[str, Any]: def __load_radians(loader: YAMLLoader, node: yaml.YAMLObject) -> float: """Safely converts rad(num) to a float value. - + Note ---- This does not support evaluation of expressions.