From a2d7904594991cfa02e2465acd430459ae0e6501 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 29 Feb 2024 22:05:37 +0000 Subject: [PATCH] [SkyMarshal] Drop py2 syntax Topic: py-6to3 Relative: sky-binding-types GitOrigin-RevId: 8503688e25589455f1993b6eee00a83aae3fce51 --- third_party/skymarshal/skymarshal/__main__.py | 5 +- .../skymarshal/skymarshal/common_util.py | 19 +- third_party/skymarshal/skymarshal/emit_cpp.py | 55 ++- .../skymarshal/skymarshal/emit_djinni.py | 31 +- .../skymarshal/skymarshal/emit_helpers.py | 31 +- .../skymarshal/skymarshal/emit_java.py | 23 +- .../skymarshal/skymarshal/emit_lcmout.py | 21 +- .../skymarshal/skymarshal/emit_proto.py | 159 ++++--- .../skymarshal/skymarshal/emit_python.py | 191 ++++---- .../skymarshal/skymarshal/emit_typescript.py | 82 ++-- .../skymarshal/skymarshal/language_plugin.py | 17 +- .../skymarshal/skymarshal/package_map.py | 32 +- third_party/skymarshal/skymarshal/parser.py | 26 +- .../skymarshal/regenerate_lextab.py | 35 +- .../skymarshal/skymarshal/skymarshal.py | 25 +- .../skymarshal/skymarshal/syntax_tree.py | 426 ++++++++---------- .../skymarshal/skymarshal/tokenizer.py | 7 +- 17 files changed, 535 insertions(+), 650 deletions(-) diff --git a/third_party/skymarshal/skymarshal/__main__.py b/third_party/skymarshal/skymarshal/__main__.py index e2b2a8030..c8047368e 100644 --- a/third_party/skymarshal/skymarshal/__main__.py +++ b/third_party/skymarshal/skymarshal/__main__.py @@ -1,6 +1,7 @@ -# aclint: py2 py3 +# aclint: py3 "Allow the skymarshal package to be executable" -from __future__ import absolute_import + +from __future__ import annotations from skymarshal import skymarshal from skymarshal.emit_cpp import SkymarshalCpp diff --git a/third_party/skymarshal/skymarshal/common_util.py b/third_party/skymarshal/skymarshal/common_util.py index 084c8a108..76f2f0d3b 100644 --- a/third_party/skymarshal/skymarshal/common_util.py +++ b/third_party/skymarshal/skymarshal/common_util.py @@ -1,5 +1,6 @@ -# aclint: py2 py3 -from __future__ import absolute_import, print_function +# aclint: py3 + +from __future__ import annotations import sys import typing as T @@ -15,26 +16,23 @@ StrType = T.TypeVar("StrType", str, T.Text) -def snakecase_to_camelcase(snake_string): - # type: (StrType) -> StrType +def snakecase_to_camelcase(snake_string: StrType) -> StrType: return "".join(word.capitalize() for word in snake_string.split("_")) -def snakecase_to_lower_camelcase(snake_string): - # type: (StrType) -> StrType +def snakecase_to_lower_camelcase(snake_string: StrType) -> StrType: words = snake_string.split("_") return words[0] + "".join(word.capitalize() for word in words[1:]) -def camelcase_to_snakecase(camelcase, to_upper=False): - # type: (StrType, bool) -> StrType +def camelcase_to_snakecase(camelcase: StrType, to_upper: bool = False) -> StrType: if "_" in camelcase: # This string is already using underscores. if to_upper: return camelcase.upper() else: return camelcase.lower() - out = [] # type: T.List[StrType] + out: T.List[StrType] = [] for char in camelcase: if out and char in uppercase: out.append("_") @@ -46,8 +44,7 @@ def camelcase_to_snakecase(camelcase, to_upper=False): return "".join(out) -def is_camelcase(string): - # type: (StrType) -> bool +def is_camelcase(string: StrType) -> bool: if not string: return False if "_" in string or " " in string: diff --git a/third_party/skymarshal/skymarshal/emit_cpp.py b/third_party/skymarshal/skymarshal/emit_cpp.py index 8a5c933a5..f0209556b 100644 --- a/third_party/skymarshal/skymarshal/emit_cpp.py +++ b/third_party/skymarshal/skymarshal/emit_cpp.py @@ -1,6 +1,7 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs -from __future__ import absolute_import +# aclint: py3 + +from __future__ import annotations import argparse # pylint: disable=unused-import import copy @@ -8,7 +9,6 @@ import typing as T import six -from six.moves import range # pylint: disable=redefined-builtin from skymarshal import syntax_tree from skymarshal.emit_helpers import BaseBuilder, Code, EnumBuilder, StructBuilder, render from skymarshal.language_plugin import SkymarshalLanguage @@ -68,9 +68,9 @@ def declare_member(member, suffix="", const_ref=False): type_str = mapped_typename if const_ref: - type_str = "const {}&".format(type_str) + type_str = f"const {type_str}&" - return "{} {}{}".format(type_str, member.name, suffix) + return f"{type_str} {member.name}{suffix}" def map_to_cpptype(type_ref): @@ -95,10 +95,10 @@ def dim_size_access(dim, array_member=None): """Get the size of a single array dimension""" if dim.auto_member: if array_member: - return "this->{}.size()".format(array_member.name) - return "v_{}".format(dim.size_str) + return f"this->{array_member.name}.size()" + return f"v_{dim.size_str}" if dim.dynamic: - return "this->{}".format(dim.size_str) + return f"this->{dim.size_str}" return dim.size_str @@ -111,7 +111,7 @@ def namespace(self): @property def filename(self): - return "{}.hpp".format(os.path.join(self.namespace, self.name)) + return f"{os.path.join(self.namespace, self.name)}.hpp" @property def fullpath(self): @@ -122,7 +122,7 @@ def fullpath(self): @property def underscored(self): - return "{}_{}".format(self.namespace, self.name) + return f"{self.namespace}_{self.name}" @property def comment(self): @@ -143,8 +143,7 @@ def comment(self): return "\n" + "\n".join(lines) @property - def size_t(self): - # type: () -> str + def size_t(self) -> str: """ The C++ type to use for buffer size variables """ @@ -164,7 +163,7 @@ def string_cast_type(self): return {"int8_t": "int16_t"}.get(storage_name, storage_name) -class CppInclude(object): +class CppInclude: def __init__(self, member=None, std=None, prefix=None): assert member or std self.member = member @@ -189,10 +188,10 @@ def absolute_path(self): def directive(self): name = self.absolute_path if self.std is not None: - name = "<{}>".format(name) + name = f"<{name}>" else: - name = '"{}"'.format(name) - return "#include {}\n".format(name) + name = f'"{name}"' + return f"#include {name}\n" @property def package(self): @@ -236,7 +235,7 @@ def include_list(self): # This is an lcmtype if include.relative_path != self.filename: includes.add(include) - std_includes = set(inc for inc in includes if inc.std) + std_includes = {inc for inc in includes if inc.std} lcm_includes = includes - std_includes return sorted(std_includes, key=str) + sorted(lcm_includes, key=str) @@ -278,7 +277,7 @@ def define_constants(self): else: mapped_typename = map_to_cpptype(constant.type_ref) if self.args.cpp_std not in ("c++98", "c++11"): - raise ValueError("Invalid C++ std: {}".format(self.args.cpp_std)) + raise ValueError(f"Invalid C++ std: {self.args.cpp_std}") if self.args.cpp_std == "c++11": code(2, "// If you are getting compiler/linker errors saying things like") @@ -324,8 +323,9 @@ def constructor_args(self): def initializers(self): return ",\n".join("{0}({0}_arg)".format(member.name) for member in self.members) - def encode_recursive(self, code, member, depth, extra_indent): - # type: (Code, syntax_tree.Member, int, int) -> None + def encode_recursive( + self, code: Code, member: syntax_tree.Member, depth: int, extra_indent: int + ) -> None: indent = extra_indent + 1 + depth # primitive array if member.ndim == (depth + 1) and member.type_ref.is_non_string_primitive_type(): @@ -444,7 +444,7 @@ def compute_hash(self): for member in self.members: if not member.type_ref.is_primitive_type(): scoped_name = map_to_cpptype(member.type_ref) - hash_calls.append("{}::_computeHash(&cp)".format(scoped_name)) + hash_calls.append(f"{scoped_name}::_computeHash(&cp)") return " +\n ".join(hash_calls) def encoded_size(self): @@ -452,7 +452,7 @@ def encoded_size(self): return " return 0;" code = Code() - code(1, "{} enc_size = 0;".format(self.size_t)) + code(1, f"{self.size_t} enc_size = 0;") for member in self.members: self.encoded_size_member(code, member) @@ -661,10 +661,9 @@ def add_args(cls, parser): @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: """Turn a list of lcm packages into C++ bindings for each struct. @param packages: the list of syntax_tree.Package objects @@ -682,7 +681,7 @@ def create_files( file_map = {} for package in packages: for struct in package.struct_definitions: - cppclass = CppStruct(package, struct, args) # type: T.Union[CppStruct, CppEnum] + cppclass: T.Union[CppStruct, CppEnum] = CppStruct(package, struct, args) file_map[cppclass.fullpath] = render( "lcmtype.hpp.template", lcmtype=cppclass, @@ -699,7 +698,7 @@ def create_files( ) type_names = sorted( - type_definition.name for type_definition in six.itervalues(package.type_definitions) + type_definition.name for type_definition in package.type_definitions.values() ) if args.cpp_aggregate: diff --git a/third_party/skymarshal/skymarshal/emit_djinni.py b/third_party/skymarshal/skymarshal/emit_djinni.py index 11bcda95d..a494468b8 100644 --- a/third_party/skymarshal/skymarshal/emit_djinni.py +++ b/third_party/skymarshal/skymarshal/emit_djinni.py @@ -1,7 +1,8 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs +# aclint: py3 """Generate djinni enum definitions and conversions.""" -from __future__ import absolute_import + +from __future__ import annotations import argparse # pylint: disable=unused-import import os @@ -14,7 +15,7 @@ from skymarshal.language_plugin import SkymarshalLanguage -class EnumCase(object): +class EnumCase: """A template-friendly wrapper object for LCM #djinni Enum Cases""" def __init__(self, int_value, name): @@ -27,7 +28,7 @@ def __init__(self, int_value, name): self.proto_name = name -class EnumType(object): +class EnumType: """A template-friendly wrapper object for LCM #djinni Enums""" # pylint: disable=too-many-instance-attributes @@ -49,7 +50,7 @@ def __init__(self, package_name, enum, args): self.default_case = self.cases[0] # names for templating - self.definition_name = "{}.{}".format(package_name, enum.name) + self.definition_name = f"{package_name}.{enum.name}" self.djinni_idl_name = snakecase_name self.djinni_name = camelcase_name self.djinni_namespace = args.djinni_module @@ -57,12 +58,12 @@ def __init__(self, package_name, enum, args): self.lcm_package = package_name # filenames for generated converter sources - self.filename_h = "{}/converters/{}.h".format(self.djinni_namespace, snakecase_name) - self.filename_cc = "{}/converters/{}.cc".format(self.djinni_namespace, snakecase_name) + self.filename_h = f"{self.djinni_namespace}/converters/{snakecase_name}.h" + self.filename_cc = f"{self.djinni_namespace}/converters/{snakecase_name}.cc" # header paths to the underlying type definitions - self.djinni_header = "djinni/{}/{}.hpp".format(self.djinni_namespace, snakecase_name) - self.lcm_header = "lcmtypes/{}/{}.hpp".format(self.lcm_package, self.lcm_name) + self.djinni_header = f"djinni/{self.djinni_namespace}/{snakecase_name}.hpp" + self.lcm_header = f"lcmtypes/{self.lcm_package}/{self.lcm_name}.hpp" if enum.get_notation("#protobuf") is None: self.is_protobuf = False @@ -78,8 +79,7 @@ def __init__(self, package_name, enum, args): class SkymarshalDjinni(SkymarshalLanguage): @classmethod - def add_args(cls, parser): - # type: (argparse.ArgumentParser) -> None + def add_args(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument("--djinni", action="store_true", help="generate converters for djinni") parser.add_argument("--djinni-idl-path", help="Full path of the .djinni file") parser.add_argument("--djinni-path", help="Location of the .cc and .h files") @@ -90,10 +90,9 @@ def add_args(cls, parser): @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: """ Turn a list of lcm packages into a djinni definition file and source files for lcm converters. @@ -118,7 +117,7 @@ def create_files( enum_types.sort(key=lambda x: x.djinni_idl_name) for enum_type in enum_types: - idl_file = "{}.djinni".format(args.djinni_module) + idl_file = f"{args.djinni_module}.djinni" if args.djinni_idl_path: idl_file = os.path.join(args.djinni_idl_path, idl_file, enum_type.djinni_idl_name) file_map[idl_file] = render("djinni_idl.djinni.template", enum_types=[enum_type]) diff --git a/third_party/skymarshal/skymarshal/emit_helpers.py b/third_party/skymarshal/skymarshal/emit_helpers.py index e80b62763..94d5ad0bd 100644 --- a/third_party/skymarshal/skymarshal/emit_helpers.py +++ b/third_party/skymarshal/skymarshal/emit_helpers.py @@ -1,6 +1,7 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs -from __future__ import absolute_import +# aclint: py3 + +from __future__ import annotations import io import os @@ -10,7 +11,7 @@ from skymarshal.syntax_tree import ConstMember -class TemplateRenderer(object): +class TemplateRenderer: "A wrapper around a jinja template Environment instantiated for this package" def __init__(self, module_dir): @@ -29,7 +30,7 @@ class Code(io.StringIO): Modeled off the emit functions from the original lcmgen.""" def start(self, indent, fmt, *args): - self.write(u" " * indent) + self.write(" " * indent) self.add(fmt, *args) def add(self, fmt, *args): @@ -37,7 +38,7 @@ def add(self, fmt, *args): def end(self, fmt, *args): self.add(fmt, *args) - self.write(u"\n") + self.write("\n") def _format(self, fmt, args): if isinstance(fmt, bytes): @@ -45,12 +46,12 @@ def _format(self, fmt, args): return fmt % args def __call__(self, indent, fmt, *args): - self.write(u" " * indent) + self.write(" " * indent) self.write(self._format(fmt, args)) - self.write(u"\n") + self.write("\n") -class BaseBuilder(object): +class BaseBuilder: def __init__(self, package, name, full_name, comments, args): self.package = package self.name = name @@ -69,17 +70,15 @@ class StructBuilder(BaseBuilder): """Helper class for converting a lcm struct into a destination code file.""" def __init__(self, package, struct, args): - super(StructBuilder, self).__init__( - package, struct.name, struct.full_name, struct.comments, args - ) + super().__init__(package, struct.name, struct.full_name, struct.comments, args) self.struct = struct - self.members = [ + self.members: T.List[T.Any] = [ member for member in struct.members if not isinstance(member, ConstMember) - ] # type: T.List[T.Any] + ] self.num_members = len(self.members) - self.constants = [ + self.constants: T.List[T.Any] = [ member for member in struct.members if isinstance(member, ConstMember) - ] # type: T.List[T.Any] + ] @property def hash(self): @@ -90,7 +89,7 @@ class EnumBuilder(BaseBuilder): """Helper class for converting a 'lcm' enum into a destination code file.""" def __init__(self, package, enum, args): - super(EnumBuilder, self).__init__(package, enum.name, enum.full_name, enum.comments, args) + super().__init__(package, enum.name, enum.full_name, enum.comments, args) self.enum = enum self.cases = enum.cases self.num_cases = len(enum.cases) diff --git a/third_party/skymarshal/skymarshal/emit_java.py b/third_party/skymarshal/skymarshal/emit_java.py index 93d129e54..0ecbef7d3 100644 --- a/third_party/skymarshal/skymarshal/emit_java.py +++ b/third_party/skymarshal/skymarshal/emit_java.py @@ -1,6 +1,7 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs -from __future__ import absolute_import +# aclint: py3 + +from __future__ import annotations import argparse # pylint: disable=unused-import import collections @@ -10,7 +11,6 @@ import zipfile from io import BytesIO -from six.moves import range # pylint: disable=redefined-builtin from skymarshal import syntax_tree from skymarshal.emit_helpers import Code, StructBuilder, render from skymarshal.language_plugin import SkymarshalLanguage @@ -64,7 +64,7 @@ def make_accessor(member, obj, get_last_array=False): else: num = member.ndim for i in range(num): - accessor += "[{}]".format(string.ascii_lowercase[i]) + accessor += f"[{string.ascii_lowercase[i]}]" return accessor @@ -72,7 +72,7 @@ def make_accessor(member, obj, get_last_array=False): def dim_size_access(dim): """Get the size of a single array dimension.""" if dim.dynamic: - return "this.{}".format(dim.size_str) + return f"this.{dim.size_str}" return dim.size_str @@ -81,7 +81,7 @@ class JavaClass(StructBuilder): def __init__(self, package, struct, args): # TODO(jeff): Implement auto arrays for java classes instead of exposing the legacy form. - super(JavaClass, self).__init__(package, struct, args) + super().__init__(package, struct, args) new_members = [] for member in self.members: if member.ndim: @@ -93,7 +93,7 @@ def __init__(self, package, struct, args): @property def filename(self): - return "{}.java".format(os.path.join(self.package.name, self.name)) + return f"{os.path.join(self.package.name, self.name)}.java" @property def fullpath(self): @@ -194,7 +194,7 @@ def encode_recursive(self, code, member, pinfo, accessor, depth): if pinfo: code.add(pinfo.encode.replace("#", accessor)) else: - code.add("{}._encodeRecursive(outs);".format(accessor)) + code.add(f"{accessor}._encodeRecursive(outs);") code.end(" ") return @@ -375,10 +375,9 @@ def add_args(cls, parser): @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: """Turn a list of lcm packages into java bindings for each struct. @param packages: the list of syntax_tree.Package objects diff --git a/third_party/skymarshal/skymarshal/emit_lcmout.py b/third_party/skymarshal/skymarshal/emit_lcmout.py index d192af0c0..a0bf2ca46 100644 --- a/third_party/skymarshal/skymarshal/emit_lcmout.py +++ b/third_party/skymarshal/skymarshal/emit_lcmout.py @@ -1,9 +1,10 @@ -# aclint: py2 py3 +# aclint: py3 """ Generate output files containing just the definition for single LCM types. This is used by get_type_for_str, used most notably in json_lcm. """ -from __future__ import absolute_import + +from __future__ import annotations import argparse # pylint: disable=unused-import import os @@ -15,8 +16,7 @@ class SkymarshalLCMOut(SkymarshalLanguage): @classmethod - def add_args(cls, parser): - # type: (argparse.ArgumentParser) -> None + def add_args(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument( "--lcmout", action="store_true", help="output sources for each LCM type" ) @@ -25,10 +25,9 @@ def add_args(cls, parser): @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: """ Turn a list of lcm packages into a set of .lcm files, where each file contains the definition for exactly one LCM type (struct or enum). @@ -36,13 +35,13 @@ def create_files( if not args.lcmout: return {} - file_map = {} # type: T.Dict[str, T.Union[str, bytes]] + file_map: T.Dict[str, T.Union[str, bytes]] = {} for package in packages: for definition in package.type_definitions.values(): output_filename = os.path.join( - args.lcmout_path, package.name, "{}.lcm".format(definition.name) + args.lcmout_path, package.name, f"{definition.name}.lcm" ) - output_contents = "package {};\n\n{}".format(package.name, str(definition)) + output_contents = f"package {package.name};\n\n{str(definition)}" file_map[output_filename] = output_contents return file_map diff --git a/third_party/skymarshal/skymarshal/emit_proto.py b/third_party/skymarshal/skymarshal/emit_proto.py index 8b7f8e1ff..80dc299f2 100644 --- a/third_party/skymarshal/skymarshal/emit_proto.py +++ b/third_party/skymarshal/skymarshal/emit_proto.py @@ -1,14 +1,14 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs +# aclint: py3 """Generate protocol buffers definition files.""" -from __future__ import absolute_import, print_function + +from __future__ import annotations import argparse # typing # pylint: disable=unused-import import json import os import typing as T -from six import next # pylint: disable=redefined-builtin from skymarshal import syntax_tree # pylint: disable=unused-import from skymarshal.common_util import camelcase_to_snakecase, snakecase_to_camelcase from skymarshal.emit_helpers import TemplateRenderer @@ -18,7 +18,7 @@ # pylint: disable=too-many-instance-attributes -class PrimitiveType(object): +class PrimitiveType: """A primitive type object that supports templating properties""" def __init__(self, lcm_decl, lcm_storage, proto_decl, proto_storage, short_int_warning): @@ -30,14 +30,14 @@ def __init__(self, lcm_decl, lcm_storage, proto_decl, proto_storage, short_int_w def convert_pb_to_lcm(self, expression): if self.short_int_warning: - return "static_cast<{}>({})".format(self.lcm_storage, expression) + return f"static_cast<{self.lcm_storage}>({expression})" return expression def single_lcm_to_pb(self, field_name, in_expression): - return "out->set_{}({})".format(field_name, in_expression) + return f"out->set_{field_name}({in_expression})" def add_lcm_to_pb(self, field_name, in_expression): - return "out->add_{}({})".format(field_name, in_expression) + return f"out->add_{field_name}({in_expression})" @property def default_lcm_value(self): @@ -70,7 +70,7 @@ def default_lcm_value(self): } -class EnumCase(object): +class EnumCase: """A template-friendly wrapper object for LCM #protobuf Enum Cases""" def __init__(self, int_value, name): @@ -78,7 +78,7 @@ def __init__(self, int_value, name): self.name = name -class EnumType(object): +class EnumType: """A template-friendly wrapper object for LCM #protobuf Enums""" def __init__(self, package_name, enum, args): @@ -123,9 +123,9 @@ def __init__(self, package_name, enum, args): self.reserved_ids = ", ".join(str(field_id) for field_id in sorted(enum.reserved_ids)) # names for templating - self.definition_name = "{}.{}".format(package_name, enum.name) + self.definition_name = f"{package_name}.{enum.name}" self.proto_package = package_name - self.proto_java_package = "com.skydio.pbtypes.{}".format(package_name) + self.proto_java_package = f"com.skydio.pbtypes.{package_name}" self.proto_java_outer_classname = "{}Proto".format( snakecase_to_camelcase(camelcase_to_snakecase(proto_filename_base)) ) @@ -135,19 +135,19 @@ def __init__(self, package_name, enum, args): package_name, self.proto_message_name, self.proto_enum_name ) - self.lcm_cpp_type = "::{}::{}".format(package_name, enum.name) - self.proto_cpp_container_type = "::{}::{}".format(package_name, self.proto_message_name) - self.proto_cpp_type = "{}::{}".format(self.proto_cpp_container_type, self.proto_enum_name) + self.lcm_cpp_type = f"::{package_name}::{enum.name}" + self.proto_cpp_container_type = f"::{package_name}::{self.proto_message_name}" + self.proto_cpp_type = f"{self.proto_cpp_container_type}::{self.proto_enum_name}" # filenames for generated converter sources - self.proto_filename = "{}/pbtypes/{}.proto".format(package_name, proto_filename_base) + self.proto_filename = f"{package_name}/pbtypes/{proto_filename_base}.proto" # include paths self.proto_import_path = os.path.join(self.args.proto_import_path, self.proto_filename) - self.protogen_header = "pbtypes/gen/{}/{}.pb.h".format(package_name, proto_filename_base) + self.protogen_header = f"pbtypes/gen/{package_name}/{proto_filename_base}.pb.h" def convert_pb_to_lcm(self, expression): - return "{}::from_int({})".format(self.lcm_cpp_type, expression) + return f"{self.lcm_cpp_type}::from_int({expression})" def single_lcm_to_pb(self, field_name, in_expression): return "out->set_{}(static_cast<{}>({}.int_value()))".format( @@ -165,10 +165,10 @@ def add_lcm_to_pb(self, field_name, in_expression): @property def default_lcm_value(self): - return "{}{{}}".format(self.lcm_cpp_type) + return f"{self.lcm_cpp_type}{{}}" -class StructField(object): +class StructField: """A template-friendly wrapper object for LCM #protobuf Struct Fields""" def __init__(self, parent, member, type_map, args): @@ -202,8 +202,8 @@ def proto_type_declaration(self): elif self.referenced_type: name = self.referenced_type.proto_reference_name else: - raise KeyError("Missing referenced type: {}".format(self.type_ref)) - return "repeated {}".format(name) if self.repeated else name + raise KeyError(f"Missing referenced type: {self.type_ref}") + return f"repeated {name}" if self.repeated else name def get_type(self): if self.type_ref.package_name is None: @@ -212,7 +212,7 @@ def get_type(self): @property def referenced_type(self): - full_type_name = "{}.{}".format(self.type_ref.package_name, self.type_ref.name) + full_type_name = f"{self.type_ref.package_name}.{self.type_ref.name}" return self._type_map.get(full_type_name) @property @@ -222,25 +222,25 @@ def pb_to_lcm(self): array_field = self.parent.array_for_dim(self.lcm_name) if array_field.type_ref.name == "byte": # bytes is a string, so call size on the string - return "out.{} = in.{}().size();".format(self.lcm_name, array_field.proto_name) + return f"out.{self.lcm_name} = in.{array_field.proto_name}().size();" # normal fields have a _size() method - return "out.{} = in.{}_size();".format(self.lcm_name, array_field.proto_name) + return f"out.{self.lcm_name} = in.{array_field.proto_name}_size();" field_type = self.get_type() if not self.repeated: - in_expression = field_type.convert_pb_to_lcm("in.{}()".format(self.proto_name)) - return "out.{} = {};".format(self.lcm_name, in_expression) + in_expression = field_type.convert_pb_to_lcm(f"in.{self.proto_name}()") + return f"out.{self.lcm_name} = {in_expression};" - in_expression = field_type.convert_pb_to_lcm("in.{}(i)".format(self.proto_name)) + in_expression = field_type.convert_pb_to_lcm(f"in.{self.proto_name}(i)") if self.type_ref.name == "byte": # bytes need special logic to convert between string and vector - in_expression = "in.{}()".format(self.proto_name) + in_expression = f"in.{self.proto_name}()" return "out.{} = std::vector({in_expr}.begin(), {in_expr}.end());".format( self.lcm_name, in_expr=in_expression ) dim = self.struct.member_map[self.lcm_name].dims[0] - var_max_expression = "in.{}_size()".format(self.proto_name) + var_max_expression = f"in.{self.proto_name}_size()" if dim.size_int is None: # dynamic array is easy because we pass along the size return "for (int i = 0; i < {}; i++) {{\n out.{}.push_back({});\n}}".format( @@ -267,7 +267,7 @@ def pb_to_lcm(self): # NOTE(jeff): std::array will default-initialize to empty string return main_loop - return "{}\n{}".format(main_loop, fill_loop) + return f"{main_loop}\n{fill_loop}" except Exception as e: # If there's an AttributeError in this function, jinja will silently "swallow" it and # put an empty string in the generated code, which can make it very hard to figure @@ -279,15 +279,15 @@ def pb_to_lcm(self): def lcm_to_pb(self): if self.field_id is None: array_field = self.parent.array_for_dim(self.lcm_name) - return "// skip {} (size of {})".format(self.lcm_name, array_field.lcm_name) + return f"// skip {self.lcm_name} (size of {array_field.lcm_name})" field_type = self.get_type() if not self.repeated: - in_expression = "in.{}".format(self.lcm_name) - return "{};".format(field_type.single_lcm_to_pb(self.proto_name, in_expression)) + in_expression = f"in.{self.lcm_name}" + return f"{field_type.single_lcm_to_pb(self.proto_name, in_expression)};" dim = self.struct.member_map[self.lcm_name].dims[0] if self.type_ref.name == "byte": - in_expression = "in.{}".format(self.lcm_name) + in_expression = f"in.{self.lcm_name}" if dim.auto_member: return "out->set_{}(std::string({in_expr}.begin(), {in_expr}.end()));".format( self.proto_name, @@ -299,14 +299,14 @@ def lcm_to_pb(self): dim=dim.size_str, ) - max_expression = "in.{}".format(dim.size_str) if dim.size_int is None else dim.size_int + max_expression = f"in.{dim.size_str}" if dim.size_int is None else dim.size_int if dim.auto_member: - max_expression = "in.{}.size()".format(self.lcm_name) - add_statement = field_type.add_lcm_to_pb(self.proto_name, "in.{}[i]".format(self.lcm_name)) - return "for (int i = 0; i < {}; i++) {{\n {};\n}}".format(max_expression, add_statement) + max_expression = f"in.{self.lcm_name}.size()" + add_statement = field_type.add_lcm_to_pb(self.proto_name, f"in.{self.lcm_name}[i]") + return f"for (int i = 0; i < {max_expression}; i++) {{\n {add_statement};\n}}" -class StructType(object): +class StructType: """A template-friendly wrapper object for LCM #protobuf Structs""" def __init__(self, package, struct, type_map, args): @@ -339,34 +339,34 @@ def __init__(self, package, struct, type_map, args): self.proto_fields = [field for field in self.fields if field.field_id is not None] # names for templating - self.definition_name = "{}.{}".format(package.name, struct.name) + self.definition_name = f"{package.name}.{struct.name}" self.proto_package = package.name - self.proto_java_package = "com.skydio.pbtypes.{}".format(package.name) + self.proto_java_package = f"com.skydio.pbtypes.{package.name}" self.proto_java_outer_classname = "{}Proto".format( snakecase_to_camelcase(camelcase_to_snakecase(proto_filename_base)) ) self.proto_message_name = proto_typename - self.proto_reference_name = "{}.{}".format(package.name, self.proto_message_name) - self.proto_cpp_type = "::{}::{}".format(package.name, self.proto_message_name) - self.lcm_cpp_type = "::{}::{}".format(package.name, struct.name) + self.proto_reference_name = f"{package.name}.{self.proto_message_name}" + self.proto_cpp_type = f"::{package.name}::{self.proto_message_name}" + self.lcm_cpp_type = f"::{package.name}::{struct.name}" # filenames for generated files sources - self.proto_filename = "{}/pbtypes/{}.proto".format(package.name, proto_filename_base) - self.protolcm_filename_h = "{}/{}.h".format(package.name, proto_filename_base) - self.protolcm_filename_cc = "{}/{}.cc".format(package.name, proto_filename_base) + self.proto_filename = f"{package.name}/pbtypes/{proto_filename_base}.proto" + self.protolcm_filename_h = f"{package.name}/{proto_filename_base}.h" + self.protolcm_filename_cc = f"{package.name}/{proto_filename_base}.cc" # include paths self.proto_import_path = os.path.join(self.args.proto_import_path, self.proto_filename) # TODO(jeff): don't hard-code these include prefixes - self.lcmgen_header = "lcmtypes/{}/{}.hpp".format(package.name, struct.name) - self.protogen_header = "pbtypes/gen/{}/{}.pb.h".format(package.name, proto_filename_base) + self.lcmgen_header = f"lcmtypes/{package.name}/{struct.name}.hpp" + self.protogen_header = f"pbtypes/gen/{package.name}/{proto_filename_base}.pb.h" self.protolcm_include_path = os.path.join( self.args.protolcm_include_path, self.protolcm_filename_h ) @property def default_lcm_value(self): - return "{}{{}}".format(self.lcm_cpp_type) + return f"{self.lcm_cpp_type}{{}}" def array_for_dim(self, dim_name): return next( @@ -378,11 +378,11 @@ def array_for_dim(self, dim_name): @property def referenced_types(self): return sorted( - set( + { field.referenced_type for field in self.proto_fields if field.referenced_type and field.referenced_type != self - ), + }, key=lambda x: x.definition_name, ) @@ -399,13 +399,13 @@ def referenced_structs(self): ] def convert_pb_to_lcm(self, expression): - return "PbToLcm({})".format(expression) + return f"PbToLcm({expression})" def single_lcm_to_pb(self, field_name, in_expression): - return "LcmToPb({}, out->mutable_{}())".format(in_expression, field_name) + return f"LcmToPb({in_expression}, out->mutable_{field_name}())" def add_lcm_to_pb(self, field_name, in_expression): - return "LcmToPb({}, out->add_{}())".format(in_expression, field_name) + return f"LcmToPb({in_expression}, out->add_{field_name}())" ProtoJsonInfo = T.TypedDict( @@ -423,9 +423,8 @@ def add_lcm_to_pb(self, field_name, in_expression): ) -class ForeignType(object): - def __init__(self, enum_info): - # type: (ProtoJsonInfo) -> None +class ForeignType: + def __init__(self, enum_info: ProtoJsonInfo) -> None: self.proto_reference_name = enum_info["reference_name"] self.pb_to_lcm_template = enum_info["pb_to_lcm_template"] self.proto_import_path = enum_info["proto_import_path"] @@ -450,8 +449,7 @@ def add_lcm_to_pb(self, field_name, in_expression): ) @staticmethod - def get_proto_info(type_map): - # type: (T.Dict[str, ProtoType]) -> T.Dict[str, ProtoJsonInfo] + def get_proto_info(type_map: T.Dict[str, ProtoType]) -> T.Dict[str, ProtoJsonInfo]: data = {} for type_wrapper in type_map.values(): data[type_wrapper.definition_name] = ProtoJsonInfo( # pylint: disable=not-callable @@ -477,10 +475,9 @@ def get_proto_info(type_map): def make_proto_type_map( - output_packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace -): - # type: (...) -> T.Dict[str, ProtoType] + output_packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, +) -> T.Dict[str, ProtoType]: """ Parses the provided packages, outputing all protobuf types that should be generated from them. @@ -495,12 +492,12 @@ def make_proto_type_map( """ # All types from output_packages - type_map = {} # type: T.Dict[str, ProtoType] + type_map: T.Dict[str, ProtoType] = {} # All types from output_packages or referenced_packages - all_referenceable_types = {} # type: T.Dict[str, MaybeForeignType] + all_referenceable_types: T.Dict[str, MaybeForeignType] = {} for path in args.proto_deps_info or []: - with open(path, "r") as f: + with open(path) as f: data = T.cast(T.Dict[str, ProtoJsonInfo], json.load(f)) for full_name, enum_info in data.items(): all_referenceable_types[full_name] = ForeignType(enum_info) @@ -514,7 +511,7 @@ def make_proto_type_map( enum_type = EnumType(package.name, enum, args) if enum_type.definition_name in type_map: raise ValueError( - "Two definitions of the same type: {}".format(enum_type.definition_name) + f"Two definitions of the same type: {enum_type.definition_name}" ) all_referenceable_types[enum_type.definition_name] = enum_type type_map[enum_type.definition_name] = enum_type @@ -524,7 +521,7 @@ def make_proto_type_map( struct_type = StructType(package, struct, all_referenceable_types, args) if struct_type.definition_name in type_map: raise ValueError( - "Two definitions of the same type: {}".format(struct_type.definition_name) + f"Two definitions of the same type: {struct_type.definition_name}" ) all_referenceable_types[struct_type.definition_name] = struct_type type_map[struct_type.definition_name] = struct_type @@ -534,8 +531,7 @@ def make_proto_type_map( class SkymarshalProto(SkymarshalLanguage): @classmethod - def add_args(cls, parser): - # type: (argparse.ArgumentParser) -> None + def add_args(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument("--proto", action="store_true", help="generate protobuf definitions") parser.add_argument( "--proto-path", default="", help="Location of the .proto file hierarchy" @@ -553,8 +549,9 @@ def add_args(cls, parser): parser.add_argument("--proto-info-out", help="output location for json info") @classmethod - def create_files(cls, packages, args): - # type: (T.Iterable[syntax_tree.Package], argparse.Namespace) -> T.Dict[str, T.Union[str, bytes]] + def create_files( + cls, packages: T.Iterable[syntax_tree.Package], args: argparse.Namespace + ) -> T.Dict[str, T.Union[str, bytes]]: """Turn a list of lcm packages into a bunch of .proto files @@ -571,7 +568,7 @@ def create_files(cls, packages, args): render = TemplateRenderer(os.path.dirname(__file__)) type_map = make_proto_type_map(packages, args) - file_map = {} # type: T.Dict[str, T.Union[str, bytes]] + file_map: T.Dict[str, T.Union[str, bytes]] = {} for type_wrapper in type_map.values(): # set paths proto_filename = os.path.join(args.proto_path, type_wrapper.proto_filename) @@ -585,7 +582,7 @@ def create_files(cls, packages, args): "proto_struct.proto.template", struct_type=type_wrapper ) else: - raise TypeError("Unknown wrapper: {}".format(type_wrapper)) + raise TypeError(f"Unknown wrapper: {type_wrapper}") if args.proto_info_out: file_map[args.proto_info_out] = json.dumps( @@ -597,8 +594,7 @@ def create_files(cls, packages, args): class SkymarshalProtoLCM(SkymarshalLanguage): @classmethod - def add_args(cls, parser): - # type: (argparse.ArgumentParser) -> None + def add_args(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument( "--protolcm", action="store_true", help="generate protobuf-lcm converters" ) @@ -614,10 +610,9 @@ def add_args(cls, parser): @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: """Turn a list of lcm packages into a bunch of .h converter files skymarshal can be invoked in two ways, the first is invoking it with the full context of the @@ -651,6 +646,6 @@ def create_files( file_map[filename_h] = render("protolcm.h.template", struct_type=type_wrapper) file_map[filename_cc] = render("protolcm.cc.template", struct_type=type_wrapper) else: - raise TypeError("Unknown wrapper: {}".format(type_wrapper)) + raise TypeError(f"Unknown wrapper: {type_wrapper}") return file_map diff --git a/third_party/skymarshal/skymarshal/emit_python.py b/third_party/skymarshal/skymarshal/emit_python.py index 1b83709b2..e69fb977b 100644 --- a/third_party/skymarshal/skymarshal/emit_python.py +++ b/third_party/skymarshal/skymarshal/emit_python.py @@ -1,5 +1,6 @@ -# aclint: py2 py3 -from __future__ import absolute_import +# aclint: py3 + +from __future__ import annotations import argparse import copy @@ -45,8 +46,7 @@ TYPE_MAP["ufixed64"] = TYPE_MAP["uint64_t"] -def module_name_for_lcmtype_name(lcmtype_name): - # type: (str) -> str +def module_name_for_lcmtype_name(lcmtype_name: str) -> str: # NOTE(matt): we add an underscore to the file name to disambiguate between type and module # when importing `from lcmtypes.package import name` return "_" + lcmtype_name @@ -55,13 +55,13 @@ def module_name_for_lcmtype_name(lcmtype_name): class PythonClass(StructBuilder): """Helper to construct a python class definition.""" - def __init__(self, package, struct, args): - # type: (syntax_tree.Package, syntax_tree.Struct, argparse.Namespace) -> None - super(PythonClass, self).__init__(package, struct, args) - self.cached_unpackers = dict() # type: T.Dict[str, T.Tuple[int, str]] + def __init__( + self, package: syntax_tree.Package, struct: syntax_tree.Struct, args: argparse.Namespace + ) -> None: + super().__init__(package, struct, args) + self.cached_unpackers: T.Dict[str, T.Tuple[int, str]] = dict() - def get_type_info(self, type_ref): - # type: (syntax_tree.TypeRef) -> TypeInfo + def get_type_info(self, type_ref: syntax_tree.TypeRef) -> TypeInfo: info = TYPE_MAP.get(type_ref.name) if not info: # Create a default info for this non-primitive type. @@ -70,8 +70,7 @@ def get_type_info(self, type_ref): info = TypeInfo(type_ref.name, constructor, None, None, 0) return info - def get_imports(self): - # type: () -> T.List[T.Tuple[str, str]] + def get_imports(self) -> T.List[T.Tuple[str, str]]: imports = set() for member in self.members: if not member.type_ref.is_primitive_type(): @@ -81,9 +80,9 @@ def get_imports(self): if package == self.package.name and module == self.module_name: continue if self.args.package_prefix: - result = "{}.{}.{}".format(self.args.package_prefix, package, module) + result = f"{self.args.package_prefix}.{package}.{module}" else: - result = "{}.{}".format(package, module) + result = f"{package}.{module}" # The {}.{} form worked for eigen_lcm packages until we removed subpackages from # all the __init__.pys. It's now necessary to be explicit: if package == "eigen_lcm": @@ -92,18 +91,18 @@ def get_imports(self): imports.add((result, result.split(".")[-1][1:])) return sorted(imports) - def member_initializers(self, optional=False): - # type: (bool) -> T.Iterable[T.Tuple[syntax_tree.Member, T.Optional[str], str, str]] + def member_initializers( + self, optional: bool = False + ) -> T.Iterable[T.Tuple[syntax_tree.Member, T.Optional[str], str, str]]: for member in self.members: kwarg_default = self.member_kwarg_default(member) initializer = self.member_initializer(member, 0) type_hint = self.type_hint(member) if optional and kwarg_default is None: - type_hint = "T.Optional[{}]".format(type_hint) + type_hint = f"T.Optional[{type_hint}]" yield member, kwarg_default, initializer, type_hint - def member_kwarg_default(self, member): - # type: (syntax_tree.Member) -> T.Optional[T.Any] + def member_kwarg_default(self, member: syntax_tree.Member) -> T.Optional[T.Any]: if member.ndim != 0: return None type_info = TYPE_MAP.get(member.type_ref.name) @@ -111,8 +110,7 @@ def member_kwarg_default(self, member): return type_info.initializer return None - def member_initializer(self, member, depth): - # type: (syntax_tree.Member, int) -> str + def member_initializer(self, member: syntax_tree.Member, depth: int) -> str: if depth == member.ndim: return self.get_type_info(member.type_ref).initializer @@ -126,10 +124,9 @@ def member_initializer(self, member, depth): if dim.dynamic: return "[]" inner = self.member_initializer(member, depth + 1) - return "[ {} for dim{} in range({}) ]".format(inner, depth, dim.size_str) + return f"[ {inner} for dim{depth} in range({dim.size_str}) ]" - def type_hint(self, member, depth=0): - # type: (syntax_tree.Member, int) -> str + def type_hint(self, member: syntax_tree.Member, depth: int = 0) -> str: """ Returns a mypy type hint for the provided member """ @@ -152,10 +149,11 @@ def type_hint(self, member, depth=0): # can be either tuples or lists. array_type = "T.Sequence" - return "{}[{}]".format(array_type, self.type_hint(member, depth + 1)) + return f"{array_type}[{self.type_hint(member, depth + 1)}]" - def encode_member(self, code, member, accessor, indent): - # type: (Code, syntax_tree.Member, str, int) -> None + def encode_member( + self, code: Code, member: syntax_tree.Member, accessor: str, indent: int + ) -> None: if member.type_ref.is_non_string_primitive_type(): info = self.get_type_info(member.type_ref) assert info.pack_format is not None @@ -179,8 +177,15 @@ def encode_member(self, code, member, accessor, indent): code(indent + 1, "assert %s.%s([]) == %s.%s([])", accessor, ghr, name, ghr) code(indent, "%s._encode_one(buf)", accessor) - def decode_list(self, code, member, accessor, indent, dim, is_first): - # type: (Code, syntax_tree.Member, str, int, syntax_tree.ArrayDim, bool) -> None + def decode_list( + self, + code: Code, + member: syntax_tree.Member, + accessor: str, + indent: int, + dim: syntax_tree.ArrayDim, + is_first: bool, + ) -> None: suffix = "" if not is_first: suffix = ")" @@ -191,10 +196,10 @@ def decode_list(self, code, member, accessor, indent, dim, is_first): if not dim.dynamic: length_str = dim.size_str elif dim.auto_member: - length_str = "v_{}".format(dim.size_str) + length_str = f"v_{dim.size_str}" self.decode_member(code, dim.auto_member, length_str + " = ", indent, "") else: - length_str = "self.{}".format(dim.size_str) + length_str = f"self.{dim.size_str}" code(indent, "%sbuf.read(%s)%s", accessor, length_str, suffix) elif member.type_ref.name == "boolean": if not dim.dynamic: @@ -215,10 +220,10 @@ def decode_list(self, code, member, accessor, indent, dim, is_first): ) else: if dim.auto_member: - length_str = "v_{}".format(dim.size_str) + length_str = f"v_{dim.size_str}" self.decode_member(code, dim.auto_member, length_str + " = ", indent, "") else: - length_str = "self.{}".format(dim.size_str) + length_str = f"self.{dim.size_str}" code( indent, "%slist(map(bool, struct.unpack('>%%d%c' %% %s, buf.read(%s))))%s", @@ -247,10 +252,10 @@ def decode_list(self, code, member, accessor, indent, dim, is_first): ) else: if dim.auto_member: - length_str = "v_{}".format(dim.size_str) + length_str = f"v_{dim.size_str}" self.decode_member(code, dim.auto_member, length_str + " = ", indent, "") else: - length_str = "self.{}".format(dim.size_str) + length_str = f"self.{dim.size_str}" assert info.size is not None size_mult = (" * %d" % info.size) if info.size > 1 else "" code( @@ -266,8 +271,7 @@ def decode_list(self, code, member, accessor, indent, dim, is_first): else: assert 0 - def pack_members(self, code, members, indent): - # type: (Code, T.List[syntax_tree.Member], int) -> None + def pack_members(self, code: Code, members: T.List[syntax_tree.Member], indent: int) -> None: """encode multiple members with a single call to struct.pack().""" if not members: return @@ -277,17 +281,16 @@ def pack_members(self, code, members, indent): assert info.struct_format is not None format_str += info.struct_format packer = self.get_or_create_pack_struct(format_str) - code.start(indent, "buf.write({}.pack(".format(packer)) + code.start(indent, f"buf.write({packer}.pack(") code.add(", ".join("self." + member.name for member in members)) code.end("))") # Clear the list members[:] = [] - def encode_members(self, indent): - # type: (int) -> T.Text + def encode_members(self, indent: int) -> T.Text: if not self.members: - return u" " * indent + u"pass" + return " " * indent + "pass" # Optimization: group adjacent non-string primitives together. # This allows us to encode them with a single call to struct.pack(...) @@ -309,7 +312,7 @@ def encode_members(self, indent): i = 0 for dim in member.dims[:-1]: - accessor += "[i{}]".format(i) + accessor += f"[i{i}]" if dim.dynamic: code(indent + i, "for i%d in range(self.%s):", i, dim.size_str) else: @@ -329,14 +332,20 @@ def encode_members(self, indent): code(indent + i, "for i%d in range(self.%s):", i, last_dim.size_str) else: code(indent + i, "for i%d in range(%s):", i, last_dim.size_str) - accessor += "[i{}]".format(i) + accessor += f"[i{i}]" self.encode_member(code, member, accessor, indent + i + 1) self.pack_members(code, grouping, indent) return code.getvalue().rstrip() - def encode_list(self, code, member, accessor, indent, dim): - # type: (Code, syntax_tree.Member, str, int, syntax_tree.ArrayDim) -> None + def encode_list( + self, + code: Code, + member: syntax_tree.Member, + accessor: str, + indent: int, + dim: syntax_tree.ArrayDim, + ) -> None: if member.type_ref.name == "byte": assert dim.size_str is not None if dim.auto_member: @@ -381,8 +390,9 @@ def encode_list(self, code, member, accessor, indent, dim): else: assert 0 - def decode_member(self, code, member, accessor, indent, suffix): - # type: (Code, syntax_tree.Member, str, int, T.Text) -> None + def decode_member( + self, code: Code, member: syntax_tree.Member, accessor: str, indent: int, suffix: T.Text + ) -> None: if member.type_ref.name == "boolean": unpacker = self.get_or_create_pack_struct("b") code(indent, "%sbool(%s.unpack(buf.read(1))[0])%s", accessor, unpacker, suffix) @@ -413,8 +423,7 @@ def decode_member(self, code, member, accessor, indent, suffix): name = member.type_ref.name code(indent, "%s%s._decode_one(buf)%s", accessor, name, suffix) - def unpack_members(self, code, members, indent): - # type: (Code, T.List[syntax_tree.Member], int) -> None + def unpack_members(self, code: Code, members: T.List[syntax_tree.Member], indent: int) -> None: """decode multiple members with a single call to struct.unpack().""" if not members: return @@ -433,14 +442,13 @@ def unpack_members(self, code, members, indent): code.start(indent, "") lhs = ", ".join("self." + member.name for member in members) code.add(lhs) - code.add(" = {}.unpack(".format(unpacker)) + code.add(f" = {unpacker}.unpack(") code.end("buf.read(%d))%s", size, suffix) # Clear the list members[:] = [] - def decode_members(self, indent): - # type: (int) -> T.Text + def decode_members(self, indent: int) -> T.Text: # pylint: disable=too-many-statements, too-many-branches code = Code() @@ -479,7 +487,7 @@ def decode_members(self, indent): code(indent + i, "for i%d in range(%s):", i, dim.size_str) if 0 < i < member.ndim - 1: - accessor += "[i{}]".format(i - 1) + accessor += f"[i{i - 1}]" i += 1 # last dimension @@ -511,7 +519,7 @@ def decode_members(self, indent): code(indent, "%s = []", accessor) else: code(indent + i, "%s.append ([])", accessor) - accessor += "[i{}]".format(i - 1) + accessor += f"[i{i - 1}]" if last_dim.auto_member: code(indent + i, "for i%d in range(v_%s):", i, last_dim.size_str) @@ -527,40 +535,33 @@ def decode_members(self, indent): return code.getvalue().rstrip() @property - def module_name(self): - # type: () -> str + def module_name(self) -> str: return "_" + self.name @property - def filename(self): - # type: () -> str + def filename(self) -> str: return os.path.join(self.package.name, self.module_name) + ".py" @property - def fullpath(self): - # type: () -> str + def fullpath(self) -> str: if self.args.python_path: return os.path.join(self.args.python_path or "", self.filename) else: return self.filename @property - def fully_qualified_name(self): - # type: () -> str + def fully_qualified_name(self) -> str: return self.struct.type_ref.name - def has_complex_members(self): - # type: () -> bool + def has_complex_members(self) -> bool: return bool(list(self.complex_members())) - def complex_members(self): - # type: () -> T.Iterator[T.Tuple[str, str]] + def complex_members(self) -> T.Iterator[T.Tuple[str, str]]: for member in self.members: if not member.type_ref.is_primitive_type(): yield member.type_ref.name, member.name - def get_or_create_pack_struct(self, format_str): - # type: (str) -> str + def get_or_create_pack_struct(self, format_str: str) -> str: """ Get a string referring to a class variable capturing a struct.Struct instance for the format_str. @@ -572,52 +573,45 @@ def get_or_create_pack_struct(self, format_str): if format_str not in self.cached_unpackers: self.cached_unpackers[format_str] = ( len(self.cached_unpackers), - 'struct.Struct("{}")'.format(format_str), + f'struct.Struct("{format_str}")', ) index, _ = self.cached_unpackers[format_str] - return "{}._CACHED_STRUCT_{}".format(self.name, index) + return f"{self.name}._CACHED_STRUCT_{index}" - def cached_structs_block(self, indent): - # type: (int) -> T.Text + def cached_structs_block(self, indent: int) -> T.Text: """Return a code block to define the struct.Struct instances as class members.""" code = Code() for index, struct_str in sorted(self.cached_unpackers.values()): - code(indent, "_CACHED_STRUCT_{} = {}".format(index, struct_str)) + code(indent, f"_CACHED_STRUCT_{index} = {struct_str}") return code.getvalue().rstrip() @property - def hashable(self): - # type: () -> bool + def hashable(self) -> bool: return any(notation.name == "#hashable" for notation in self.struct.notations) class PyEnum(EnumBuilder): # TODO(nikhilm): Add unpacker support for enums, which should be pretty easy. - def decode_value(self): - # type: () -> str + def decode_value(self) -> str: info = TYPE_MAP[self.storage_type.name] - return "struct.unpack('{}', buf.read({}))[0]".format(info.pack_format, info.size) + return f"struct.unpack('{info.pack_format}', buf.read({info.size}))[0]" - def encode_value(self): - # type: () -> str + def encode_value(self) -> str: info = TYPE_MAP[self.storage_type.name] - return "buf.write(struct.pack('{}', self.value))".format(info.pack_format) + return f"buf.write(struct.pack('{info.pack_format}', self.value))" @property - def module_name(self): - # type: () -> str + def module_name(self) -> str: return "_" + self.name @property - def filename(self): - # type: () -> str + def filename(self) -> str: return os.path.join(self.package.name, self.module_name) + ".py" @property - def fullpath(self): - # type: () -> str + def fullpath(self) -> str: if self.args.python_path: return os.path.join(self.args.python_path or "", self.filename) else: @@ -626,8 +620,7 @@ def fullpath(self): class SkymarshalPython(SkymarshalLanguage): @classmethod - def add_args(cls, parser): - # type: (argparse.ArgumentParser) -> None + def add_args(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument("--python", action="store_true", help="generate bindings for python") parser.add_argument( "--python-package-prefix", default=None, help="override package-prefix for python only" @@ -654,15 +647,14 @@ def add_args(cls, parser): @classmethod def create_module( cls, - package, # type: syntax_tree.Package - args, # type: argparse.Namespace - file_map, # type: T.Dict[str, T.Union[str, bytes]] - ): - # type: (...) -> T.List[T.Tuple[str, str]] + package: syntax_tree.Package, + args: argparse.Namespace, + file_map: T.Dict[str, T.Union[str, bytes]], + ) -> T.List[T.Tuple[str, str]]: module_items = [] for struct in package.struct_definitions: - pyclass = PythonClass(package, struct, args) # type: T.Union[PythonClass, PyEnum] + pyclass: T.Union[PythonClass, PyEnum] = PythonClass(package, struct, args) file_map[pyclass.fullpath] = render( "python_struct_default_wrapper.py.template", lcmtype=pyclass, @@ -681,10 +673,9 @@ def create_module( @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: """Turn a list of lcm packages into python bindings for each struct. @param packages: the list of syntax_tree.Package objects @@ -699,7 +690,7 @@ def create_files( args = copy.copy(args) args.package_prefix = args.python_package_prefix - file_map = {} # type: T.Dict[str, T.Union[str, bytes]] + file_map: T.Dict[str, T.Union[str, bytes]] = {} for package in packages: # Create a list of (module_name, lcmtype_name) pairs. module_items = cls.create_module(package, args, file_map) diff --git a/third_party/skymarshal/skymarshal/emit_typescript.py b/third_party/skymarshal/skymarshal/emit_typescript.py index 0eaec227f..4bad4e9ff 100644 --- a/third_party/skymarshal/skymarshal/emit_typescript.py +++ b/third_party/skymarshal/skymarshal/emit_typescript.py @@ -1,5 +1,5 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs +# aclint: py3 """LCM definition files for TypeScript""" # TODO(danny): I think this whole module could use a refactor. This code uses a combination of @@ -7,7 +7,8 @@ # a AST pretty-printer might be a cleaner and clearer way of doing code generation. There's not a # lot of precendent for that, though. -from __future__ import absolute_import + +from __future__ import annotations import argparse # pylint: disable=unused-import import os @@ -164,14 +165,12 @@ class TsBase(BaseBuilder): @property - def bare_base_path(self): - # type: () -> str + def bare_base_path(self) -> str: """package_name/filename""" return os.path.join(self.package.name, self.name) @property - def bare_import_path(self): - # type: () -> str + def bare_import_path(self) -> str: """ import_dir/package_name/filename @@ -180,8 +179,7 @@ def bare_import_path(self): return os.path.join(self.args.typescript_import_path, self.bare_base_path) @property - def generation_path(self): - # type: () -> str + def generation_path(self) -> str: """ generation_dir/package_name/filename.ts @@ -190,8 +188,7 @@ def generation_path(self): return os.path.join(self.args.typescript_path, self.bare_base_path + ".ts") @property - def comment(self): - # type: () -> str + def comment(self) -> str: """ class comment at top of definition taken directly from emit_cpp.py @@ -213,13 +210,12 @@ class comment at top of definition class TsEnum(EnumBuilder, TsBase): - def __init__(self, package, enum, args): - # type: (syntax_tree.Package, syntax_tree.Enum, T.Any) -> None - super(TsEnum, self).__init__(package, enum, args) + def __init__(self, package: syntax_tree.Package, enum: syntax_tree.Enum, args: T.Any) -> None: + super().__init__(package, enum, args) self.storage_type = TsTypeRef(self.storage_type) -class TsInclude(object): +class TsInclude: def __init__(self, member=None, is_primitive=False, prefix=None): self.member = member self.is_primitive = is_primitive @@ -228,7 +224,7 @@ def __init__(self, member=None, is_primitive=False, prefix=None): @property def absolute_path(self): if self.is_primitive: - return "{}/types".format(self.prefix) + return f"{self.prefix}/types" return os.path.join( self.prefix, self.member.type_ref.package_name, self.member.type_ref.name @@ -236,7 +232,7 @@ def absolute_path(self): @property def directive(self): - return '{{ {} }} from "{}"'.format(self.member.type_ref.name, self.absolute_path) + return f'{{ {self.member.type_ref.name} }} from "{self.absolute_path}"' def __hash__(self): return hash(self.directive) @@ -248,8 +244,7 @@ def __repr__(self): return self.directive -def type_ref_to_reflection_dict(type_ref): - # type: (TsTypeRef) -> str +def type_ref_to_reflection_dict(type_ref: TsTypeRef) -> str: """Takes a type_ref and generates a JS dictionary that corresponds to a ReflectionType in TS. Does not handle array types, as TsMember has the information about if a member is an array, but the TsTypeRef is unaware. @@ -261,24 +256,23 @@ def type_ref_to_reflection_dict(type_ref): a string that corresponds to a ReflectionType in TS. """ if type_ref.is_primitive_type(): - return '{ kind: "primitive", type: %s, typeStr: "%s" }' % ( + return '{{ kind: "primitive", type: {}, typeStr: "{}" }}'.format( type_ref.reflection_constructor, type_ref.name, ) else: - return '{ kind: "struct", type: %s, typeStr: "%s" }' % ( + return '{{ kind: "struct", type: {}, typeStr: "{}" }}'.format( type_ref.reflection_constructor, type_ref.name, ) -class TsMember(object): +class TsMember: """ Passthrough type for Member that defines specifically needed extra methods """ - def __init__(self, member): - # type: (syntax_tree.Member) -> None + def __init__(self, member: syntax_tree.Member) -> None: self._member = member self.type_ref = TsTypeRef(member.type_ref) @@ -301,8 +295,7 @@ def has_auto_member(self): return self.ndim == 1 and self.dims[0].auto_member @property - def auto_member_type_ref(self): - # type: () -> T.Optional[TsTypeRef] + def auto_member_type_ref(self) -> T.Optional[TsTypeRef]: """ We need to know storage_size for auto_members, wrap """ @@ -311,13 +304,11 @@ def auto_member_type_ref(self): return None @property - def is_byte_array(self): - # type: () -> bool + def is_byte_array(self) -> bool: return self.ndim == 1 and self.type_ref.name == "byte" @property - def type_declaration(self): - # type: () -> str + def type_declaration(self) -> str: """ Canonical TS definition for a type Example: double[][] for a 2d array member of type double @@ -328,8 +319,7 @@ def type_declaration(self): return "{}{}".format(self.type_ref.name, "[]" * self.ndim) @property - def type_reflection_array_dims(self): - # type: () -> str + def type_reflection_array_dims(self) -> str: """Reflection metadata generation for array dimensions for ReflectionTypeArray Returns: @@ -348,8 +338,7 @@ def type_reflection_array_dims(self): return "[{}]".format(", ".join(dim_strings)) @property - def type_reflection_dict(self): - # type: () -> str + def type_reflection_dict(self) -> str: """Reflection metadata generation for this member, corresponds to a ReflectionType Returns: @@ -359,7 +348,7 @@ def type_reflection_dict(self): return '{ kind: "bytes", dims: %s, nDim: 1 }' % (self.type_reflection_array_dims) elif self.ndim > 0: - return '{ kind: "array", dims: %s, nDim: %s, inner: %s }' % ( + return '{{ kind: "array", dims: {}, nDim: {}, inner: {} }}'.format( self.type_reflection_array_dims, self.ndim, type_ref_to_reflection_dict(self.type_ref), @@ -395,10 +384,10 @@ def default_initializer(self): return "[]" elif self.type_ref.is_primitive_type(): return TYPE_INITALIZER_MAP[self.type_ref.name] - return "new {}()".format(self.type_ref.name) + return f"new {self.type_ref.name}()" -class TsTypeRef(object): +class TsTypeRef: """ Passthrough type for TypeRef that overrides some behavior """ @@ -407,19 +396,16 @@ def __init__(self, type_ref): self._type_ref = type_ref @property - def name(self): - # type: () -> str + def name(self) -> str: """Map the protobuf types to the standard types""" return PROTO_TYPE_MAP.get(self._type_ref.name, self._type_ref.name) @property - def storage_size(self): - # type: () -> T.Optional[int] + def storage_size(self) -> T.Optional[int]: return TYPE_ENCODED_SIZE_MAP.get(self.name, None) @property - def reflection_constructor(self): - # type: () -> str + def reflection_constructor(self) -> str: """Get the constructor function for a type in JS. Checks a primitive map, or just returns the struct/enum name. Array types are handled by TsMember, which calls this for the type contained in the array. @@ -440,7 +426,7 @@ def __setattr__(self, attr, val): class TsStruct(StructBuilder, TsBase): def __init__(self, package, struct, args): - super(TsStruct, self).__init__(package, struct, args) + super().__init__(package, struct, args) self.members = [TsMember(member) for member in self.members] self.constants = [TsMember(member) for member in self.constants] @@ -494,8 +480,7 @@ def include_list(self): class SkymarshalTypeScript(SkymarshalLanguage): @classmethod - def add_args(cls, parser): - # type: (argparse.ArgumentParser) -> None + def add_args(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument( "--typescript", action="store_true", help="generate typescript definitions" ) @@ -517,10 +502,9 @@ def add_args(cls, parser): @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: """Turn a list of lcm packages into a bunch of .ts files @param packages: the list of syntax_tree.Package objects diff --git a/third_party/skymarshal/skymarshal/language_plugin.py b/third_party/skymarshal/skymarshal/language_plugin.py index c10cf51c2..aa7ca2858 100644 --- a/third_party/skymarshal/skymarshal/language_plugin.py +++ b/third_party/skymarshal/skymarshal/language_plugin.py @@ -1,6 +1,7 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs -from __future__ import absolute_import +# aclint: py3 + +from __future__ import annotations import argparse # pylint: disable=unused-import import typing as T @@ -8,17 +9,15 @@ from skymarshal import syntax_tree # pylint: disable=unused-import -class SkymarshalLanguage(object): +class SkymarshalLanguage: @classmethod - def add_args(cls, parser): - # type: (argparse.ArgumentParser) -> None + def add_args(cls, parser: argparse.ArgumentParser) -> None: raise NotImplementedError() @classmethod def create_files( cls, - packages, # type: T.Iterable[syntax_tree.Package] - args, # type: argparse.Namespace - ): - # type: (...) -> T.Dict[str, T.Union[str, bytes]] + packages: T.Iterable[syntax_tree.Package], + args: argparse.Namespace, + ) -> T.Dict[str, T.Union[str, bytes]]: raise NotImplementedError() diff --git a/third_party/skymarshal/skymarshal/package_map.py b/third_party/skymarshal/skymarshal/package_map.py index 8d628e118..6feb0755b 100644 --- a/third_party/skymarshal/skymarshal/package_map.py +++ b/third_party/skymarshal/skymarshal/package_map.py @@ -1,6 +1,7 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs -from __future__ import absolute_import, print_function +# aclint: py3 + +from __future__ import annotations import os import typing as T @@ -8,11 +9,12 @@ from skymarshal import parser, syntax_tree, tokenizer # Path -> packages map of things we already parsed. -FILE_CACHE = dict() # type: T.Dict[T.Tuple[str, bool], T.List[syntax_tree.Package]] +FILE_CACHE: T.Dict[T.Tuple[str, bool], T.List[syntax_tree.Package]] = dict() -def merge_packages(packages, package_map=None): - # type: (T.List[syntax_tree.Package], T.Dict[str, syntax_tree.Package]) -> T.Dict[str, syntax_tree.Package] +def merge_packages( + packages: T.List[syntax_tree.Package], package_map: T.Dict[str, syntax_tree.Package] = None +) -> T.Dict[str, syntax_tree.Package]: """Converts a list of packages to a map. NOTE(matt): This makes copies instead of modifying the input packages. """ @@ -50,8 +52,7 @@ def find_lcmtypes_dirs(root_path, excluded_paths=None): dirnames[:] = [] -def _flatten_paths(lcmtypes_paths): - # type: (T.Iterable[str]) -> T.Iterable[str] +def _flatten_paths(lcmtypes_paths: T.Iterable[str]) -> T.Iterable[str]: for path in lcmtypes_paths: if not os.path.exists(path): continue @@ -65,14 +66,13 @@ def _flatten_paths(lcmtypes_paths): def parse_lcmtypes( - lcmtypes_paths, # type: T.Iterable[str] - verbose=False, # type: bool - print_debug_tokens=False, # type: bool - cache_parser=False, # type: bool - allow_unknown_notations=False, # type: bool - include_source_paths=True, # type: bool -): - # type: (...) -> T.Dict[str, syntax_tree.Package] + lcmtypes_paths: T.Iterable[str], + verbose: bool = False, + print_debug_tokens: bool = False, + cache_parser: bool = False, + allow_unknown_notations: bool = False, + include_source_paths: bool = True, +) -> T.Dict[str, syntax_tree.Package]: """ Parse LCM definitions and assemble a map from package to syntax tree nodes. @@ -82,7 +82,7 @@ def parse_lcmtypes( :param include_source_paths: If true, include the source file in the generated bindings :return: Map from package name to syntax_tree.Package. """ - package_map = {} # type: T.Dict[str, syntax_tree.Package] + package_map: T.Dict[str, syntax_tree.Package] = {} for path in _flatten_paths(lcmtypes_paths): if (path, include_source_paths) in FILE_CACHE: diff --git a/third_party/skymarshal/skymarshal/parser.py b/third_party/skymarshal/skymarshal/parser.py index aa1e07d8a..d29273a2f 100644 --- a/third_party/skymarshal/skymarshal/parser.py +++ b/third_party/skymarshal/skymarshal/parser.py @@ -1,4 +1,3 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs # LCM Defintion File Parsing # ========================== @@ -7,7 +6,9 @@ # Functions are required to have p_ prefix and docstrings that explain the syntax rules. # During parsing, the parse stack matches a rule, the corresponding function is executed. # A function's p argument is the list of productions in the rule, with p[0] being the output. -from __future__ import absolute_import, print_function +# aclint: py3 + +from __future__ import annotations import typing as T @@ -78,7 +79,7 @@ def p_enum(p): type_ref = syntax_tree.TypeRef("int32_t") reserved_ids, cases = p[5] else: - raise AssertionError("parser bug: len(p)={}".format(len(p))) + raise AssertionError(f"parser bug: len(p)={len(p)}") p[0] = syntax_tree.Enum( name=p[3], type_ref=type_ref, cases=cases, notations=p[1], reserved_ids=reserved_ids ) @@ -300,18 +301,21 @@ def p_error(token): # NOTE: Call yacc.errok() to proceed with parsing. # TODO(matt): add more error handling. if token: - raise LcmParseError( - 'Unable to parse starting from "{}" on line {}'.format(token.value, token.lineno) - ) + raise LcmParseError(f'Unable to parse starting from "{token.value}" on line {token.lineno}') else: raise LcmParseError("Unexpected end of input") -PARSER = None # type: T.Optional[yacc.LRParser] +PARSER: T.Optional[yacc.LRParser] = None -def lcmparse(src, verbose=True, cache=False, debug_src_path=None, allow_unknown_notations=False): - # type: (str, bool, bool, str, bool) -> T.List[syntax_tree.Package] +def lcmparse( + src: str, + verbose: bool = True, + cache: bool = False, + debug_src_path: str = None, + allow_unknown_notations: bool = False, +) -> T.List[syntax_tree.Package]: """Parse an LCM definition source into a list of packages""" global PARSER # pylint: disable=global-statement lexer.lineno = 1 # reset the line number on repeat calls to lcmgen @@ -336,9 +340,9 @@ def lcmparse(src, verbose=True, cache=False, debug_src_path=None, allow_unknown_ packages = parser.parse(src) except LcmParseError as ex: if debug_src_path: - raise LcmParseError("{} of {}".format(str(ex), debug_src_path)) + raise LcmParseError(f"{str(ex)} of {debug_src_path}") else: - raise LcmParseError("{}\n==========\n{}\n==========".format(str(ex), src)) + raise LcmParseError(f"{str(ex)}\n==========\n{src}\n==========") # NOTE(matt): this does not perform deduplication on package or struct names. return packages diff --git a/third_party/skymarshal/skymarshal/regenerate_lextab.py b/third_party/skymarshal/skymarshal/regenerate_lextab.py index 2f33915e7..45e06af04 100644 --- a/third_party/skymarshal/skymarshal/regenerate_lextab.py +++ b/third_party/skymarshal/skymarshal/regenerate_lextab.py @@ -1,8 +1,9 @@ -# aclint: py2 py3 +# aclint: py3 """ NOTE: if running with bazel, should run from the skymarshal directory """ -from __future__ import absolute_import + +from __future__ import annotations import argparse import imp @@ -51,14 +52,13 @@ class PyVersion(Enum): } -def get_py_ver(): - # type: () -> PyVersion +def get_py_ver() -> PyVersion: if sys.version_info.major < 3: return PyVersion.PY2 return PyVersion.PY3 -class Lextab(object): +class Lextab: """ Helper class to compare two instances of the lexstatere """ @@ -75,13 +75,11 @@ class Lextab(object): "_tabversion", ) - def __init__(self, lextab_module): - # type: (types.ModuleType) -> None + def __init__(self, lextab_module: types.ModuleType) -> None: for attrname in self.ATTRS: setattr(self, attrname, getattr(lextab_module, attrname)) - def __eq__(self, other): - # type: (T.Any) -> bool + def __eq__(self, other: T.Any) -> bool: if not isinstance(other, Lextab): return False @@ -94,16 +92,12 @@ def __eq__(self, other): return True - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: return "Lextab({})".format( - ", ".join( - ["{}={}".format(attrname, str(getattr(self, attrname))) for attrname in self.ATTRS] - ) + ", ".join([f"{attrname}={str(getattr(self, attrname))}" for attrname in self.ATTRS]) ) - def _compare_lexstatere(self, other): - # type: (Lextab) -> bool + def _compare_lexstatere(self, other: Lextab) -> bool: """ This attribute requires special handling because the regex is constructed in a non-deterministic fashion, and so the actual content of the regex may be sorted @@ -130,8 +124,7 @@ def _compare_lexstatere(self, other): return self_re == other_re and self_token == other_token -def regenerate_lextab(py_ver, write=False): - # type: (PyVersion, bool) -> None +def regenerate_lextab(py_ver: PyVersion, write: bool = False) -> None: tokenizer_path = os.path.join(SKYMARSHAL_DIR, "tokenizer.py") generated_path = os.path.join(SKYMARSHAL_DIR, "lextab.py") @@ -162,8 +155,7 @@ def regenerate_lextab(py_ver, write=False): os.remove(generated_path) -def get_parser(): - # type: () -> argparse.ArgumentParser +def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--check", @@ -173,8 +165,7 @@ def get_parser(): return parser -def main(): - # type: () -> None +def main() -> None: parser = get_parser() args = parser.parse_args() diff --git a/third_party/skymarshal/skymarshal/skymarshal.py b/third_party/skymarshal/skymarshal/skymarshal.py index 7e4c0deda..13d4314cc 100644 --- a/third_party/skymarshal/skymarshal/skymarshal.py +++ b/third_party/skymarshal/skymarshal/skymarshal.py @@ -1,9 +1,10 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs +# aclint: py3 """ Parse lcm defintion files and generate bindings in different languages. """ -from __future__ import absolute_import, print_function + +from __future__ import annotations import argparse import os @@ -16,8 +17,9 @@ from skymarshal.package_map import parse_lcmtypes -def parse_args(languages, args=None): - # type: (T.Sequence[T.Type[SkymarshalLanguage]], T.Optional[T.Sequence[str]]) -> argparse.Namespace +def parse_args( + languages: T.Sequence[T.Type[SkymarshalLanguage]], args: T.Optional[T.Sequence[str]] = None +) -> argparse.Namespace: """Parse the argument list and return an options object.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("source_path", nargs="+") @@ -67,11 +69,10 @@ def parse_args(languages, args=None): def main( - languages, # type: T.Sequence[T.Type[SkymarshalLanguage]] - args=None, # type: T.Sequence[str] - print_generated=True, # type: bool -): - # type: (...) -> None + languages: T.Sequence[T.Type[SkymarshalLanguage]], + args: T.Sequence[str] = None, + print_generated: bool = True, +) -> None: """The primary executable for generating lcmtypes code from struct definitions. This is mostly an example of how to use the generator.""" @@ -95,15 +96,15 @@ def main( files.update(lang.create_files(packages, options)) # Write any generated files that have changed. - for filename, content in six.iteritems(files): + for filename, content in files.items(): dirname = os.path.dirname(filename) if bool(dirname) and not os.path.exists(dirname): os.makedirs(dirname, 0o755) with open(filename, mode="wb") as output_file: - if isinstance(content, six.text_type): + if isinstance(content, str): output_file.write(content.encode("utf-8")) else: output_file.write(content) if print_generated: - print("Generated {} files".format(len(files))) + print(f"Generated {len(files)} files") diff --git a/third_party/skymarshal/skymarshal/syntax_tree.py b/third_party/skymarshal/skymarshal/syntax_tree.py index 0bc080b81..2f03cfde2 100644 --- a/third_party/skymarshal/skymarshal/syntax_tree.py +++ b/third_party/skymarshal/skymarshal/syntax_tree.py @@ -1,5 +1,6 @@ -# aclint: py2 py3 -from __future__ import absolute_import, print_function +# aclint: py3 + +from __future__ import annotations import typing as T @@ -26,7 +27,7 @@ # NOTE(eric): This should be synchronized with the definitions in tools/gazelle/lcm/lcmparser.go PRIMITIVE_TYPES = NUMERIC_TYPES + ("boolean", "byte", "string") -CONST_TYPE_MAP = {} # type: T.Dict[str, T.Any] +CONST_TYPE_MAP: T.Dict[str, T.Any] = {} for integer_type in INTEGER_TYPES: CONST_TYPE_MAP[integer_type] = lambda num: int(num, base=0) # automatically detect hex or dec. @@ -34,20 +35,17 @@ CONST_TYPE_MAP[float_type] = float -class Hash(object): - def __init__(self): - # type: () -> None +class Hash: + def __init__(self) -> None: self.val = int64(0x12345678) # TODO(matt): is it possible to remove the int64 dependency? - def update(self, byte): - # type: (int) -> None + def update(self, byte: int) -> None: """Make the hash dependent on the value of the given character. The order that hash_update is called in IS important.""" self.val = ((self.val << 8) ^ (self.val >> 55)) + byte - def update_string(self, string): - # type: (str) -> None + def update_string(self, string: str) -> None: "Make the hash dependent on each character in a string." self.update(len(string)) @@ -56,68 +54,57 @@ def update_string(self, string): self.update(ord(char)) @property - def int_value(self): - # type: () -> int + def int_value(self) -> int: # convert to uint64-like number # NOTE(matt): this is still a python int, and thus has infinite size return int(self.val) & 0xFFFFFFFFFFFFFFFF - def hex_no_padding(self): - # type: () -> str - return "0x{0:x}".format(self.int_value) + def hex_no_padding(self) -> str: + return f"0x{self.int_value:x}" - def hex_str(self): - # type: () -> str + def hex_str(self) -> str: # Convert to hexidecimal with padding return "{0:#0{1}x}".format(self.int_value, 18) # 18 is 16 + 2 - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: # Add the L since most languages use that. return self.hex_str() + "L" - def __eq__(self, rhs): - # type: (T.Any) -> bool + def __eq__(self, rhs: T.Any) -> bool: return self.int_value == rhs -class AstNode(object): +class AstNode: """Base class in the syntax tree""" - def __init__(self): - # type: () -> None + def __init__(self) -> None: self.lineno = -1 - self.comments = [] # type: T.List[str] + self.comments: T.List[str] = [] class Package(AstNode): """A named container of type definitions""" - def __init__(self, name, type_definitions): - # type: (str, T.Sequence[T.Union[Enum, Struct]]) -> None - super(Package, self).__init__() + def __init__(self, name: str, type_definitions: T.Sequence[T.Union[Enum, Struct]]) -> None: + super().__init__() self.name = name self.type_definitions = {t.name: t for t in type_definitions} for type_definition in self.type_definitions.values(): type_definition.add_package_name(self.name) @property - def struct_definitions(self): - # type: () -> T.List[Struct] + def struct_definitions(self) -> T.List[Struct]: return [s for s in self.type_definitions.values() if isinstance(s, Struct)] @property - def enum_definitions(self): - # type: () -> T.List[Enum] + def enum_definitions(self) -> T.List[Enum]: return [e for e in self.type_definitions.values() if isinstance(e, Enum)] - def reference_check(self): - # type: () -> None + def reference_check(self) -> None: for type_definition in self.type_definitions.values(): type_definition.reference_check() - def type_definition_with_name(self, name): - # type: (str) -> T.Union[Enum, Struct] + def type_definition_with_name(self, name: str) -> T.Union[Enum, Struct]: """Return a struct in the package by name.""" if name not in self.type_definitions: raise NameError( @@ -129,24 +116,20 @@ def type_definition_with_name(self, name): # should be no duplicates return self.type_definitions[name] - def extend_with_package(self, other_package): - # type: (Package) -> None + def extend_with_package(self, other_package: Package) -> None: """Add the given package's structs and enums to this package""" assert self.name == other_package.name - for name, t in iteritems(other_package.type_definitions): + for name, t in other_package.type_definitions.items(): if name in self.type_definitions: - raise NameError( - "Struct/Enum `{}` duplicated in package `{}`.".format(name, self.name) - ) + raise NameError(f"Struct/Enum `{name}` duplicated in package `{self.name}`.") self.type_definitions[name] = t - def __repr__(self): - # type: () -> str - lines = [] # type: T.List[str] - for _, type_definition in iteritems(self.type_definitions): + def __repr__(self) -> str: + lines: T.List[str] = [] + for _, type_definition in self.type_definitions.items(): lines.extend(repr(type_definition).splitlines()) children = "\n".join(lines) - return "package {};\n{}".format(self.name, children) + return f"package {self.name};\n{children}" NotationSpecProperty = T.NamedTuple("NotationSpecProperty", [("name", str), ("type", str)]) @@ -218,9 +201,8 @@ class Notation(AstNode): # entry will raise a KeyError. allow_unknown_notations = False - def __init__(self, name, properties, lineno): - # type: (str, T.Dict[str, str], int) -> None - super(Notation, self).__init__() + def __init__(self, name: str, properties: T.Dict[str, str], lineno: int) -> None: + super().__init__() self.name = name self.raw_properties = properties self.lineno = lineno @@ -230,9 +212,9 @@ def __init__(self, name, properties, lineno): self.spec = self.NOTATION_SPECS[name] except KeyError: if self.allow_unknown_notations: - print("Warning: Unknown notation: {}".format(name)) + print(f"Warning: Unknown notation: {name}") else: - raise KeyError("Unknown notation: {}".format(name)) + raise KeyError(f"Unknown notation: {name}") # Only check properties if this is a known notation. if self.spec is not None: @@ -241,9 +223,7 @@ def __init__(self, name, properties, lineno): unknown_props = provided_props - allowed_props if unknown_props: - raise KeyError( - "Unknown properties for notation {}: {}".format(name, sorted(unknown_props)) - ) + raise KeyError(f"Unknown properties for notation {name}: {sorted(unknown_props)}") # Parse the property values self.properties = { @@ -251,8 +231,7 @@ def __init__(self, name, properties, lineno): for prop, value in self.raw_properties.items() } - def parse_property(self, prop_name, raw_value): - # type: (str, str) -> T.Union[None, str, bool] + def parse_property(self, prop_name: str, raw_value: str) -> T.Union[None, str, bool]: if self.spec is None: return None [prop_spec] = [spec for spec in self.spec.properties if spec.name == prop_name] @@ -277,34 +256,33 @@ def parse_property(self, prop_name, raw_value): ) return dict(true=True, false=False)[raw_value] else: - raise AssertionError("unhandled prop_spec.type: {}".format(prop_spec.type)) + raise AssertionError(f"unhandled prop_spec.type: {prop_spec.type}") - def allowed_on_enum(self): - # type: () -> bool + def allowed_on_enum(self) -> bool: return self.spec is None or "enum" in self.spec.allowed - def allowed_on_struct(self): - # type: () -> bool + def allowed_on_struct(self) -> bool: return self.spec is None or "struct" in self.spec.allowed - def __repr__(self): - # type: () -> str - properties = ", ".join( - "{} = {}".format(key, value) for key, value in self.raw_properties.items() - ) - return "{} {{ {} }}\n".format(self.name, properties) + def __repr__(self) -> str: + properties = ", ".join(f"{key} = {value}" for key, value in self.raw_properties.items()) + return f"{self.name} {{ {properties} }}\n" class Enum(AstNode): """A description of an lcm enum type""" @classmethod - def from_name_and_cases(cls, name, case_names_and_values, type_name="int32_t"): - # type: (str, T.Sequence[T.Tuple[str, int]], str) -> Enum - value_to_case = {} # type: T.Dict[int, str] + def from_name_and_cases( + cls, + name: str, + case_names_and_values: T.Sequence[T.Tuple[str, int]], + type_name: str = "int32_t", + ) -> Enum: + value_to_case: T.Dict[int, str] = {} for case_name, case_value in case_names_and_values: - if case_name in value_to_case.values(): - raise KeyError("Case name {} is not unique".format(case_name)) + if case_name in list(value_to_case.values()): + raise KeyError(f"Case name {case_name} is not unique") if case_value in value_to_case: existing_name = value_to_case[case_value] raise KeyError( @@ -316,15 +294,21 @@ def from_name_and_cases(cls, name, case_names_and_values, type_name="int32_t"): cases = [ EnumCase(name=case_name, value_str=str(case_value)) - for case_value, case_name in sorted(iteritems(value_to_case)) + for case_value, case_name in sorted(value_to_case.items()) ] return cls( name=name, type_ref=TypeRef(type_name), cases=cases, notations=[], reserved_ids=[] ) - def __init__(self, name, type_ref, cases, notations, reserved_ids): - # type: (str, TypeRef, T.Sequence[EnumCase], T.Sequence[Notation], T.Sequence[int]) -> None - super(Enum, self).__init__() + def __init__( + self, + name: str, + type_ref: TypeRef, + cases: T.Sequence[EnumCase], + notations: T.Sequence[Notation], + reserved_ids: T.Sequence[int], + ) -> None: + super().__init__() equivalent_members = [Member(type_ref, "value")] equivalent_members += [ConstMember(type_ref, case.name, case.value_str) for case in cases] self.equivalent_struct = Struct(name, equivalent_members, []) @@ -336,55 +320,47 @@ def __init__(self, name, type_ref, cases, notations, reserved_ids): self.type_ref = TypeRef(name) self.notations = list(notations) self.reserved_ids = set(reserved_ids) - self.source_file = None # type: T.Optional[str] + self.source_file: T.Optional[str] = None - def set_source_file(self, source_file): - # type: (str) -> None + def set_source_file(self, source_file: str) -> None: self.source_file = source_file - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: notations = "".join(repr(notation) for notation in self.notations) children = "\n".join(" " + repr(case) for case in self.cases) reserved = "" if self.reserved_ids: - reserved = " {}\n".format(ReservedFieldGroup(self.reserved_ids)) + reserved = f" {ReservedFieldGroup(self.reserved_ids)}\n" return "{}enum {} : {} {{\n{}{}\n}};".format( notations, self.name, self.storage_type_ref, reserved, children ) - def __eq__(self, other): - # type: (T.Any) -> bool + def __eq__(self, other: T.Any) -> bool: try: other_tuple = (other.name, other.storage_type_ref, other.cases) except AttributeError: return False return (self.name, self.storage_type_ref, self.cases) == other_tuple - def __ne__(self, other): - # type: (T.Any) -> bool + def __ne__(self, other: T.Any) -> bool: return not self == other @property - def name(self): - # type: () -> str + def name(self) -> str: return self.type_ref.name @property - def full_name(self): - # type: () -> str + def full_name(self) -> str: return self.type_ref.full_name - def get_notation(self, name): - # type: (str) -> T.Optional[Notation] + def get_notation(self, name: str) -> T.Optional[Notation]: # get the first notation with the given name, else None for notation in self.notations: if notation.name == name: return notation return None - def get_notation_property(self, name, prop_name): - # type: (str, str) -> T.Union[None, str, bool] + def get_notation_property(self, name: str, prop_name: str) -> T.Union[None, str, bool]: notation = self.get_notation(name) if notation is None: return None @@ -393,35 +369,28 @@ def get_notation_property(self, name, prop_name): return None return prop_value - def case_for_int_value(self, int_value): - # type: (int) -> EnumCase + def case_for_int_value(self, int_value: int) -> EnumCase: for case in self.cases: if case.int_value == int_value: return case - raise KeyError("Value {} not found in:\n{}".format(int_value, self)) + raise KeyError(f"Value {int_value} not found in:\n{self}") - def add_package_name(self, package_name): - # type: (str) -> None + def add_package_name(self, package_name: str) -> None: self.type_ref.add_package_name(package_name) self.equivalent_struct.add_package_name(package_name) self.option_type_ref.add_package_name(self.type_ref.full_name) - def reference_check(self): - # type: () -> None + def reference_check(self) -> None: self.equivalent_struct.reference_check() disallowed_notations = [x.name for x in self.notations if not x.allowed_on_enum()] if disallowed_notations: - raise ValueError( - "Invalid notations {} on an enum ({})".format(disallowed_notations, self.name) - ) + raise ValueError(f"Invalid notations {disallowed_notations} on an enum ({self.name})") # validate the case values and reservations - cases_by_id = {} # type: T.Dict[int, str] + cases_by_id: T.Dict[int, str] = {} for case in self.cases: if case.int_value in self.reserved_ids: - raise KeyError( - "Enum case {} cannot use reserved id {}".format(case.name, case.int_value) - ) + raise KeyError(f"Enum case {case.name} cannot use reserved id {case.int_value}") if case.int_value in cases_by_id: raise KeyError( "Enum case {} reuses id {}. Also used by {}".format( @@ -432,100 +401,87 @@ def reference_check(self): ) cases_by_id[case.int_value] = case.name - def compute_hash(self): - # type: () -> Hash + def compute_hash(self) -> Hash: return self.equivalent_struct.compute_hash() class EnumCase(AstNode): """A name/value case of an lcm enum type""" - def __init__(self, name, value_str): - # type: (str, str) -> None - super(EnumCase, self).__init__() + def __init__(self, name: str, value_str: str) -> None: + super().__init__() self.name = name self.value_str = value_str self.type_ref = TypeRef(name) - def __repr__(self): - # type: () -> str - return "{} = {},".format(self.name, self.value_str) + def __repr__(self) -> str: + return f"{self.name} = {self.value_str}," - def __eq__(self, other): - # type: (T.Any) -> bool + def __eq__(self, other: T.Any) -> bool: try: other_tuple = (other.name, other.value_str) except AttributeError: return False return (self.name, self.value_str) == other_tuple - def __ne__(self, other): - # type: (T.Any) -> bool + def __ne__(self, other: T.Any) -> bool: return not self == other @property - def int_value(self): - # type: () -> int + def int_value(self) -> int: return int(self.value_str, base=0) class Struct(AstNode): """A description of an lcmtype""" - def __init__(self, name, members, notations): - # type: (str, T.Sequence[Member], T.Sequence[Notation]) -> None - super(Struct, self).__init__() + def __init__( + self, name: str, members: T.Sequence[Member], notations: T.Sequence[Notation] + ) -> None: + super().__init__() self.reserved_ids = [ field_id for group in members if isinstance(group, ReservedFieldGroup) for field_id in group.field_ids ] - self.members = [ + self.members: T.List[Member] = [ member for member in members if not isinstance(member, ReservedFieldGroup) - ] # type: T.List[Member] - self.member_map = { - member.name: member for member in self.members - } # type: T.Dict[str, Member] + ] + self.member_map: T.Dict[str, Member] = {member.name: member for member in self.members} self.type_ref = TypeRef(name) self.notations = list(notations) - self.source_file = None # type: T.Optional[str] + self.source_file: T.Optional[str] = None - def set_source_file(self, source_file): - # type: (str) -> None + def set_source_file(self, source_file: str) -> None: self.source_file = source_file - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: notations = "".join(repr(notation) for notation in self.notations) - reserved = ( + reserved: T.List[AstNode] = ( [ReservedFieldGroup(self.reserved_ids)] if self.reserved_ids else [] - ) # type: T.List[AstNode] + ) children = "\n".join( " " + repr(member) for member in reserved + T.cast(T.List[AstNode], self.members) ) - return "{}struct {} {{\n{}\n}};".format(notations, self.name, children) + return f"{notations}struct {self.name} {{\n{children}\n}};" @property - def name(self): - # type: () -> str + def name(self) -> str: return self.type_ref.name @property - def full_name(self): - # type: () -> str + def full_name(self) -> str: return self.type_ref.full_name - def get_notation(self, name): - # type: (str) -> T.Optional[Notation] + def get_notation(self, name: str) -> T.Optional[Notation]: # get the first notation with the given name, else None for notation in self.notations: if notation.name == name: return notation return None - def get_notation_property(self, name, prop_name): - # type: (str, str) -> T.Union[None, str, bool] + def get_notation_property(self, name: str, prop_name: str) -> T.Union[None, str, bool]: notation = self.get_notation(name) if notation is None: return None @@ -534,22 +490,18 @@ def get_notation_property(self, name, prop_name): return None return prop_value - def add_package_name(self, package_name): - # type: (str) -> None + def add_package_name(self, package_name: str) -> None: self.type_ref.add_package_name(package_name) for member in self.members: member.add_package_name(package_name) - def reference_check(self): - # type: () -> None + def reference_check(self) -> None: for member in self.members: member.reference_check(self) disallowed_notations = [x.name for x in self.notations if not x.allowed_on_struct()] if disallowed_notations: - raise ValueError( - "Invalid notations {} on a struct ({})".format(disallowed_notations, self.name) - ) + raise ValueError(f"Invalid notations {disallowed_notations} on a struct ({self.name})") has_protobuf_notation = bool(self.get_notation("#protobuf")) @@ -631,8 +583,7 @@ def reference_check(self): if member.field_id is not None: field_ids.add(member.field_id) - def compute_hash(self): - # type: () -> Hash + def compute_hash(self) -> Hash: type_hash = Hash() for member in self.members: if not isinstance(member, ConstMember): @@ -643,40 +594,40 @@ def compute_hash(self): class ReservedFieldGroup(AstNode): """A group of reserved field ids""" - def __init__(self, field_ids): - # type: (T.Iterable[int]) -> None - super(ReservedFieldGroup, self).__init__() + def __init__(self, field_ids: T.Iterable[int]) -> None: + super().__init__() self.field_ids = set(field_ids) - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: return "reserved {};".format(", ".join(str(fid) for fid in sorted(self.field_ids))) class Member(AstNode): """A field of an lcmtype""" - def __init__(self, type_ref, name, field_id=None, comments=None): - # type: (TypeRef, str, T.Optional[int], T.Optional[T.List[str]]) -> None - super(Member, self).__init__() + def __init__( + self, + type_ref: TypeRef, + name: str, + field_id: T.Optional[int] = None, + comments: T.Optional[T.List[str]] = None, + ) -> None: + super().__init__() self.name = name self.type_ref = type_ref self.field_id = field_id self.comments = comments or [] @property - def ndim(self): - # type: () -> int + def ndim(self) -> int: return 0 - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: if self.field_id is None: - return "{} {};".format(self.type_ref, self.name) - return "{} {} = {};".format(self.type_ref, self.name, self.field_id) + return f"{self.type_ref} {self.name};" + return f"{self.type_ref} {self.name} = {self.field_id};" - def compute_hash(self, type_hash): - # type: (Hash) -> None + def compute_hash(self, type_hash: Hash) -> None: self.compute_hash_prefix_for_auto_members(type_hash) # Hash the member name @@ -691,31 +642,26 @@ def compute_hash(self, type_hash): self.compute_hash_for_dimensions(type_hash) - def compute_hash_prefix_for_auto_members(self, type_hash): - # type: (Hash) -> None + def compute_hash_prefix_for_auto_members(self, type_hash: Hash) -> None: # Normal members don't have dimensions, thus don't have virtual members to add to the hash pass - def compute_hash_for_dimensions(self, type_hash): - # type: (Hash) -> None + def compute_hash_for_dimensions(self, type_hash: Hash) -> None: # Normal members don't have dimensions type_hash.update(0) - def add_package_name(self, package_name): - # type: (str) -> None + def add_package_name(self, package_name: str) -> None: self.type_ref.add_package_name(package_name) - def reference_check(self, _): - # type: (Struct) -> None + def reference_check(self, _: Struct) -> None: pass class TypeRef(AstNode): """A named reference to an existing type""" - def __init__(self, ref_str): - # type: (str) -> None - super(TypeRef, self).__init__() + def __init__(self, ref_str: str) -> None: + super().__init__() if ref_str in PRIMITIVE_TYPES: # This is a primitive. self.package_name = None @@ -728,70 +674,59 @@ def __init__(self, ref_str): self.package_name = "" self.name = ref_str - def add_package_name(self, package_name): - # type: (str) -> None + def add_package_name(self, package_name: str) -> None: if self.package_name == "": # We only set the package name if the given one was implied. self.package_name = package_name @property - def full_name(self): - # type: () -> str + def full_name(self) -> str: if self.package_name: - return "{}.{}".format(self.package_name, self.name) + return f"{self.package_name}.{self.name}" else: return self.name - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: return self.full_name - def __eq__(self, other): - # type: (T.Any) -> bool + def __eq__(self, other: T.Any) -> bool: try: other_tuple = (other.name, other.package_name) except AttributeError: return False return (self.name, self.package_name) == other_tuple - def __ne__(self, other): - # type: (T.Any) -> bool + def __ne__(self, other: T.Any) -> bool: return not self == other - def is_non_string_primitive_type(self): - # type: () -> bool + def is_non_string_primitive_type(self) -> bool: # Many languages treat strings differently than other primitives, so this is a common check. return self.name != "string" and self.is_primitive_type() - def is_primitive_type(self): - # type: () -> bool + def is_primitive_type(self) -> bool: return self.name in PRIMITIVE_TYPES - def is_numeric_type(self): - # type: () -> bool + def is_numeric_type(self) -> bool: return self.name in NUMERIC_TYPES - def is_const_type(self): - # type: () -> bool + def is_const_type(self) -> bool: return self.name in CONST_TYPES class ArrayDim(AstNode): """A static or dynamic size of an array in a single axis""" - def __init__(self, sizes): - # type: (T.Tuple[str, ...]) -> None - super(ArrayDim, self).__init__() + def __init__(self, sizes: T.Tuple[str, ...]) -> None: + super().__init__() self.sizes_as_declared = sizes # This is a tuple of the 0-2 strings in the .lcm definition. # These will be set after the reference check. - self.size_str = None # type: T.Optional[str] - self.size_int = None # type: T.Optional[int] - self._dynamic = None # type: T.Optional[bool] - self._auto_member = None # type: T.Optional[Member] + self.size_str: T.Optional[str] = None + self.size_int: T.Optional[int] = None + self._dynamic: T.Optional[bool] = None + self._auto_member: T.Optional[Member] = None @property - def dynamic(self): - # type: () -> bool + def dynamic(self) -> bool: """Return True if the dimension is dynamic based on struct contents. This value is not known until after struct.reference_check(...) is called.""" if self._dynamic is None: @@ -799,22 +734,19 @@ def dynamic(self): return self._dynamic @property - def auto_member(self): - # type: () -> T.Optional[Member] + def auto_member(self) -> T.Optional[Member]: """Return the virtual Member of the dimension if using automatic length encoding. This value is not known until after struct.reference_check(...) is called.""" if self._dynamic is None: raise RuntimeError("Cannot determine until reference_check() is called") return self._auto_member - def compute_hash(self, type_hash): - # type: (Hash) -> None + def compute_hash(self, type_hash: Hash) -> None: type_hash.update(1 if self.dynamic else 0) assert self.size_str is not None, "size_str must be set before calling compute_hash" type_hash.update_string(self.size_str) - def reference_check(self, struct, member): - # type: (Struct, Member) -> None + def reference_check(self, struct: Struct, member: Member) -> None: """ Use the references to determine if this arraydim is static or dynamic, and to build the rest of the info. There are 6 valid ways of specifying an array dimension: @@ -853,10 +785,10 @@ def reference_check(self, struct, member): pass # collapse case 6 into case 5 - decl_tuple = self.sizes_as_declared or ("int32_t",) # type: T.Tuple[str, ...] + decl_tuple: T.Tuple[str, ...] = self.sizes_as_declared or ("int32_t",) # collapse case 5 into case 4 if len(decl_tuple) == 1 and decl_tuple[0] in INTEGER_TYPES: - auto_name = "num_{}".format(member.name) + auto_name = f"num_{member.name}" decl_tuple = (decl_tuple[0], auto_name) # Now decl_tuple is len=1 for cases 2 and 3, and len=2 for cases 4, 5, and 6 @@ -895,11 +827,11 @@ def reference_check(self, struct, member): member_name = decl_tuple[0] # Confirm that the size is a reference to a member of this struct. if member_name not in struct.member_map: - raise NameError("ArrayDim {} is not a defined member".format(size_declaration)) + raise NameError(f"ArrayDim {size_declaration} is not a defined member") size_member = struct.member_map[member_name] if size_member.type_ref.name not in INTEGER_TYPES: - raise TypeError("ArrayDim {} is not a valid array type".format(size_declaration)) + raise TypeError(f"ArrayDim {size_declaration} is not a valid array type") # Determine if this member is a const, or a regular member if isinstance(size_member, ConstMember): @@ -913,9 +845,7 @@ def reference_check(self, struct, member): # Check that the non-static member appears before the array if struct.members.index(size_member) >= struct.members.index(member): - raise ValueError( - "ArrayDim {} must appear before the array {}".format(member_name, member.name) - ) + raise ValueError(f"ArrayDim {member_name} must appear before the array {member.name}") # This references a variable in the message whose value is not known statically. self._dynamic = True @@ -923,8 +853,7 @@ def reference_check(self, struct, member): self.size_str = member_name return # case 3 - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: return "[{}]".format(" ".join(self.sizes_as_declared)) @@ -933,53 +862,51 @@ class ArrayMember(Member): # TODO(matt): might be better to just combine with Member, as lcmgen original does. - def __init__(self, type_ref, name, dims, field_id=None): - # type: (TypeRef, str, T.Sequence[ArrayDim], T.Optional[int]) -> None - super(ArrayMember, self).__init__(type_ref, name, field_id=field_id) + def __init__( + self, + type_ref: TypeRef, + name: str, + dims: T.Sequence[ArrayDim], + field_id: T.Optional[int] = None, + ) -> None: + super().__init__(type_ref, name, field_id=field_id) self.dims = dims - def compute_hash_prefix_for_auto_members(self, type_hash): - # type: (Hash) -> None + def compute_hash_prefix_for_auto_members(self, type_hash: Hash) -> None: # this adds the hash of virtual fields would have been used by a manual dynamic array for dim in self.dims: if dim.auto_member: dim.auto_member.compute_hash(type_hash) - def compute_hash_for_dimensions(self, type_hash): - # type: (Hash) -> None + def compute_hash_for_dimensions(self, type_hash: Hash) -> None: # hash the dimensionality information type_hash.update(len(self.dims)) for dim in self.dims: dim.compute_hash(type_hash) - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: dims_str = "".join(repr(dim) for dim in self.dims) if self.field_id is None: - return "{} {}{};".format(self.type_ref, self.name, dims_str) - return "{} {}{} = {};".format(self.type_ref, self.name, dims_str, self.field_id) + return f"{self.type_ref} {self.name}{dims_str};" + return f"{self.type_ref} {self.name}{dims_str} = {self.field_id};" - def is_constant_size(self): - # type: () -> bool + def is_constant_size(self) -> bool: return not any(dim.dynamic for dim in self.dims) - def reference_check(self, struct): - # type: (Struct) -> None + def reference_check(self, struct: Struct) -> None: for dim in self.dims: dim.reference_check(struct, self) @property - def ndim(self): - # type: () -> int + def ndim(self) -> int: return len(self.dims) class ConstMember(Member): """An attribute whose value is bound to the type itself, not encoded in a message""" - def __init__(self, type_ref, name, value_str): - # type: (TypeRef, str, str) -> None - super(ConstMember, self).__init__(type_ref, name) + def __init__(self, type_ref: TypeRef, name: str, value_str: str) -> None: + super().__init__(type_ref, name) if not type_ref.is_const_type(): raise TypeError( "Constant '{}' from line {} must be one of {}. '{}' found.".format( @@ -989,10 +916,9 @@ def __init__(self, type_ref, name, value_str): try: self.value = CONST_TYPE_MAP[type_ref.name](value_str) except ValueError: - print("Error parsing const type {}.".format(self.name)) + print(f"Error parsing const type {self.name}.") raise self.value_str = value_str - def __repr__(self): - # type: () -> str - return "const {} {} = {};".format(self.type_ref, self.name, self.value_str) + def __repr__(self) -> str: + return f"const {self.type_ref} {self.name} = {self.value_str};" diff --git a/third_party/skymarshal/skymarshal/tokenizer.py b/third_party/skymarshal/skymarshal/tokenizer.py index ef8be5766..59dedb925 100644 --- a/third_party/skymarshal/skymarshal/tokenizer.py +++ b/third_party/skymarshal/skymarshal/tokenizer.py @@ -1,8 +1,9 @@ -# aclint: py2 py3 # mypy: allow-untyped-defs # LCM Definition File tokenizer # We use PLY and regexes to convert a byte stream into a sequence of tokens. -from __future__ import absolute_import, print_function +# aclint: py3 + +from __future__ import annotations import os import shutil @@ -54,7 +55,7 @@ # So for now I create a COMMENT_QUEUE which holds unprocessed comments # When a STRUCT, PACKAGE, CLOSE_BRACE, or SEMICOLOR token is found, the queue gets processed. # This enables the parser to associate comments without complicating the syntax rules. -COMMENT_QUEUE = [] # type: T.List[str] +COMMENT_QUEUE: T.List[str] = [] def consume_comment_queue(f):