Python: Numba based SBP parsing [ESD-1154]

.flake8

@@ -0,0 +1 @@
+max-line-length = 120

Makefile

@@ -61,6 +61,7 @@ docs: verify-prereq-docs pdf html
 
 c:          deps-c          gen-c          test-c
 python:     deps-python     gen-python     test-python
+pythonNG:   deps-python     gen-pythonNG
 javascript: deps-javascript gen-javascript test-javascript
 java:       deps-java       gen-java       test-java
 haskell:    deps-haskell    gen-haskell    test-haskell
@@ -144,6 +145,15 @@ gen-python:
 		       --python
 	$(call announce-end,"Finished generating Python bindings. Please check $(SWIFTNAV_ROOT)/python/sbp")
 
+gen-pythonNG:
+	$(call announce-begin,"Generating Python bindings")
+	cd $(SWIFTNAV_ROOT)/generator; \
+	$(SBP_GEN_BIN) -i $(SBP_SPEC_DIR) \
+		       -o $(SWIFTNAV_ROOT)/python/sbp/jit \
+                       -r $(SBP_MAJOR_VERSION).$(SBP_MINOR_VERSION).$(SBP_PATCH_VERSION) \
+		       --pythonNG
+	$(call announce-end,"Finished generating Python bindings. Please check $(SWIFTNAV_ROOT)/python/sbp")
+
 gen-javascript:
 	$(call announce-begin,"Generating JavaScript bindings")
 	cd $(SWIFTNAV_ROOT)/generator; \

generator/sbpg/generator.py

@@ -25,6 +25,7 @@
 import sbpg.targets.haskell as hs
 import sbpg.targets.protobuf as pb
 import sbpg.targets.python as py
+import sbpg.targets.pythonNG as pyNG
 import sbpg.targets.javascript as js
 
 def get_args():
@@ -42,6 +43,9 @@ def get_args():
   parser.add_argument('--python',
                       action="store_true",
                       help='Target language: Python.')
+  parser.add_argument('--pythonNG',
+                      action="store_true",
+                      help='Target language: Python.')
   parser.add_argument('--javascript',
                       action="store_true",
                       help='Target language: JavaScript.')
@@ -80,7 +84,7 @@ def main():
     # Parse and validate arguments.
     args = get_args().parse_args()
     verbose = args.verbose
-    assert args.python or args.javascript or args.c or args.test_c or args.haskell or args.latex or args.protobuf or args.java, \
+    assert args.pythonNG or args.python or args.javascript or args.c or args.test_c or args.haskell or args.latex or args.protobuf or args.java, \
       "Please specify a target language."
     input_file = os.path.abspath(args.input_file[0])
     assert len(args.input_file) == 1
@@ -122,6 +126,8 @@ def main():
           continue
         if args.python:
           py.render_source(output_dir, parsed)
+        if args.pythonNG:
+          pyNG.render_source(output_dir, parsed)
         elif args.javascript:
           js.render_source(output_dir, parsed)
         elif args.c:

generator/sbpg/targets/pythonNG.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# Copyright (C) 2015 Swift Navigation Inc.
+# Contact: Bhaskar Mookerji <mookerji@swiftnav.com>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+
+"""Generator for Python target.
+
+This module consumes the YAML spec and generates some message class
+files.
+
+"""
+
+from sbpg.targets.templating import JENV, ACRONYMS
+from sbpg.utils import comment_links
+import copy
+
+TEMPLATE_NAME = "sbp_numba.py.j2"
+
+NUMBA_TYPE = {
+  'u8': 'u1',
+  'u16': 'u2',
+  'u32': 'u4',
+  'u64': 'u8',
+  's8': 'i1',
+  's16': 'i2',
+  's32': 'i4',
+  's64': 'i8',
+  'float': 'f4',
+  'double': 'f8',
+}
+
+NUMBA_GET_FN = {
+  'u8': 'get_u8',
+  'u16': 'get_u16',
+  'u32': 'get_u32',
+  'u64': 'get_u64',
+  's8': 'get_s8',
+  's16': 'get_s16',
+  's32': 'get_s32',
+  's64': 'get_s64',
+  'float': 'get_f32',
+  'double': 'get_f64',
+}
+
+NUMBA_TY_BYTES = {
+  'u8': 1,
+  'u16': 2,
+  'u32': 4,
+  'u64': 8,
+  's8': 1,
+  's16': 2,
+  's32': 4,
+  's64': 8,
+  'float': 4,
+  'double': 8,
+}
+
+PYDOC_CODE = {
+  'u8': 'int',
+  'u16': 'int',
+  'u32': 'int',
+  'u64': 'int',
+  's8': 'int',
+  's16': 'int',
+  's32': 'int',
+  's64': 'int',
+  'float': 'float',
+  'double': 'double',
+}
+
+
+def is_array():
+  return False
+
+
+def numba_type(f):
+  if f.type_id == 'float':
+    return 'judicious_round(nb.' + NUMBA_TYPE[f.type_id] + \
+      '(__' + f.identifier + ')) if SBP.judicious_rounding else __' + f.identifier
+  else:
+    return '__' + f.identifier
+
+
+def numba_size(f):
+  # the worst case 255 - 6 (header) - 2 (crc)
+  UNKNOWN_LEN = 255 - 6 - 2
+
+  if f.type_id in NUMBA_TY_BYTES:
+    return NUMBA_TY_BYTES[f.type_id]
+  elif f.type_id == 'string':
+    if f.options.get('size', None) is not None:
+      return f.options.get('size', None).value
+    return UNKNOWN_LEN
+  elif f.type_id == 'array':
+    # NOTE: arrays of arrays are not supported
+    t = f.options['fill'].value
+    count = f.options.get('size', None)
+    if count:
+      if t in NUMBA_TY_BYTES:
+        return "%d * %d" % (NUMBA_TY_BYTES[t], count.value)
+      else:
+        return t + "._payload_size() * %d" % (count.value)
+    else:
+      return UNKNOWN_LEN
+  else:
+    return f.type_id + '._payload_size()'
+
+
+def numba_format(f):
+  if NUMBA_GET_FN.get(f.type_id, None):
+    return NUMBA_GET_FN.get(f.type_id)
+  elif f.type_id == 'string' and f.identifier == 'setting' and not f.options.get('size', None):
+    # setting string with null delimiters as a special case
+    return 'get_setting'
+  elif f.type_id == 'string' and f.options.get('size', None):
+    s = f.options.get('size', None).value
+    return 'get_fixed_string(%d)' % (s,)
+  elif f.type_id == 'string' and not f.options.get('size', None):
+    return 'get_string'
+  elif f.type_id == 'array' and f.options.get('size', None):
+    count = f.options.get('size', None).value
+    t = f.options['fill'].value
+    if t in NUMBA_GET_FN:
+      fill_func = NUMBA_GET_FN[t]
+      el_size = NUMBA_TY_BYTES[t]
+      if f.options['fill'].value == 'float':
+        return "get_fixed_array(%s, %d, %d, %s if SBP.judicious_rounding else None)" \
+          % (fill_func, count, el_size, 'nb.f4')
+      else:
+        return "get_fixed_array(%s, %d, %d)" % (fill_func, count, el_size)
+    else:
+      fill_func = f.options['fill'].value + '._unpack_members'
+      el_size = f.options['fill'].value + '._payload_size()'
+      return "get_fixed_array(%s, %d, %s)" % (fill_func, count, el_size)
+  elif f.type_id == 'array':
+    fill = f.options['fill'].value
+    f_ = copy.copy(f)
+    f_.type_id = fill
+    return "get_array(%s)" % (numba_format(f_),)
+  else:
+    return '%s.parse_members' % (f.type_id)
+  raise NotImplementedError()
+
+
+def pydoc_format(type_id, pydoc=PYDOC_CODE):
+  """
+  Formats type for pydoc.
+  """
+  return pydoc.get(type_id, type_id)
+
+
+def classnameify(s):
+  """
+  Makes a classname.
+  """
+  return ''.join(w if w in ACRONYMS else w.title() for w in s.split('_'))
+
+
+JENV.filters['numba_py'] = numba_format
+JENV.filters['numba_type'] = numba_type
+JENV.filters['numba_size'] = numba_size
+JENV.filters['classnameify'] = classnameify
+JENV.filters['pydoc'] = pydoc_format
+JENV.filters['comment_links'] = comment_links
+
+
+def render_source(output_dir, package_spec, jenv=JENV):
+  """
+  Render and output
+  """
+  path, name = package_spec.filepath
+  directory = output_dir
+  destination_filename = "%s/%s.py" % (directory, name)
+  py_template = jenv.get_template(TEMPLATE_NAME)
+  module_path = ".".join(package_spec.identifier.split(".")[1:-1])
+  includes = [".".join(i.split(".")[:-1]) for i in package_spec.includes]
+  includes = [i for i in includes if i != "types"]
+  print(destination_filename, includes)
+  with open(destination_filename, 'w') as f:
+    f.write(py_template.render(msgs=package_spec.definitions,
+                               filepath="/".join(package_spec.filepath) + ".yaml",
+                               module_path=module_path,
+                               include=includes,
+                               timestamp=package_spec.creation_timestamp,
+                               description=package_spec.description))

generator/sbpg/targets/resources/sbp_numba.py.j2

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# Copyright (C) 2015-2018 Swift Navigation Inc.
+# Contact: Swift Navigation <dev@swiftnav.com>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+
+
+"""
+(((description|comment_links)))
+"""
+
+import json
+
+import numba as nb
+
+from sbp.jit.msg import SBP, SENDER_ID
+from sbp.jit.msg import get_u8, get_u16, get_u32, get_u64
+from sbp.jit.msg import get_s8, get_s16, get_s32, get_s64
+from sbp.jit.msg import get_f32, get_f64, judicious_round
+from sbp.jit.msg import get_string, get_fixed_string, get_setting
+from sbp.jit.msg import get_array, get_fixed_array
+
+((*- for i in include *))
+from (((module_path))).jit.(((i))) import *
+((*- endfor *))
+
+# Automatically generated from piksi/yaml/(((filepath))) with generate.py.
+# Please do not hand edit!
+
+((*- for m in msgs *))
+((*- if m.static *))
+
+((*- if m.sbp_id *))
+SBP_(((m.identifier))) = ((('0x%04X'|format(m.sbp_id))))
+((*- endif *))
+
+((*- if m.sbp_id *))
+class ((( m.identifier | classnameify )))(SBP):
+  """SBP class for message (((m.identifier))) ((('(0x%04X)'|format(m.sbp_id)))).
+((*- else *))
+class ((( m.identifier )))(object):
+  """SBP class for message (((m.identifier)))
+((*- endif *))
+
+  You can have (((m.identifier))) inherit its fields directly
+  from an inherited SBP object, or construct it inline using a dict
+  of its fields.
+
+  ((* if m.desc *))
+  (((m.desc)))
+  ((*- endif *))
+
+  """
+  ((*- if m.fields *))
+  __slots__ = [((*- for f in m.fields *))'((( f.identifier )))',
+               ((* endfor *))]
+  ((*- else *))
+  __slots__ = []
+  ((*- endif *))
+
+  ((*- if m.fields *))
+  @classmethod
+  def parse_members(cls, buf, offset, length):
+    ret = {}
+    ((*- for f in m.fields *))
+    (__(((f.identifier))), offset, length) = ((( f | numba_py )))(buf, offset, length)
+    ret['(((f.identifier)))'] = ((( f | numba_type )))
+    ((*- endfor *))
+    return ret, offset, length
+
+  def _unpack_members(self, buf, offset, length):
+    res, off, length = self.parse_members(buf, offset, length)
+    if off == offset:
+      return {}, offset, length
+    ((*- for f in m.fields *))
+    self.(((f.identifier))) = res['(((f.identifier)))']
+    ((*- endfor *))
+    return res, off, length
+
+  @classmethod
+  def _payload_size(self):
+    ret = 0
+    ((*- for f in m.fields *))
+    # (((f.identifier))): (((f.type_id))) ((*- if f.options['fill'] *)) of (((f.options['fill'].value))) ((*- endif *))
+    ret += (((f | numba_size)))
+    ((*- endfor *))
+    return ret
+  ((* else *))
+  def _unpack_members(self, buf, offset, length):
+    return {}, offset, length
+
+  def _payload_size(self):
+    return 0
+  ((* endif *))
+((*- endif *))
+((*- endfor *))
+
+msg_classes = {
+((*- for m in msgs *))
+  ((*- if m.sbp_id *))
+  ((('0x%04X'|format(m.sbp_id)))): ((( m.identifier | classnameify ))),
+  ((*- endif*))
+((*- endfor *))
+}