Automated Travis benchmark [ESD-1158]

.flake8

@@ -0,0 +1 @@
+max-line-length = 120

.travis.yml

@@ -22,15 +22,27 @@ cache:
 matrix:
   include:
     - env:
-        - TRAVIS_TARGET=python
+        - TRAVIS_TARGET=python-haskell
       language: python
       python: 3.7-dev
       before_install:
         - sudo apt-get -qq update
         - sudo apt-get install python2.7 python2.7-dev
         - sudo apt-get install python3.5 python3.5-dev
+        - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
         - sudo pip install tox
-      script: make test-python
+        - git clone https://github.com/swift-nav/piksi_tools.git ../piksi_tools
+        - git -C ../piksi_tools fetch
+        - git -C ../piksi_tools checkout silverjam/sbp2json
+      script: |
+        pushd haskell
+        docker build -t sbp2json .
+        docker run --rm --name sbp2json sbp2json >sbp_linux_tools.tar
+        tar xf sbp_linux_tools.tar
+        gzip sbp_linux_tools.tar
+        ls -l
+        popd
+        make test-python
     - env:
         - TRAVIS_TARGET=default JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
       before_install:
@@ -54,18 +66,6 @@ matrix:
         - npm install -g mocha
         - npm install
       script: make test-c test-haskell test-java test-javascript
-    - env:
-        - TRAVIS_TARGET=haskell-tools
-      before_install: |
-        sudo apt-get update
-        sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-      script: |
-        pushd haskell
-        docker build -t sbp2json .
-        docker run --rm --name sbp2json sbp2json >sbp_linux_tools.tar
-        gzip sbp_linux_tools.tar
-        ls -l
-        popd
 
 deploy:
   provider: releases

Makefile

@@ -61,6 +61,7 @@ docs: verify-prereq-docs pdf html
 
 c:          deps-c          gen-c          test-c
 python:     deps-python     gen-python     test-python
+pythonNG:   deps-python     gen-pythonNG
 javascript: deps-javascript gen-javascript test-javascript
 java:       deps-java       gen-java       test-java
 haskell:    deps-haskell    gen-haskell    test-haskell
@@ -144,6 +145,15 @@ gen-python:
 		       --python
 	$(call announce-end,"Finished generating Python bindings. Please check $(SWIFTNAV_ROOT)/python/sbp")
 
+gen-pythonNG:
+	$(call announce-begin,"Generating Python bindings")
+	cd $(SWIFTNAV_ROOT)/generator; \
+	$(SBP_GEN_BIN) -i $(SBP_SPEC_DIR) \
+		       -o $(SWIFTNAV_ROOT)/python/sbp/jit \
+                       -r $(SBP_MAJOR_VERSION).$(SBP_MINOR_VERSION).$(SBP_PATCH_VERSION) \
+		       --pythonNG
+	$(call announce-end,"Finished generating Python bindings. Please check $(SWIFTNAV_ROOT)/python/sbp")
+
 gen-javascript:
 	$(call announce-begin,"Generating JavaScript bindings")
 	cd $(SWIFTNAV_ROOT)/generator; \
@@ -206,7 +216,7 @@ test-c:
 test-python:
 	$(call announce-begin,"Running Python tests")
 ifdef TRAVIS_TARGET
-	cd $(SWIFTNAV_ROOT)/python/ && tox
+	cd $(SWIFTNAV_ROOT)/python/ && tox -- $(SWIFTNAV_ROOT)/haskell
 else
 	cd $(SWIFTNAV_ROOT)/python/ && tox --skip-missing-interpreters
 endif

generator/sbpg/generator.py

@@ -25,6 +25,7 @@
 import sbpg.targets.haskell as hs
 import sbpg.targets.protobuf as pb
 import sbpg.targets.python as py
+import sbpg.targets.pythonNG as pyNG
 import sbpg.targets.javascript as js
 
 def get_args():
@@ -42,6 +43,9 @@ def get_args():
   parser.add_argument('--python',
                       action="store_true",
                       help='Target language: Python.')
+  parser.add_argument('--pythonNG',
+                      action="store_true",
+                      help='Target language: Python.')
   parser.add_argument('--javascript',
                       action="store_true",
                       help='Target language: JavaScript.')
@@ -80,7 +84,7 @@ def main():
     # Parse and validate arguments.
     args = get_args().parse_args()
     verbose = args.verbose
-    assert args.python or args.javascript or args.c or args.test_c or args.haskell or args.latex or args.protobuf or args.java, \
+    assert args.pythonNG or args.python or args.javascript or args.c or args.test_c or args.haskell or args.latex or args.protobuf or args.java, \
       "Please specify a target language."
     input_file = os.path.abspath(args.input_file[0])
     assert len(args.input_file) == 1
@@ -122,6 +126,8 @@ def main():
           continue
         if args.python:
           py.render_source(output_dir, parsed)
+        if args.pythonNG:
+          pyNG.render_source(output_dir, parsed)
         elif args.javascript:
           js.render_source(output_dir, parsed)
         elif args.c:

generator/sbpg/targets/pythonNG.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# Copyright (C) 2015 Swift Navigation Inc.
+# Contact: Bhaskar Mookerji <mookerji@swiftnav.com>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+
+"""Generator for Python target.
+
+This module consumes the YAML spec and generates some message class
+files.
+
+"""
+
+from sbpg.targets.templating import JENV, ACRONYMS
+from sbpg.utils import comment_links
+import copy
+
+TEMPLATE_NAME = "sbp_numba.py.j2"
+
+NUMBA_TYPE = {
+  'u8': 'u1',
+  'u16': 'u2',
+  'u32': 'u4',
+  'u64': 'u8',
+  's8': 'i1',
+  's16': 'i2',
+  's32': 'i4',
+  's64': 'i8',
+  'float': 'f4',
+  'double': 'f8',
+}
+
+NUMBA_GET_FN = {
+  'u8': 'get_u8',
+  'u16': 'get_u16',
+  'u32': 'get_u32',
+  'u64': 'get_u64',
+  's8': 'get_s8',
+  's16': 'get_s16',
+  's32': 'get_s32',
+  's64': 'get_s64',
+  'float': 'get_f32',
+  'double': 'get_f64',
+}
+
+NUMBA_TY_BYTES = {
+  'u8': 1,
+  'u16': 2,
+  'u32': 4,
+  'u64': 8,
+  's8': 1,
+  's16': 2,
+  's32': 4,
+  's64': 8,
+  'float': 4,
+  'double': 8,
+}
+
+PYDOC_CODE = {
+  'u8': 'int',
+  'u16': 'int',
+  'u32': 'int',
+  'u64': 'int',
+  's8': 'int',
+  's16': 'int',
+  's32': 'int',
+  's64': 'int',
+  'float': 'float',
+  'double': 'double',
+}
+
+
+def is_array():
+  return False
+
+
+def numba_type(f):
+  if f.type_id == 'float':
+    return 'judicious_round(nb.' + NUMBA_TYPE[f.type_id] + \
+      '(__' + f.identifier + ')) if SBP.judicious_rounding else __' + f.identifier
+  else:
+    return '__' + f.identifier
+
+
+def numba_size(f):
+  # the worst case 255 - 6 (header) - 2 (crc)
+  UNKNOWN_LEN = 255 - 6 - 2
+
+  if f.type_id in NUMBA_TY_BYTES:
+    return NUMBA_TY_BYTES[f.type_id]
+  elif f.type_id == 'string':
+    if f.options.get('size', None) is not None:
+      return f.options.get('size', None).value
+    return UNKNOWN_LEN
+  elif f.type_id == 'array':
+    # NOTE: arrays of arrays are not supported
+    t = f.options['fill'].value
+    count = f.options.get('size', None)
+    if count:
+      if t in NUMBA_TY_BYTES:
+        return "%d * %d" % (NUMBA_TY_BYTES[t], count.value)
+      else:
+        return t + "._payload_size() * %d" % (count.value)
+    else:
+      return UNKNOWN_LEN
+  else:
+    return f.type_id + '._payload_size()'
+
+
+def numba_format(f):
+  if NUMBA_GET_FN.get(f.type_id, None):
+    return NUMBA_GET_FN.get(f.type_id)
+  elif f.type_id == 'string' and f.identifier == 'setting' and not f.options.get('size', None):
+    # setting string with null delimiters as a special case
+    return 'get_setting'
+  elif f.type_id == 'string' and f.options.get('size', None):
+    s = f.options.get('size', None).value
+    return 'get_fixed_string(%d)' % (s,)
+  elif f.type_id == 'string' and not f.options.get('size', None):
+    return 'get_string'
+  elif f.type_id == 'array' and f.options.get('size', None):
+    count = f.options.get('size', None).value
+    t = f.options['fill'].value
+    if t in NUMBA_GET_FN:
+      fill_func = NUMBA_GET_FN[t]
+      el_size = NUMBA_TY_BYTES[t]
+      if f.options['fill'].value == 'float':
+        return "get_fixed_array(%s, %d, %d, %s if SBP.judicious_rounding else None)" \
+          % (fill_func, count, el_size, 'nb.f4')
+      else:
+        return "get_fixed_array(%s, %d, %d)" % (fill_func, count, el_size)
+    else:
+      fill_func = f.options['fill'].value + '._unpack_members'
+      el_size = f.options['fill'].value + '._payload_size()'
+      return "get_fixed_array(%s, %d, %s)" % (fill_func, count, el_size)
+  elif f.type_id == 'array':
+    fill = f.options['fill'].value
+    f_ = copy.copy(f)
+    f_.type_id = fill
+    return "get_array(%s)" % (numba_format(f_),)
+  else:
+    return '%s.parse_members' % (f.type_id)
+  raise NotImplementedError()
+
+
+def pydoc_format(type_id, pydoc=PYDOC_CODE):
+  """
+  Formats type for pydoc.
+  """
+  return pydoc.get(type_id, type_id)
+
+
+def classnameify(s):
+  """
+  Makes a classname.
+  """
+  return ''.join(w if w in ACRONYMS else w.title() for w in s.split('_'))
+
+
+JENV.filters['numba_py'] = numba_format
+JENV.filters['numba_type'] = numba_type
+JENV.filters['numba_size'] = numba_size
+JENV.filters['classnameify'] = classnameify
+JENV.filters['pydoc'] = pydoc_format
+JENV.filters['comment_links'] = comment_links
+
+
+def render_source(output_dir, package_spec, jenv=JENV):
+  """
+  Render and output
+  """
+  path, name = package_spec.filepath
+  directory = output_dir
+  destination_filename = "%s/%s.py" % (directory, name)
+  py_template = jenv.get_template(TEMPLATE_NAME)
+  module_path = ".".join(package_spec.identifier.split(".")[1:-1])
+  includes = [".".join(i.split(".")[:-1]) for i in package_spec.includes]
+  includes = [i for i in includes if i != "types"]
+  print(destination_filename, includes)
+  with open(destination_filename, 'w') as f:
+    f.write(py_template.render(msgs=package_spec.definitions,
+                               filepath="/".join(package_spec.filepath) + ".yaml",
+                               module_path=module_path,
+                               include=includes,
+                               timestamp=package_spec.creation_timestamp,
+                               description=package_spec.description))