Python3 support

Add python3 support to ubpf-(dis)assembler.py
Also add python3 test in CI.

Signed-off-by: Masanori Misono <m.misono760@gmail.com>

.travis.yml

@@ -1,15 +1,27 @@
 language: cpp
 compiler:
   - gcc
-python:
-  - "2.7"
+os: linux
+dist: xenial
+jobs:
+  include:
+    - name: python 2.7
+      env: PYTHON=python2
+      before_install:
+        - sudo apt-get update
+        - sudo apt-get -y install python python-pip python-setuptools python-wheel
+      after_success:
+        - coveralls --gcov-options '\-lp' -i $PWD/vm/ubpf_vm.c -i $PWD/vm/ubpf_jit_x86_64.c -i $PWD/vm/ubpf_loader.c
+    - name: python 3.5
+      env: PYTHON=python3
+      before_install:
+        - sudo apt-get update
+        - sudo apt-get -y install python3 python3-pip python3-setuptools python3-wheel
 # command to install dependencies
 install: 
-  - pip install --user -r requirements.txt
-  - pip install --user cpp-coveralls
+  - $PYTHON -m pip install --user -r requirements.txt
+  - $PYTHON -m pip install --user cpp-coveralls
 # command to run tests
 script: 
   - make -C vm COVERAGE=1
   - nosetests -v
-after_success:
-  - coveralls --gcov-options '\-lp' -i $PWD/vm/ubpf_vm.c -i $PWD/vm/ubpf_jit_x86_64.c -i $PWD/vm/ubpf_loader.c

bin/ubpf-assembler

@@ -19,11 +19,15 @@ import ubpf.assembler
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument('input', type=argparse.FileType('r'), default='-')
-    parser.add_argument('output', type=argparse.FileType('w'), default='-')
+    parser.add_argument('input', type=argparse.FileType('r'), default='-', nargs='?')
+    parser.add_argument('output', type=argparse.FileType('wb'), default='-', nargs='?')
     args = parser.parse_args()
 
-    args.output.write(ubpf.assembler.assemble(args.input.read()))
+    if args.output.name == "<stdout>" and hasattr(args.output, "buffer"):
+        # python 3
+        args.output.buffer.write(ubpf.assembler.assemble(args.input.read()))
+    else:
+        args.output.write(ubpf.assembler.assemble(args.input.read()))
 
 if __name__ == "__main__":
     main()

bin/ubpf-disassembler

@@ -19,11 +19,17 @@ import ubpf.disassembler
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument('input', type=argparse.FileType('r'), default='-')
-    parser.add_argument('output', type=argparse.FileType('w'), default='-')
+    parser.add_argument('input', type=argparse.FileType('rb'), default='-', nargs='?')
+    parser.add_argument('output', type=argparse.FileType('w'), default='-', nargs='?')
     args = parser.parse_args()
 
-    args.output.write(ubpf.disassembler.disassemble(args.input.read()))
+    if args.input.name == "<stdin>" and hasattr(args.input, "buffer"):
+        # python 3
+        input_ = args.input.buffer.read()
+    else:
+        input_ = args.input.read()
+
+    args.output.write(ubpf.disassembler.disassemble(input_))
 
 if __name__ == "__main__":
     main()

requirements.txt

@@ -1,3 +1,4 @@
-parcon ~= 0.1.25
+# pypi version of parcon does not support python3
+git+https://github.com/javawizard/parcon
 nose ~= 1.3.1
 pyelftools ~= 0.23

test_framework/test_assembler.py

@@ -3,6 +3,10 @@
 import ubpf.assembler
 import ubpf.disassembler
 import testdata
+try:
+    xrange
+except NameError:
+    xrange = range
 
 # Just for assertion messages
 def try_disassemble(inst):
@@ -28,7 +32,7 @@ def check_datafile(filename):
     assert len(bin_result) / 8 == len(data['raw'])
 
     for i in xrange(0, len(bin_result), 8):
-        j = i/8
+        j = int(i/8)
         inst, = struct.unpack_from("=Q", bin_result[i:i+8])
         exp = data['raw'][j]
         if exp != inst:

test_framework/test_disassembler.py

@@ -15,7 +15,7 @@ def check_datafile(filename):
     if 'raw' not in data:
         raise SkipTest("no raw section in datafile")
 
-    binary = ''.join(struct.pack("=Q", x) for x in data['raw'])
+    binary = b''.join(struct.pack("=Q", x) for x in data['raw'])
     result = ubpf.disassembler.disassemble(binary)
 
     # TODO strip whitespace and comments from asm

test_framework/test_elf.py

@@ -111,7 +111,7 @@ def add(name, value):
     text = ubpf.assembler.assemble(asm)
 
     add("text", text)
-    add("strtab", "\0.text\0.strtab\0.symtab\0.rel\0sqrti\0")
+    add("strtab", b"\0.text\0.strtab\0.symtab\0.rel\0sqrti\0")
     add("first_sym", Container(
         st_name=0,
         st_value=0,
@@ -144,7 +144,7 @@ def serialize(parts):
 
     for name in parts['order']:
         part = parts[name]
-        serializer = str
+        serializer = lambda x: x
         if name == 'ehdr':
             serializer = s.Elf_Ehdr.build
         elif name.endswith('shdr'):
@@ -153,12 +153,13 @@ def serialize(parts):
             serializer = s.Elf_Rel.build
         elif name.endswith('sym'):
             serializer = s.Elf_Sym.build
+
         data = serializer(part)
         tmp.append(data)
         #sys.stderr.write("Wrote %s size %d at offset %d\n" % (name, len(data), offset))
         offset += len(data)
 
-    return ''.join(tmp)
+    return b''.join(tmp)
 
 def generate_elf(pyelf):
     parts = template()
@@ -185,6 +186,8 @@ def check_datafile(filename):
     vm = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
 
     stdout, stderr = vm.communicate(elf)
+    stdout = stdout.decode("utf-8")
+    stderr = stderr.decode("utf-8")
     stderr = stderr.strip()
 
     if 'error' in data:

test_framework/test_jit.py

@@ -7,6 +7,10 @@
 import ubpf.assembler
 import testdata
 VM = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "vm", "test")
+try:
+    xrange
+except NameError:
+    xrange = range
 
 def check_datafile(filename):
     """
@@ -24,7 +28,7 @@ def check_datafile(filename):
         raise SkipTest("JIT disabled for this testcase (%s)" % data['no jit'])
 
     if 'raw' in data:
-        code = ''.join(struct.pack("=Q", x) for x in data['raw'])
+        code = b''.join(struct.pack("=Q", x) for x in data['raw'])
     else:
         code = ubpf.assembler.assemble(data['asm'])
 
@@ -50,6 +54,8 @@ def check_datafile(filename):
             vm = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
 
             stdout, stderr = vm.communicate(code)
+            stdout = stdout.decode("utf-8")
+            stderr = stderr.decode("utf-8")
             stderr = stderr.strip()
 
             if 'error' in data:

test_framework/test_vm.py

@@ -22,7 +22,7 @@ def check_datafile(filename):
         raise SkipTest("VM not found")
 
     if 'raw' in data:
-        code = ''.join(struct.pack("=Q", x) for x in data['raw'])
+        code = b''.join(struct.pack("=Q", x) for x in data['raw'])
     else:
         code = ubpf.assembler.assemble(data['asm'])
 
@@ -40,6 +40,8 @@ def check_datafile(filename):
     vm = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
 
     stdout, stderr = vm.communicate(code)
+    stdout = stdout.decode("utf-8")
+    stderr = stderr.decode("utf-8")
     stderr = stderr.strip()
 
     if memfile:

test_framework/testdata.py

@@ -41,7 +41,8 @@ def read(name):
                 section_lines[cur_section] = []
             elif cur_section:
                 section_lines[cur_section].append(line)
-    data = { section: '\n'.join(lines) for (section, lines) in section_lines.items() }
+    data = { section: '\n'.join(lines) for (section, lines) in list(section_lines.items()) }
+
 
     # Resolve links
     for k in data:
@@ -51,7 +52,7 @@ def read(name):
             section = section.strip()
             path = path.strip()
             fullpath = os.path.join(_test_data_dir, os.path.dirname(name), path)
-            with file(fullpath) as f:
+            with open(fullpath) as f:
                 data[section] = f.read()
 
     # Special case: convert 'raw' section into binary
@@ -72,6 +73,6 @@ def read(name):
             if ':' in line:
                 line = line[(line.rindex(':')+1):]
             hex_strs.extend(re.findall(r"[0-9A-Fa-f]{2}", line))
-        data['mem'] = ''.join(map(lambda x: chr(int(x, 16)), hex_strs))
+        data['mem'] = bytes(bytearray([(int(x, 16)) for x in hex_strs]))
 
     return data

tests/elf/ehdr-short.data

@@ -1,6 +1,6 @@
 -- pyelf
 del order[:]
-magic = '\x7fELF'
+magic = b'\x7fELF'
 order.append('magic')
 -- error
 Failed to load code: not enough data for ELF header

ubpf/asm_parser.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from parcon import *
 from collections import namedtuple
 
@@ -60,4 +61,4 @@ def parse(source):
     args = parser.parse_args()
     result = parse(args.file.read())
     for inst in result:
-        print repr(inst)
+        print(repr(inst))

ubpf/assembler.py

@@ -1,6 +1,9 @@
-from asm_parser import parse, Reg, Imm, MemRef
+from .asm_parser import parse, Reg, Imm, MemRef
 import struct
-import StringIO
+try:
+    from StringIO import StringIO as io
+except ImportError:
+    from io import BytesIO as io
 
 Inst = struct.Struct("BBHI")
 
@@ -11,9 +14,9 @@
     'dw': 3,
 }
 
-MEM_LOAD_OPS = { 'ldx' + k: (0x61 | (v << 3)) for k, v in MEM_SIZES.items() }
-MEM_STORE_IMM_OPS = { 'st' + k: (0x62 | (v << 3))  for k, v in MEM_SIZES.items() }
-MEM_STORE_REG_OPS = { 'stx' + k: (0x63 | (v << 3)) for k, v in MEM_SIZES.items() }
+MEM_LOAD_OPS = { 'ldx' + k: (0x61 | (v << 3)) for k, v in list(MEM_SIZES.items()) }
+MEM_STORE_IMM_OPS = { 'st' + k: (0x62 | (v << 3))  for k, v in list(MEM_SIZES.items()) }
+MEM_STORE_REG_OPS = { 'stx' + k: (0x63 | (v << 3)) for k, v in list(MEM_SIZES.items()) }
 
 UNARY_ALU_OPS = {
     'neg': 8,
@@ -34,8 +37,8 @@
     'arsh': 12,
 }
 
-UNARY_ALU32_OPS = { k + '32': v for k, v in UNARY_ALU_OPS.items() }
-BINARY_ALU32_OPS = { k + '32': v for k, v in BINARY_ALU_OPS.items() }
+UNARY_ALU32_OPS = { k + '32': v for k, v in list(UNARY_ALU_OPS.items()) }
+BINARY_ALU32_OPS = { k + '32': v for k, v in list(BINARY_ALU_OPS.items()) }
 
 END_OPS = {
     'le16': (0xd4, 16),
@@ -119,7 +122,7 @@ def assemble_one(inst):
 
 def assemble(source):
     insts = parse(source)
-    output = StringIO.StringIO()
+    output = io()
     for inst in insts:
         output.write(assemble_one(inst))
     return output.getvalue()

ubpf/disassembler.py

@@ -1,5 +1,8 @@
 import struct
-import StringIO
+try:
+    from StringIO import StringIO as io
+except ImportError:
+    from io import StringIO as io
 
 Inst = struct.Struct("BBHI")
 
@@ -155,7 +158,7 @@ def disassemble_one(data, offset):
         return "unknown instruction %#x" % code
 
 def disassemble(data):
-    output = StringIO.StringIO()
+    output = io()
     offset = 0
     while offset < len(data):
         s = disassemble_one(data, offset)