python3 WIP (#674)

* python3: parens around single-line print statements

* python3: subprocess.check_output() => subprocess.getoutput()

* python3: 2to3

* python3: don't use relative protos, generate from root and include relative to root. Delete string.decode(), will fix with future unicode_literal later.

* Python3: PhaseExecutionOutcome has __new__ and not __init__ since named tuples are immutable

* Python3: Initialize TestRecord.start_time_millis to 0. In PlugsTest.test_initialize, only compare the knowable values of the plug_manager dictionary.

* Python3: self.join fails with float_info.max, replace with 5 seconds (it seems to be a placeholder value). Encode the test_attachment string as utf-8

* Python3: CodeInfo must derive from HashableRecord since it's held in a dict.

* Python3: Don't utf8-encode the unit suffixes

* Python3: explicitly encode value_binary fields as utf-8

* Python3: BytesIO instead of StringIO in tests.

* Python3: Store attachments as unicode strings containing base64-encoded payloads (used to be byte strings of b64 payloads). Use a StringIO for the test that needs it.

* Python3: Remove 2to3 unnecessary double-parentheses around print calls.

* Python3: Point Travis at Python 3.6

* Python3: Require protobuf 3.4.0, exploring Travis failures

* Python3: Require protobuf package >3, explicitly download + unpack protoc > 3. From Brandon Wallace (wallacbe)

.travis.yml

@@ -1,23 +1,31 @@
 language: python
 python:
-- '2.7'
+  # - '2.7'
+  - '3.6'
 addons:
   apt:
     packages:
+    - python3
+    - python3-pip
     - swig
     - libusb-1.0-0-dev
     - libprotobuf-dev
-    - protobuf-compiler
+      #    - protobuf-compiler
 cache:
   pip: true
   apt: true
   directories:
   - $HOME/.cache/pip
 env:
-- TOXENV=py27
+  #- TOXENV=py27
+- TOXENV=py36
 install:
 - pip install tox coveralls
 - pip install -e .
+- wget https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
+- unzip protoc-3.5.0-linux-x86_64.zip
+- sudo cp bin/protoc /usr/bin/protoc && sudo chmod 777 /usr/bin/protoc
+- protoc --version
 script: tox
 after_success:
 - coveralls

bin/units_from_xls.py

@@ -158,10 +158,10 @@ def __call__(self, name_or_suffix):
                          '15': 'FIFTEEN',
                          '30': 'THIRTY',
                          '\\': '_',
-                         unichr(160): '_',
-                         unichr(176): 'DEG_',
-                         unichr(186): 'DEG_',
-                         unichr(8211): '_',
+                         chr(160): '_',
+                         chr(176): 'DEG_',
+                         chr(186): 'DEG_',
+                         chr(8211): '_',
                         }
 
 
@@ -182,7 +182,7 @@ def main():
   args = parser.parse_args()
 
   if not os.path.exists(args.xlsfile):
-    print 'Unable to locate the file "%s".' % args.xlsfile
+    print('Unable to locate the file "%s".' % args.xlsfile)
     parser.print_help()
     sys.exit()
 
@@ -217,7 +217,7 @@ def unit_defs_from_sheet(sheet, column_names):
     rows = sheet.get_rows()
 
     # Find the indices for the columns we care about.
-    for idx, cell in enumerate(rows.next()):
+    for idx, cell in enumerate(next(rows)):
       if cell.value in column_names:
         col_indices[cell.value] = idx
 
@@ -245,7 +245,7 @@ def unit_key_from_name(name):
   """Return a legal python name for the given name for use as a unit key."""
   result = name
 
-  for old, new in UNIT_KEY_REPLACEMENTS.iteritems():
+  for old, new in UNIT_KEY_REPLACEMENTS.items():
     result = result.replace(old, new)
 
   # Collapse redundant underscores and convert to uppercase.

contrib/poll_stations.py

@@ -32,7 +32,7 @@
 
 import logging
 import os
-import Queue
+import queue
 import socket
 import sys
 import threading
@@ -111,7 +111,7 @@ class StationList(object):
   """
 
   def __init__(self):
-    self.update_queue = Queue.Queue()
+    self.update_queue = queue.Queue()
     self.stations = set()
     # Really, these threads should be tracked on a per-station basis, because
     # two stations *could* have RemoteTest instances that would compare equal
@@ -168,7 +168,7 @@ def watch_test(self, remote_test):
   def print_station(self, station):
     print(station)
     try:
-      for remote_test in station.tests.itervalues():
+      for remote_test in station.tests.values():
         # Trigger an update of the local history cache and state.
         remote_test.state
         remote_test.history
@@ -182,7 +182,7 @@ def print_station(self, station):
           self.update_threads[remote_test] = update_thread
     except socket.error as e:
       print(' |-- Connection Error: %s' % e)
-    print
+    print()
 
   def check_for_stations(self):
     """Discover for new stations, doesn't remove any stations."""
@@ -199,7 +199,7 @@ def mainloop(self):
       station_list.check_for_stations()
       try:
         self.update(self.update_queue.get(timeout=5))
-      except Queue.Empty:
+      except queue.Empty:
         pass
 
 

examples/all_the_things.py

@@ -28,7 +28,7 @@
 from openhtf.output import callbacks
 from openhtf.output.callbacks import json_factory
 
-import example_plugs
+from . import example_plugs
 
 
 @htf.plug(example=example_plugs.ExamplePlug)
@@ -89,7 +89,7 @@ def set_measurements(test):
 def dimensions(test):
   for dim in range(5):
     test.measurements.dimensions[dim] = 1 << dim
-  for x, y, z in zip(range(1, 5), range(21, 25), range (101, 105)):
+  for x, y, z in zip(list(range(1, 5)), list(range(21, 25)), list(range(101, 105))):
     test.measurements.lots_of_dims[x, y, z] = x + y + z
 
 
@@ -105,7 +105,7 @@ def measures_with_args(test, min, max):
 
 
 def attachments(test):
-  test.attach('test_attachment', 'This is test attachment data.')
+  test.attach('test_attachment', 'This is test attachment data.'.encode('utf-8'))
   test.attach_from_file(
       os.path.join(os.path.dirname(__file__), 'example_attachment.txt'))
 

examples/repeat.py

@@ -40,7 +40,7 @@ def __init__(self):
   def run(self):
     """Increments counter and raises an exception for first two runs."""
     self.count += 1
-    print 'FailTwicePlug: Run number %s' % (self.count)
+    print('FailTwicePlug: Run number %s' % (self.count))
     if self.count < 3:
       raise RuntimeError('Fails a couple times')
 
@@ -56,7 +56,7 @@ def __init__(self):
   def run(self):
     """Increments counter and returns False indicating failure"""
     self.count += 1
-    print "FailAlwaysPlug: Run number %s" % (self.count)
+    print("FailAlwaysPlug: Run number %s" % (self.count))
 
     return False
 
@@ -70,10 +70,10 @@ def phase_repeat(test, test_plug):
     test_plug.run()
 
   except:
-    print "Error in phase_repeat, will retry"
+    print("Error in phase_repeat, will retry")
     return openhtf.PhaseResult.REPEAT
 
-  print "Completed phase_repeat"
+  print("Completed phase_repeat")
 
 
 # This phase demonstrates repeating a phase based upon a result returned from a
@@ -86,7 +86,7 @@ def phase_repeat_with_limit(test, test_plug):
   result = test_plug.run()
 
   if not result:
-    print "Invalid result in phase_repeat_with_limit, will retry"
+    print("Invalid result in phase_repeat_with_limit, will retry")
     return openhtf.PhaseResult.REPEAT
 
 if __name__ == '__main__':

examples/with_plugs.py

@@ -52,7 +52,7 @@ def _get_command(self, count):
 
   def run(self, count):
     command = self._get_command(count)
-    print "running: %s" % ' '.join(command)
+    print("running: %s" % ' '.join(command))
     return subprocess.call(command)
 
 

openhtf/__init__.py

@@ -192,14 +192,14 @@ def configure(self, **kwargs):
     # side effects.
     create_arg_parser(add_help=True).parse_known_args()
     logs.setup_logger()
-    for key, value in kwargs.iteritems():
+    for key, value in kwargs.items():
       setattr(self._test_options, key, value)
 
   @classmethod
   def handle_sig_int(cls, *_):
     if cls.TEST_INSTANCES:
       _LOG.error('Received SIGINT, stopping all tests.')
-      for test in cls.TEST_INSTANCES.values():
+      for test in list(cls.TEST_INSTANCES.values()):
         test.stop_from_sig_int()
     station_api.stop_server()
     # The default SIGINT handler does this. If we don't, then nobody above
@@ -390,7 +390,7 @@ def format_strings(self, **kwargs):
         self, name=util.format_string(self.name, kwargs))
 
   def update(self, **kwargs):
-    for key, value in kwargs.iteritems():
+    for key, value in kwargs.items():
       if key not in self.__slots__:
         raise AttributeError('Type %s does not have attribute %s' % (
             type(self).__name__, key))
@@ -481,7 +481,7 @@ def with_plugs(self, **subplugs):
     plugs_by_name = {plug.name: plug for plug in self.plugs}
     new_plugs = dict(plugs_by_name)
 
-    for name, sub_class in subplugs.iteritems():
+    for name, sub_class in subplugs.items():
       original_plug = plugs_by_name.get(name)
       if (original_plug is None
           or not isinstance(original_plug.cls, plugs.PlugPlaceholder)
@@ -493,7 +493,7 @@ def with_plugs(self, **subplugs):
 
     return mutablerecords.CopyRecord(
         self,
-        plugs=new_plugs.values(),
+        plugs=list(new_plugs.values()),
         options=self.options.format_strings(**subplugs),
         measurements=[m.with_args(**subplugs) for m in self.measurements])
 

openhtf/core/__init__.py

@@ -1 +1 @@
-from test_executor import TestExecutionError, TestStopError, TestExecutor
+from .test_executor import TestExecutionError, TestStopError, TestExecutor

openhtf/core/measurements.py

@@ -198,7 +198,7 @@ def with_dimensions(self, *dimensions):
 
   def with_validator(self, validator):
     """Add a validator callback to this Measurement, chainable."""
-    if not callable(validator):
+    if not isinstance(validator, collections.Callable):
       raise ValueError('Validator must be callable', validator)
     self.validators.append(validator)
     return self
@@ -325,7 +325,7 @@ def is_value_set(self):
 
   def __iter__(self):  # pylint: disable=invalid-name
     """Iterate over items, allows easy conversion to a dict."""
-    return self.value_dict.iteritems()
+    return iter(self.value_dict.items())
 
   def __setitem__(self, coordinates, value):  # pylint: disable=invalid-name
     coordinates_len = len(coordinates) if hasattr(coordinates, '__len__') else 1
@@ -366,7 +366,7 @@ def value(self):
     if not self.is_value_set:
       raise MeasurementNotSetError('Measurement not yet set', self.name)
     return [dimensions + (value,) for dimensions, value in
-            self.value_dict.iteritems()]
+            self.value_dict.items()]
 
 
 class Collection(mutablerecords.Record('Collection', ['_measurements'])):
@@ -421,7 +421,7 @@ def _assert_valid_key(self, name):
   def __iter__(self):  # pylint: disable=invalid-name
     """Extract each MeasurementValue's value."""
     return ((key, meas.measured_value.value)
-            for key, meas in self._measurements.iteritems())
+            for key, meas in self._measurements.items())
 
   def __setattr__(self, name, value):  # pylint: disable=invalid-name
     self[name] = value
@@ -469,7 +469,7 @@ def _maybe_make(meas):
     """Turn strings into Measurement objects if necessary."""
     if isinstance(meas, Measurement):
       return meas
-    elif isinstance(meas, basestring):
+    elif isinstance(meas, str):
       return Measurement(meas, **kwargs)
     raise InvalidMeasurementType('Expected Measurement or string', meas)
 

openhtf/core/phase_executor.py

@@ -91,12 +91,13 @@ class PhaseExecutionOutcome(collections.namedtuple(
   other value will raise an InvalidPhaseResultError.
   """
 
-  def __init__(self, phase_result):
+  def __new__(cls, phase_result):
     if (phase_result is not None and
         not isinstance(phase_result, (openhtf.PhaseResult, ExceptionInfo)) and
         not isinstance(phase_result, threads.ThreadTerminationError)):
       raise InvalidPhaseResultError('Invalid phase result', phase_result)
-    super(PhaseExecutionOutcome, self).__init__(phase_result)
+    self = super(PhaseExecutionOutcome, cls).__new__(cls, phase_result)
+    return self
 
   @property
   def is_fail_and_continue(self):
@@ -208,7 +209,7 @@ def execute_phase(self, phase):
       hit its limit for repetitions.
     """
     repeat_count = 1
-    repeat_limit = phase.options.repeat_limit or sys.maxint
+    repeat_limit = phase.options.repeat_limit or sys.maxsize
     while not self._stopping.is_set():
       is_last_repeat = repeat_count >= repeat_limit
       phase_execution_outcome = self._execute_phase_once(phase, is_last_repeat)
@@ -260,7 +261,7 @@ def stop(self, timeout_s=None):
 
     if phase_thread.is_alive():
       phase_thread.kill()
-    
+
       _LOG.debug('Waiting for cancelled phase to exit: %s', phase_thread)
       timeout = timeouts.PolledTimeout.from_seconds(timeout_s)
       while phase_thread.is_alive() and not timeout.has_expired():

openhtf/core/station_api.py

@@ -69,7 +69,7 @@
 import socket
 import threading
 import time
-import xmlrpclib
+import xmlrpc.client
 
 import mutablerecords
 
@@ -86,7 +86,7 @@
 
 # Fix for xmlrpclib to use <i8> for longs and ints instead of <int>, because our
 # timestamps are in millis, which are too big for 4-byte ints.
-xmlrpclib.Marshaller.dispatch[long] = xmlrpclib.Marshaller.dispatch[int] = (
+xmlrpc.client.Marshaller.dispatch[int] = xmlrpc.client.Marshaller.dispatch[int] = (
     lambda _, v, w: w('<value><i8>%d</i8></value>' % v))
 
 _LOG = logging.getLogger(__name__)
@@ -408,7 +408,7 @@ def wait_for_update(self, timeout_s=1):
     try:
       remote_state_dict = self.proxy_factory(timeout_s + 1).wait_for_update(
           self.test_uid, summary_dict, timeout_s)
-    except xmlrpclib.Fault as fault:
+    except xmlrpc.client.Fault as fault:
       # TODO(madsci): This is a super kludge, eventually implement the
       # ReraisingMixin for ServerProxy, but that's hard, so do this for now.
       if 'openhtf.io.station_api.UpdateTimeout' in fault.faultString:
@@ -453,7 +453,7 @@ def history(self):
     """
     last_start_time = self._cached_history.last_start_time(self.test_uid)
     new_history = self.shared_proxy.get_history_after(
-        self.test_uid, long(last_start_time))
+        self.test_uid, int(last_start_time))
     _LOG.debug('Requested history update for %s after %s, got %s results.',
                self.test_uid, last_start_time, len(new_history))
 
@@ -680,10 +680,10 @@ def list_tests(self):
     retval = [{
         'test_uid': test.uid,
         'test_name': test.get_option('name'),
-        'created_time_millis': long(test.created_time_millis),
+        'created_time_millis': int(test.created_time_millis),
         'last_run_time_millis':
-            test.last_run_time_millis and long(test.last_run_time_millis),
-    } for test in openhtf.Test.TEST_INSTANCES.values() if test.uid is not None]
+            test.last_run_time_millis and int(test.last_run_time_millis),
+    } for test in list(openhtf.Test.TEST_INSTANCES.values()) if test.uid is not None]
     _LOG.debug('RPC:list_tests() -> %s results', len(retval))
     return retval
 
@@ -770,7 +770,7 @@ def wait_for_plug_update(self, test_uid, plug_name, current_state, timeout_s):
   def _summary_for_state_dict(state_dict):
     """Return a dict for state with counts swapped in for phase/log records."""
     state_dict_summary = {
-        k: v for k, v in state_dict.iteritems() if k != 'plugs'}
+        k: v for k, v in state_dict.items() if k != 'plugs'}
     state_dict_summary['test_record'] = data.convert_to_base_types(
         state_dict_summary['test_record'])
     state_dict_summary['test_record']['phases'] = len(

openhtf/core/test_executor.py

@@ -109,7 +109,7 @@ def finalize(self):
   def wait(self):
     """Waits until death."""
     try:
-      self.join(sys.float_info.max)  # Timeout needed for SIGINT handling.
+      self.join(5)  # Timeout needed for SIGINT handling.
     except KeyboardInterrupt:
       self.test_state.logger.info('KeyboardInterrupt caught, aborting test.')
       raise