First pass at Python 3 compatibility.

All but one 'not bdd' test now passes. That error is only a test artefact due to gawel/WSGIProxy2#7.
Needed to remove aws as that is blocked by gevent. It can be moved to a separate buildout or virtualenv.

buildout.cfg

@@ -24,9 +24,10 @@ parts =
     grunt
     compile-js
     test
-    aws
     generate-ontology
-develop = .
+develop =
+    .
+    jsonschema
 sources-dir = develop
 auto-checkout =
 
@@ -65,7 +66,6 @@ eggs =
     waitress
     psycopg2
     repoze.debug
-    subprocess32
     rutter
     wsgi-request-logger
 

setup.py

@@ -13,6 +13,7 @@
     'WebTest',
     'boto',
     'elasticsearch',
+    'future',
     'humanfriendly',
     'jsonschema',
     'loremipsum',

src/encoded/__init__.py

@@ -1,5 +1,8 @@
+from future.standard_library import install_aliases
+install_aliases()
 import base64
 import json
+import os
 try:
     import subprocess32 as subprocess  # Closes pipes on failure
 except ImportError:
@@ -119,7 +122,7 @@ def session(config):
             secret = open(secret).read()
             secret = base64.b64decode(secret)
     else:
-        secret = open('/dev/urandom').read(256)
+        secret = os.urandom(256)
     # auth_tkt has no timeout set
     # cookie will still expire at browser close
     if 'session.timeout' in settings:

src/encoded/audit/item.py

@@ -1,3 +1,4 @@
+from past.builtins import basestring
 from ..auditor import (
     AuditFailure,
     audit_checker,

src/encoded/auditor.py

@@ -4,6 +4,7 @@
 We also need to perform higher order checking between linked objects.
 """
 
+from past.builtins import basestring
 import logging
 import venusian
 from .embedding import embed
@@ -69,7 +70,7 @@ def __init__(self):
     def add_audit_checker(self, checker, item_type, condition=None, frame='embedded'):
         checkers = self.type_checkers.setdefault(item_type, [])
         self._order += 1  # consistent execution ordering
-        if not isinstance(frame, basestring):
+        if isinstance(frame, list):
             frame = tuple(sorted(frame))
         checkers.append((self._order, checker, condition, frame))
 

src/encoded/authentication.py

@@ -62,8 +62,7 @@ def __new__(cls, namespace, base, *args, **kw):
         # Dynamically create a subclass
         name = 'Namespaced_%s_%s' % (namespace, base.__name__)
         klass = type(name, (cls, base), {'_namespace_prefix': namespace + '.'})
-        return super(NamespacedAuthenticationPolicy, klass) \
-            .__new__(klass, *args, **kw)
+        return super(NamespacedAuthenticationPolicy, klass).__new__(klass)
 
     def __init__(self, namespace, base, *args, **kw):
         super(NamespacedAuthenticationPolicy, self).__init__(*args, **kw)
@@ -123,7 +122,7 @@ def generate_user():
     # Take a random 5 char binary string (80 bits of
     # entropy) and encode it as upper cased base32 (8 chars)
     random_bytes = os.urandom(5)
-    user = base64.b32encode(random_bytes).rstrip('=').upper()
+    user = base64.b32encode(random_bytes).decode('ascii').rstrip('=').upper()
     return user
 
 
@@ -133,5 +132,5 @@ def generate_password():
     # Take a random 10 char binary string (80 bits of
     # entropy) and encode it as lower cased base32 (16 chars)
     random_bytes = os.urandom(10)
-    password = base64.b32encode(random_bytes).rstrip('=').lower()
+    password = base64.b32encode(random_bytes).decode('ascii').rstrip('=').lower()
     return password

src/encoded/commands/add_date_created.py

@@ -12,6 +12,7 @@
     %(prog)s development.ini --app-name app
 
 """
+from future.utils import iteritems
 from pyramid.traversal import resource_path
 import logging
 import pytz
@@ -53,7 +54,7 @@ def run(testapp, collections=None, exclude=None, dry_run=False):
         count = 0
         errors = 0
         logger.info('Upgrading %s', collection_name)
-        for uuid, item in collection.iteritems():
+        for uuid, item in iteritems(collection):
             history = item.model.data[''].history
             first_propsheet = history[0]
             if 'date_created' in first_propsheet.properties:

src/encoded/commands/check_files.py

@@ -9,7 +9,7 @@
 import boto
 import requests
 import sys
-from urlparse import urljoin
+from urllib.parse import urljoin
 
 EPILOG = __doc__
 
@@ -121,7 +121,7 @@ def check_format(item, path):
 def check_file(item):
     import hashlib
     import os.path
-    from urlparse import urlparse
+    from urllib.parse import urlparse
     result = None
     errors = {}
     r = requests.head(

src/encoded/commands/check_rendering.py

@@ -12,6 +12,7 @@
 """
 import json
 import logging
+from future.utils import itervalues
 from pyramid.traversal import resource_path
 
 EPILOG = __doc__
@@ -48,7 +49,7 @@ def run(testapp, collections=None):
         collection_path = resource_path(collection, '')
         check_path(testapp, collection_path)
         failed = 0
-        for count, item in enumerate(collection.itervalues()):
+        for count, item in enumerate(itervalues(collection)):
             path = resource_path(item, '')
             if not check_path(testapp, path):
                 failed += 1

src/encoded/commands/create_mapping.py

@@ -51,7 +51,7 @@ def schema_mapping(name, schema):
 
     if type_ == 'object':
         properties = {}
-        all_props = schema['properties'].items() + schema.get('calculated_props', {}).items()
+        all_props = list(schema['properties'].items()) + list(schema.get('calculated_props', {}).items())
         for k, v in all_props:
             mapping = schema_mapping(k, v)
             if mapping is not None:
@@ -343,7 +343,7 @@ def run(app, collections=None, dry_run=False):
                 es.indices.create(index=index, body=index_settings())
 
     if not collections:
-        collections = ['meta'] + root.by_item_type.keys()
+        collections = ['meta'] + list(root.by_item_type.keys())
 
     for collection_name in collections:
         if collection_name == 'meta':
@@ -357,8 +357,7 @@ def run(app, collections=None, dry_run=False):
         if mapping is None:
             continue  # Testing collections
         if dry_run:
-            print json.dumps(
-                sorted_dict({index: {doc_type: mapping}}), indent=4)
+            print(json.dumps(sorted_dict({index: {doc_type: mapping}}), indent=4))
             continue
 
         if collection_name is not 'meta':

src/encoded/commands/deploy.py

@@ -52,19 +52,18 @@ def run(wale_s3_prefix, branch=None, name=None, persistent=False):
     instance.add_tag('Name', name)
     instance.add_tag('commit', commit)
     instance.add_tag('started_by', username)
-    print instance
-    print instance.state,
+    print(instance)
+    sys.stdout.write(instance.state)
 
     while instance.state == 'pending':
         sys.stdout.write('.')
         sys.stdout.flush()
         time.sleep(1)
         instance.update()
 
-    print
-    print instance.state
-
-    print instance.public_dns_name  # u'ec2-54-219-26-167.us-west-1.compute.amazonaws.com'
+    print()
+    print(instance.state)
+    print(instance.public_dns_name)  # u'ec2-54-219-26-167.us-west-1.compute.amazonaws.com'
 
 
 def main():

src/encoded/commands/dev_servers.py

@@ -12,6 +12,7 @@
 import os.path
 import select
 import shutil
+import sys
 
 EPILOG = __doc__
 
@@ -60,7 +61,7 @@ def cleanup_process():
         for process in processes:
             try:
                 for line in process.stdout:
-                    print line,
+                    sys.stdout.write(line)
             except IOError:
                 pass
             process.wait()
@@ -83,7 +84,7 @@ def cleanup_process():
         docsdir = [resource_filename('encoded', 'tests/data/documents/')]
         load_all(testapp, inserts, docsdir)
 
-    print 'Started. ^C to exit.'
+    print('Started. ^C to exit.')
 
     stdouts = [p.stdout for p in processes]
 
@@ -92,11 +93,11 @@ def cleanup_process():
         readable, writable, err = select.select(stdouts, [], stdouts, 5)
         for stdout in readable:
             for line in iter(stdout.readline, ''):
-                print line,
+                sys.stdout.write(line)
         if err:
             for stdout in err:
                 for line in iter(stdout.readline, ''):
-                    print line,
+                    sys.stdout.write(line)
             break
 
 if __name__ == '__main__':

src/encoded/commands/es_index_listener.py

@@ -20,7 +20,7 @@
 import sqlalchemy.exc
 import threading
 import time
-import urlparse
+from urllib.parse import parse_qsl
 
 log = logging.getLogger(__name__)
 
@@ -91,7 +91,7 @@ def run(testapp, timeout=DEFAULT_TIMEOUT, dry_run=False, control=None, update_st
                         log.debug(res.json)
                     result = res.json
                     result['stats'] = {
-                        k: int(v) for k, v in urlparse.parse_qsl(
+                        k: int(v) for k, v in parse_qsl(
                             res.headers.get('X-Stats', ''))
                     }
                     result['timestamp'] = timestamp

src/encoded/commands/extract_test_data.py

@@ -21,7 +21,7 @@ def __init__(self):
 
     def replace_emails(self, dictrows):
         for row in dictrows:
-            for k, v in list(row.iteritems()):
+            for k, v in list(row.items()):
                 if v is None:
                     continue
                 new_value, num_subs = self.email_re.subn(
@@ -39,7 +39,7 @@ def replace_non_pi_names(self, dictrows):
             yield row
 
     def _random_email(self):
-        for _ in xrange(1000):
+        for _ in range(1000):
             generated = "%s.%s@%s.%s" % \
                 tuple(random.choice(self.random_words) for n in range(4))
             if generated not in self.generated_emails:
@@ -60,7 +60,7 @@ def _replace_emails(self, matchobj):
         return new
 
     def _random_name(self):
-        for _ in xrange(1000):
+        for _ in range(1000):
             if random.choice(range(4)):
                 generated = random.choice(self.random_words).capitalize()
             else:
@@ -76,7 +76,7 @@ def _random_name(self):
 def set_existing_key_value(**kw):
     def component(dictrows):
         for row in dictrows:
-            for k, v in kw.iteritems():
+            for k, v in kw.items():
                 if k in row:
                     row[k] = v
             yield row
@@ -87,7 +87,7 @@ def component(dictrows):
 def drop_rows_with_all_key_value(**kw):
     def component(dictrows):
         for row in dictrows:
-            if not all(row[k] == v if k in row else False for k, v in kw.iteritems()):
+            if not all(row[k] == v if k in row else False for k, v in kw.items()):
                 yield row
 
     return component

src/encoded/commands/generate_ontology.py

@@ -313,7 +313,7 @@ def get_last_bit(uri_string):
         return sorted(uri_list, key=lambda x: get_last_bit(x.__str__()))
     except:
         # TODO: do more testing.. maybe use a unicode-safe method instead of __str__
-        print "Error in <sort_uri_list_by_name>: possibly a UnicodeEncodeError"
+        print("Error in <sort_uri_list_by_name>: possibly a UnicodeEncodeError")
         return uri_list
 
 

src/encoded/commands/import_data.py

@@ -21,7 +21,7 @@
 
 """
 from webtest import TestApp
-from urlparse import urlparse
+from urllib.parse import urlparse
 from .. import loadxl
 import logging
 import os.path
@@ -30,7 +30,7 @@
 
 def basic_auth(username, password):
     from base64 import b64encode
-    return 'Basic ' + b64encode('%s:%s' % (username, password))
+    return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('utf-8')).decode('ascii')
 
 
 def remote_app(base, username='', password=''):

src/encoded/commands/update_file_status.py

@@ -10,7 +10,7 @@
 import json
 import requests
 import sys
-from urlparse import urljoin
+from urllib.parse import urljoin
 
 EPILOG = __doc__
 

src/encoded/commands/update_keys_links.py

@@ -14,6 +14,7 @@
 """
 import logging
 import transaction
+from future.utils import itervalues
 from pyramid.paster import get_app
 from pyramid.traversal import resource_path
 
@@ -33,7 +34,7 @@ def run(app, collections=None):
         collection = root[collection_name]
         collection_path = resource_path(collection)
         updated = 0
-        for count, item in enumerate(collection.itervalues()):
+        for count, item in enumerate(itervalues(collection)):
             path = resource_path(item)
             keys_add, keys_remove = item.update_keys()
             update = False