Get 100% of Python 3.4 tests to pass.

.gitignore

@@ -6,6 +6,7 @@
 *.log
 *.log.*
 *.pid
+*.sublime-*
 celerybeat-schedule
 coverage.xml
 loadtest/src/*.json

.travis.yml

@@ -14,6 +14,7 @@ language: python
 python:
     - "2.6"
     - "2.7"
+    - "3.4"
 
 notifications:
   email:

Makefile

@@ -4,6 +4,7 @@ BUILD_DIRS = .tox bin build dist include lib lib64 libmaxminddb man node_modules
 TESTS ?= ichnaea
 TRAVIS ?= false
 
+TOXBUILD ?= yes
 TOXENVDIR ?= $(HERE)/.tox/tmp
 TOXINIDIR ?= $(HERE)
 
@@ -231,9 +232,12 @@ endif
 	rm -rf $(TOXENVDIR)/ichnaea
 	cp -f $(TOXINIDIR)/lib/libmaxminddb* $(TOXENVDIR)/lib/
 	cd $(TOXENVDIR); make build_req
-	cd $(TOXENVDIR); bin/pip install -e /opt/mozilla/ichnaea/
+	cd $(TOXENVDIR); bin/pip install -e /opt/mozilla/ichnaea/ --no-compile
 
-tox_test: tox_install
+tox_test:
+ifeq ($(TOXBUILD),yes)
+	make tox_install
+endif
 	cd $(TOXENVDIR); make test
 
 $(BIN)/sphinx-build:

ichnaea/api/submit/views.py

@@ -22,7 +22,7 @@ def __init__(self, request):
 
     def decode_request_header(self, header_name):
         value = self.request.headers.get(header_name, None)
-        if isinstance(value, str):
+        if isinstance(value, str):  # pragma: no cover
             value = value.decode('utf-8', 'ignore')
         return value
 

ichnaea/api/views.py

@@ -1,5 +1,3 @@
-import zlib
-
 import colander
 import simplejson as json
 
@@ -79,7 +77,7 @@ def preprocess_request(self):
             # handle gzip self.request bodies
             try:
                 request_content = util.decode_gzip(self.request.body)
-            except zlib.error as exc:
+            except OSError as exc:
                 errors.append({'name': None, 'description': repr(exc)})
 
         request_data = {}

ichnaea/config.py

@@ -67,6 +67,6 @@ def asdict(self):
 def read_config(filename=None):
     if filename is None:
         filename = os.environ.get('ICHNAEA_CFG', 'ichnaea.ini')
-        if PY2:
+        if PY2:  # pragma: no cover
             filename = filename.decode('utf-8')
     return Config(filename)

ichnaea/content/views.py

@@ -1,3 +1,4 @@
+import operator
 import os
 
 import boto
@@ -121,7 +122,7 @@ def s3_list_downloads(assets_bucket, assets_url, raven_client):
     except S3ResponseError:  # pragma: no cover
         raven_client.captureException()
         return []
-    return sorted(files, reverse=True)
+    return sorted(files, key=operator.itemgetter('name'), reverse=True)
 
 
 class Layout(object):

ichnaea/export/tasks.py

@@ -1,7 +1,6 @@
 from contextlib import contextmanager, closing
 import csv
 from datetime import datetime, timedelta
-import gzip
 import os
 import shutil
 import tempfile
@@ -70,23 +69,6 @@ def selfdestruct_tempdir():
         shutil.rmtree(base_path)
 
 
-# Python 2.6 Gzipfile doesn't have __exit__
-class GzipFile(gzip.GzipFile):
-
-    # Add a default value, as this is used in the __repr__ and only
-    # set in the __init__ on successfully opening the file. Sentry/raven
-    # would bark on this while trying to capture the stack frame locals.
-    fileobj = None
-
-    def __enter__(self):
-        if self.fileobj is None:  # pragma: no cover
-            raise ValueError('I/O operation on closed GzipFile object')
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-
 def make_cell_export_dict(row):
     data = {
         'changeable': 1,
@@ -150,7 +132,7 @@ def val(key, default, _type):
 
 def write_stations_to_csv(session, table, columns,
                           cond, path, make_dict, fields):
-    with GzipFile(path, 'wb') as gzip_file:
+    with util.gzip_open(path, 'w') as gzip_file:
         writer = csv.DictWriter(gzip_file, fields, extrasaction='ignore')
         limit = 10000
         offset = 0
@@ -227,8 +209,8 @@ def commit_batch(ins, rows, commit=True):
         else:  # pragma: no cover
             session.flush()
 
-    with GzipFile(filename, 'rb') as zip_file:
-        csv_reader = csv.DictReader(zip_file, fields)
+    with util.gzip_open(filename, 'r') as gzip_file:
+        csv_reader = csv.DictReader(gzip_file, fields)
         batch = 10000
         rows = []
         area_keys = set()

ichnaea/export/tests.py

@@ -8,6 +8,7 @@
 from mock import MagicMock, patch
 from pytz import UTC
 import requests_mock
+import six
 
 from ichnaea.constants import CELL_MIN_ACCURACY
 from ichnaea.data.tasks import update_statcounter
@@ -21,7 +22,6 @@
     CELL_COLUMNS,
     CELL_FIELDS,
     CELL_HEADER_DICT,
-    GzipFile
 )
 from ichnaea.models import (
     Cell,
@@ -81,10 +81,10 @@ def test_local_export(self):
                 session, Cell.__table__, CELL_COLUMNS, cond,
                 path, make_cell_export_dict, CELL_FIELDS)
 
-            with GzipFile(path, 'rb') as gzip_file:
+            with util.gzip_open(path, 'r') as gzip_file:
                 reader = csv.DictReader(gzip_file, CELL_FIELDS)
 
-                header = reader.next()
+                header = six.next(reader)
                 self.assertTrue('area' in header.values())
                 self.assertEqual(header, CELL_HEADER_DICT)
 
@@ -163,8 +163,8 @@ def get_test_csv(self, lo=1, hi=10, time=1408604686):
 
         with selfdestruct_tempdir() as d:
             path = os.path.join(d, 'import.csv.gz')
-            with GzipFile(path, 'wb') as f:
-                f.write(txt)
+            with util.gzip_open(path, 'w') as gzip_file:
+                gzip_file.write(txt)
             yield path
 
     def import_test_csv(self, lo=1, hi=10, time=1408604686, session=None):
@@ -255,9 +255,9 @@ def test_local_import_delta(self):
 
     def test_local_import_latest_through_http(self):
         with self.get_test_csv() as path:
-            with open(path, 'r') as f:
-                with requests_mock.Mocker() as m:
-                    m.register_uri('GET', re.compile('.*'), body=f)
+            with open(path, 'rb') as gzip_file:
+                with requests_mock.Mocker() as req_m:
+                    req_m.register_uri('GET', re.compile('.*'), body=gzip_file)
                     import_latest_ocid_cells()
 
         cells = (self.session.query(OCIDCell)

ichnaea/geoip.py

@@ -130,7 +130,7 @@ def ping(self):
 
     def check_extension(self):
         builtin_module = 'builtins'
-        if PY2:
+        if PY2:  # pragma: no cover
             builtin_module = '__builtin__'
         for instance in (self.metadata(), self._db_reader):
             if type(instance).__module__ != builtin_module:

ichnaea/monitor/tasks.py

@@ -18,7 +18,7 @@ def monitor_api_key_limits(self):
         keys = self.redis_client.keys('apilimit:*:' + today)
         if keys:
             values = self.redis_client.mget(keys)
-            keys = [k.split(':')[1] for k in keys]
+            keys = [k.decode('utf-8').split(':')[1] for k in keys]
         else:
             values = []
 

ichnaea/scripts/tests/test_map.py

@@ -39,7 +39,7 @@ def test_export_to_csv(self):
         try:
             result = export_to_csv(session, filename, multiplier=3)
             self.assertEqual(result, 9)
-            written = os.read(fd, 10240)
+            written = os.read(fd, 10240).decode('utf-8')
             lines = [line.split(',') for line in written.split()]
             self.assertEqual(len(lines), 9)
             self.assertEqual(set([round(float(l[0]), 2) for l in lines]),

ichnaea/tests/test_util.py

@@ -0,0 +1,35 @@
+from datetime import datetime
+
+from pytz import UTC
+
+from ichnaea.tests.base import TestCase
+from ichnaea import util
+
+
+class TestUtil(TestCase):
+
+    gzip_foo = (b'\x1f\x8b\x08\x00\xed\x7f\x9aU\x02\xffK'
+                b'\xcb\xcf\x07\x00!es\x8c\x03\x00\x00\x00')
+
+    def test_utcnow(self):
+        now = util.utcnow()
+        self.assertTrue(isinstance(now, datetime))
+        self.assertEqual(now.tzinfo, UTC)
+
+    def test_encode_gzip(self):
+        data = util.encode_gzip(u'foo')
+        self.assertEqual(data[:4], self.gzip_foo[:4])
+        self.assertEqual(data[-13:], self.gzip_foo[-13:])
+
+    def test_encode_gzip_bytes(self):
+        data = util.encode_gzip(b'foo')
+        self.assertEqual(data[:4], self.gzip_foo[:4])
+        self.assertEqual(data[-13:], self.gzip_foo[-13:])
+
+    def test_decode_gzip(self):
+        data = util.decode_gzip(self.gzip_foo)
+        self.assertEqual(data, u'foo')
+
+    def test_roundtrip_gzip(self):
+        data = util.decode_gzip(util.encode_gzip(b'foo'))
+        self.assertEqual(data, u'foo')

ichnaea/util.py

@@ -1,18 +1,61 @@
 from datetime import datetime
-import zlib
+import gzip
+from io import BytesIO
+import sys
 
 from pytz import UTC
-from webob.response import gzip_app_iter
+import six
 
+if sys.version_info < (2, 7):  # pragma: no cover
 
-def encode_gzip(data):
-    # based on webob.response.Response.encode_content
-    return ''.join(gzip_app_iter(data))
+    # the io module was pure-python in 2.6, use a C version
+    from cStringIO import StringIO as BytesIO  # NOQA
 
+    class GzipFile(gzip.GzipFile):  # NOQA
+        # Python 2.6 GzipFile isn't a context manager
 
-def decode_gzip(data):
-    # based on webob.response.Response.decode_content
-    return zlib.decompress(data, 16 + zlib.MAX_WBITS)
+        # Add a default value, as this is used in the __repr__ and only
+        # set in the __init__ on successfully opening the file. Sentry/raven
+        # would bark on this while trying to capture the stack frame locals.
+        fileobj = None
+
+        def __enter__(self):
+            if self.fileobj is None:  # pragma: no cover
+                raise ValueError('I/O operation on closed GzipFile object')
+            return self
+
+        def __exit__(self, *args):
+            self.close()
+
+else:  # pragma: no cover
+    GzipFile = gzip.GzipFile
+
+
+def gzip_open(filename, mode):  # pragma: no cover
+    # open with either mode r or w
+    if six.PY2:
+        return GzipFile(filename, mode)
+    else:
+        fd = open(filename, mode + 'b')
+        return gzip.open(fd, mode=mode + 't', encoding='utf-8')
+
+
+def encode_gzip(data, encoding='utf-8'):
+    if isinstance(data, six.string_types):
+        data = data.encode(encoding)
+    out = BytesIO()
+    with GzipFile(None, 'wb', compresslevel=9, fileobj=out) as gzip_file:
+        gzip_file.write(data)
+    return out.getvalue()
+
+
+def decode_gzip(data, encoding='utf-8'):
+    try:
+        with GzipFile(None, mode='rb', fileobj=BytesIO(data)) as gzip_file:
+            out = gzip_file.read()
+        return out.decode(encoding)
+    except (IOError, OSError) as exc:
+        raise OSError(str(exc))
 
 
 def utcnow():