Merge cd5427e into 14b1731

.travis.yml

@@ -1,28 +1,18 @@
 language: python
 python:
-  - 2.7
-  - 3.5
   - 3.6
-  - pypy
-dist: xenial
+  - 3.7
+  - 3.8
+dist: bionic
 cache:
   pip: true
   apt: true
 matrix:
   fast_finish: true
-  allow_failures:
-    - python: 3.5
-    - python: 3.6
-    - python: pypy
-  exclude:
-    - python: 3.5
-      env: LINT_TEST=1
-    - python: 3.6
-      env: LINT_TEST=1
-    - python: pypy
+  include:
+    - python: 3.7
       env: LINT_TEST=1
 env:
-  - LINT_TEST=1
   - INTEGRATION_TEST=1
   - UNIT_TEST=1
 addons:
@@ -32,14 +22,13 @@ addons:
     - libjpeg-progs
     - libimage-exiftool-perl
     - gifsicle
-    - python-all-dev
     - scons
+    - python-all-dev
     - libboost-python-dev
     - libexiv2-dev
     - ffmpeg
 install:
   - pip install --upgrade pip
-  - pip install -I https://github.com/escaped/pyexiv2/archive/69dd6448f9831bd826137b7519f9d797b23ab4ec.zip
   - cd $TRAVIS_BUILD_DIR && make setup_python
   - pip install coveralls
 before_script:

docs/metadata.rst

@@ -1,16 +1,16 @@
 Image Metadata
 ==============
 
-Thumbor uses `Pyexiv2 <http://tilloy.net/dev/pyexiv2/>`_ to read and write image metadata.
+Thumbor uses `py3exiv2 <https://launchpad.net/py3exiv2>`_ to read and write image metadata.
 
-If the Pyexif2 or Py3exif2 Python library is available, the PIL engine also stores image metadata
+If the Py3exif2 Python library is available, the PIL engine also stores image metadata
 in ``engine.metadata``.
 
 
 
 Reading and writing Metadata
 ----------------------------
-This part is copied from the `Pyexiv2 Tutorial <http://tilloy.net/dev/pyexiv2/tutorial.html>`_
+This part is copied from the `py3exiv2 Tutorial <https://python3-exiv2.readthedocs.io/en/latest/tutorial.html>`_
 
 Let's retrieve a list of all the available EXIF tags available in the image::
 
@@ -112,15 +112,16 @@ On OSX you can use homebrew to install the dependencies::
     brew install boost-python
     brew install exiv2
 
-    pip install git+https://github.com/escaped/pyexiv2.git
+    pip install py3exiv2
 
 If you are updating thumbor and already have an existing virtualenv, then you have to recreate it.
 If you have both a System Python and a Homebrew Python with the same version, then make sure
 the Virtualenv uses the Homebrew Python binary.
 
 On Linux Pyexiv2 can be installed with apt-get:
 
-    apt-get install python-pyexiv2
+    apt-get install python-all-dev libboost-python-dev libexiv2-dev
+    pip install py3exiv2
 
 
 pyexiv2.metadata API reference
@@ -132,7 +133,3 @@ pyexiv2.metadata API reference
              exif_keys, iptc_keys, iptc_charset, xmp_keys,
              __getitem__, __setitem__, __delitem__,
              comment, previews, copy, buffer
-
-
-Currently PyExiv is deprecated in favor of GExiv. However, it is really difficult
-to install GExiv with Python on a non-Ubuntu system. Therefore Pyexiv2 is used.

requirements

@@ -15,5 +15,4 @@ memcached
 libmemcache-dev
 libmemcached-dev
 python-scipy
-python-pyexiv2
 cython

setup.py

@@ -31,8 +31,9 @@
     "pyssim>=0.4.0",
     "cairosvg>=1.0.0,<2.0.0,!=1.0.21",
     "preggy>=1.3.0",
-    "opencv-python",
+    "opencv-python-headless",
     "yanc>=0.3.3",
+    "py3exiv2",
 ]
 
 
@@ -83,7 +84,12 @@ def run_setup(extension_modules=[]):
             'Natural Language :: English',
             'Operating System :: MacOS',
             'Operating System :: POSIX :: Linux',
-            'Programming Language :: Python :: 2.7',
+            'Programming Language :: Python :: 3',
+            'Programming Language :: Python :: 3.6',
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3 :: Only',
+            'Programming Language :: Python :: Implementation :: CPython',
             'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
             'Topic :: Multimedia :: Graphics :: Presentation'
         ],
@@ -102,10 +108,7 @@ def run_setup(extension_modules=[]):
             "piexif>=1.0.13,<2.0.0",
             "statsd>=3.0.1",
             "libthumbor>=1.3.2",
-            "futures",
-            "argparse",
             "pytz",
-            "six",
             "webcolors",
         ],
 

tests/base.py

@@ -8,7 +8,6 @@
 # http://www.opensource.org/licenses/mit-license
 # Copyright (c) 2011 globo.com thumbor@googlegroups.com
 
-from __future__ import print_function
 import random
 import unicodedata
 from io import BytesIO
@@ -20,8 +19,7 @@
 from PIL import Image
 from ssim import compute_ssim
 from preggy import create_assertions
-from six import StringIO
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 from thumbor.app import ThumborServiceApp
 from thumbor.context import Context, RequestParameters
@@ -32,11 +30,6 @@
 
 from tornado.testing import AsyncHTTPTestCase
 
-try:
-    unicode        # Python 2
-except NameError:
-    unicode = str  # Python 3
-
 
 @create_assertions
 def to_exist(topic):
@@ -46,7 +39,7 @@ def to_exist(topic):
 def normalize_unicode_path(path):
     normalized_path = path
     for format in ['NFD', 'NFC', 'NFKD', 'NFKC']:
-        normalized_path = unicodedata.normalize(format, unicode(path))
+        normalized_path = unicodedata.normalize(format, str(path))
         if exists(normalized_path):
             break
     return normalized_path
@@ -70,33 +63,33 @@ def to_be_the_same_as(topic, expected):
 
 @create_assertions
 def to_be_similar_to(topic, expected):
-    topic_image = Image.open(StringIO(topic))
-    expected_image = Image.open(StringIO(expected))
+    topic_image = Image.open(BytesIO(topic))
+    expected_image = Image.open(BytesIO(expected))
 
     return get_ssim(topic_image, expected_image) > 0.95
 
 
 @create_assertions
 def to_be_webp(topic):
-    im = Image.open(StringIO(topic))
+    im = Image.open(BytesIO(topic))
     return im.format.lower() == 'webp'
 
 
 @create_assertions
 def to_be_png(topic):
-    im = Image.open(StringIO(topic))
+    im = Image.open(BytesIO(topic))
     return im.format.lower() == 'png'
 
 
 @create_assertions
 def to_be_gif(topic):
-    im = Image.open(StringIO(topic))
+    im = Image.open(BytesIO(topic))
     return im.format.lower() == 'gif'
 
 
 @create_assertions
 def to_be_jpeg(topic):
-    im = Image.open(StringIO(topic))
+    im = Image.open(BytesIO(topic))
     return im.format.lower() == 'jpeg'
 
 
@@ -123,25 +116,25 @@ def to_be_cropped(image):
 
 
 def encode_multipart_formdata(fields, files):
-    BOUNDARY = 'thumborUploadFormBoundary'
-    CRLF = '\r\n'
+    BOUNDARY = b'thumborUploadFormBoundary'
+    CRLF = b'\r\n'
     L = []
     for key, value in fields.items():
-        L.append('--' + BOUNDARY)
-        L.append('Content-Disposition: form-data; name="%s"' % key)
-        L.append('')
+        L.append(b'--' + BOUNDARY)
+        L.append(b'Content-Disposition: form-data; name="%s"' % key.encode())
+        L.append(b'')
         L.append(value)
     for (key, filename, value) in files:
-        L.append('--' + BOUNDARY)
-        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
-        L.append('Content-Type: %s' % mimetypes.guess_type(filename)[0] or 'application/octet-stream')
-        L.append('')
+        L.append(b'--' + BOUNDARY)
+        L.append(b'Content-Disposition: form-data; name="%s"; filename="%s"' % (key.encode(), filename.encode()))
+        L.append(b'Content-Type: %s' % mimetypes.guess_type(filename)[0].encode() or b'application/octet-stream')
+        L.append(b'')
         L.append(value)
-    L.append('')
-    L.append('')
-    L.append('--' + BOUNDARY + '--')
-    body = CRLF.join([str(item) for item in L])
-    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
+    L.append(b'')
+    L.append(b'')
+    L.append(b'--' + BOUNDARY + b'--')
+    body = CRLF.join(L)
+    content_type = b'multipart/form-data; boundary=%s' % BOUNDARY
     return content_type, body
 
 

tests/detectors/test_face_detector.py

@@ -18,7 +18,7 @@
 
 class FaceDetectorTestCase(DetectorTestCase):
     def test_should_detect_one_face(self):
-        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01.jpg')) as f:
+        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01.jpg'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         for i, detector in enumerate([
@@ -41,7 +41,7 @@ def test_should_detect_one_face(self):
             expect(detection_result.height).to_be_numeric()
 
     def test_should_not_detect(self):
-        with open(abspath('./tests/fixtures/images/no_face.jpg')) as f:
+        with open(abspath('./tests/fixtures/images/no_face.jpg'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         self.context.config.FACE_DETECTOR_CASCADE_FILE = abspath(
@@ -52,7 +52,7 @@ def test_should_not_detect(self):
         expect(self.context.request.focal_points).to_be_empty()
 
     def test_should_run_on_grayscale_images(self):
-        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01-grayscale.jpg')) as f:
+        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01-grayscale.jpg'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         self.context.config.FACE_DETECTOR_CASCADE_FILE = abspath(
@@ -69,7 +69,7 @@ def test_should_run_on_grayscale_images(self):
         expect(detection_result.height).to_be_numeric()
 
     def test_should_run_on_cmyk_images(self):
-        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01-cmyk.jpg')) as f:
+        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01-cmyk.jpg'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         self.context.config.FACE_DETECTOR_CASCADE_FILE = abspath(
@@ -86,7 +86,7 @@ def test_should_run_on_cmyk_images(self):
         expect(detection_result.height).to_be_numeric()
 
     def test_should_run_on_images_with_alpha(self):
-        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01.png')) as f:
+        with open(abspath('./tests/fixtures/images/Giunchedi%2C_Filippo_January_2015_01.png'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         self.context.config.FACE_DETECTOR_CASCADE_FILE = abspath(

tests/detectors/test_feature_detector.py

@@ -19,7 +19,7 @@
 class FeatureDetectorTestCase(DetectorTestCase):
 
     def test_should_detect_multiple_points(self):
-        with open(abspath('./tests/fixtures/images/no_face.jpg')) as f:
+        with open(abspath('./tests/fixtures/images/no_face.jpg'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         FeatureDetector(self.context, 0, None).detect(lambda: None)
@@ -28,7 +28,7 @@ def test_should_detect_multiple_points(self):
         expect(detection_result[0].origin).to_equal('alignment')
 
     def test_should_detect_a_single_point(self):
-        with open(abspath('./tests/fixtures/images/single_point.jpg')) as f:
+        with open(abspath('./tests/fixtures/images/single_point.jpg'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         FeatureDetector(self.context, 0, None).detect(lambda: None)
@@ -37,7 +37,7 @@ def test_should_detect_a_single_point(self):
         expect(detection_result[0].origin).to_equal('alignment')
 
     def test_should_not_detect_points(self):
-        with open(abspath('./tests/fixtures/images/1x1.png')) as f:
+        with open(abspath('./tests/fixtures/images/1x1.png'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         FeatureDetector(self.context, 0, []).detect(lambda: None)

tests/detectors/test_glasses_detector.py

@@ -30,7 +30,7 @@ def test_detector_uses_proper_cascade(self):
         expect(detector).not_to_be_null()
 
     def test_should_detect_glasses(self):
-        with open(abspath('./tests/fixtures/images/Christophe_Henner_-_June_2016.jpg')) as f:
+        with open(abspath('./tests/fixtures/images/Christophe_Henner_-_June_2016.jpg'), 'rb') as f:
             self.engine.load(f.read(), None)
 
         self.context.config.GLASSES_DETECTOR_CASCADE_FILE = abspath(

tests/detectors/test_queued_detector.py

@@ -66,7 +66,7 @@ def validate(data):
         }
 
         result = self.redis.lpop('resque:queue:Detect')
-        expect(loads(result)).to_be_like(expected_payload)
+        expect(loads(result.decode('utf-8'))).to_be_like(expected_payload)
 
     def test_detector_fails_properly(self):
         ctx = mock.Mock(