Merge pull request #28 from OnroerendErfgoed/move_to_python_3

Move to python 3

.travis.yml

@@ -2,6 +2,7 @@ language: python
 sudo: false
 python:
   - "2.7"
+  - "3.5"
 env:
   - PROJECT=augeias
 notifications:

augeias/stores/PairTreeFileSystemStore.py

@@ -9,6 +9,7 @@
 import os
 import magic
 import datetime
+import sys
 
 
 class PairTreeFileSystemStore(IStore):
@@ -58,7 +59,7 @@ def get_object_info(self, container_key, object_key):
         return {
             'time_last_modification': datetime.datetime.fromtimestamp(file_stat.st_mtime).isoformat(),
             'size': file_stat.st_size,
-            'mime': magic.from_buffer(open(file_path).read(1048576), mime=True) or 'application/octet-stream'
+            'mime': magic.from_buffer(open(file_path, 'rb').read(1048576), mime=True) or 'application/octet-stream'
         }
 
     def create_object(self, container_key, object_key, object_data):
@@ -70,6 +71,7 @@ def create_object(self, container_key, object_key, object_data):
         :param str object_data: The data for the object to create.
         :raises augeias.stores.error.NotFoundException: When the container could not be found.
         '''
+        _validate_data(object_data)
         container = self._get_container(container_key)
         container.add_bytestream(object_key, object_data)
 
@@ -82,6 +84,7 @@ def update_object(self, container_key, object_key, object_data):
         :param str object_data: New data for the object.
         :raises augeias.stores.error.NotFoundException: When the object or container could not be found.
         '''
+        _validate_data(object_data)
         container = self.store.get_object(container_key, False)
         container.add_bytestream(object_key, object_data)
 
@@ -130,3 +133,20 @@ def delete_container(self, container_key):
             self.store.delete_object(container_key)
         except ObjectNotFoundException:
             raise NotFoundException
+
+
+def _is_allowed_data(data):
+    # not exhaustive.
+    python_version = sys.version_info.major
+    if python_version < 3 and isinstance(data, unicode):
+        return False
+
+    if python_version >= 3 and isinstance(data, str):
+        return False
+
+    return True
+
+
+def _validate_data(data):
+    if not _is_allowed_data(data):
+        raise IOError('Data type is not allowed')

augeias/views.py

@@ -183,9 +183,9 @@ def delete_container(self):
 # HELPERS
 
 
-def _is_long(s):
+def _is_integer(s):
     try:
-        long(s)
+        int(s)  # python 2.7 'auto-promotes' int to long if required
         return True
     except ValueError:
         return False
@@ -199,9 +199,9 @@ def _get_object_data(request):
 
 
 def _get_object_data_from_stream(request):
-    if 'Content-Length' not in request.headers or not _is_long(request.headers['Content-Length']):
+    if 'Content-Length' not in request.headers or not _is_integer(request.headers['Content-Length']):
         raise HTTPLengthRequired
-    content_length = long(request.headers['Content-Length'])
+    content_length = int(request.headers['Content-Length'])
     object_data = request.body_file
     if content_length == 0:
         raise HTTPBadRequest('body is empty')

requirements-dev.txt

@@ -19,7 +19,7 @@ sphinxcontrib-httpdomain==1.2.1
 sphinx_rtd_theme==0.1.9
 
 # waitress
-waitress==0.8.8
+waitress==1.1.0
 
 # Linting
 flake8==2.1.0

requirements.txt

@@ -1,13 +1,14 @@
 # pyramid
-pyramid==1.5
+pyramid==1.8
 pyramid_rewrite==0.2
 
 # pairtree
-pairtree==0.7.1-T
+git+https://github.com/benosteen/pairtree.git@56a5ffdf258c30a3ce210868a2ea1258d779509e#egg=pairtree
+#pairtree==0.7.1-T
 python-magic==0.4.13
 
 venusian==1.0
-webob==1.4
+webob==1.7
 PasteDeploy==1.5.2
 
 repoze.lru==0.6

setup.py

@@ -10,7 +10,7 @@
 
 requires = [
     'pyramid',
-    'pairtree==0.7.1-T',
+    'pairtree',
     'python-magic',
     'pyramid_rewrite'
     ]

tests/test_functional.py

@@ -9,10 +9,13 @@
 from augeias.collections import Collection
 from augeias.stores.PairTreeFileSystemStore import PairTreeFileSystemStore
 import json
+import sys
 
 here = os.path.dirname(__file__)
 settings = get_appsettings(os.path.join(here, 'conf_test.ini'))
 
+python_version = sys.version_info.major
+
 
 def collections_include(config, store_dir):
     test_collection = Collection(name='TEST_COLLECTION', object_store=PairTreeFileSystemStore(store_dir))
@@ -50,8 +53,8 @@ def test_create_container(self):
         res = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', res.status)
         self.assertIn('application/json', res.headers['Content-Type'])
-        self.assertIn('TEST_CONTAINER_ID', res.body)
-        self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID', res.body)
+        self.assertIn('TEST_CONTAINER_ID', res.text)
+        self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID', res.text)
 
 
     def test_add_object(self):
@@ -65,9 +68,9 @@ def test_add_object(self):
         res = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertEqual('200 OK', res.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      res.body)
-        self.assertIn('TEST_CONTAINER_ID', res.body)
-        self.assertIn('200x300', res.body)
+                      res.text)
+        self.assertIn('TEST_CONTAINER_ID', res.text)
+        self.assertIn('200x300', res.text)
 
     def test_add_object_content_length_errors(self):
         # create a container
@@ -90,14 +93,14 @@ def test_get_object(self):
         cres = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', cres.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID',
-                      cres.body)
+                      cres.text)
         testdata = os.path.join(here, '../', 'fixtures/kasteel.jpg')
         with open(testdata, 'rb') as f:
             bdata = f.read()
         file_size = len(bdata)
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
         self.assertEqual('200 OK', ores.status)
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300')
@@ -110,13 +113,13 @@ def test_get_object_info(self):
         cres = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', cres.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID',
-                      cres.body)
+                      cres.text)
         testdata = os.path.join(here, '../', 'fixtures/kasteel.jpg')
         with open(testdata, 'rb') as f:
             bdata = f.read()
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
         self.assertEqual('200 OK', ores.status)
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300/meta')
@@ -154,7 +157,7 @@ def test_list_object_keys_for_container(self):
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID')
         self.assertEqual('200 OK', res.status)
-        l = ast.literal_eval(res.body)
+        l = ast.literal_eval(res.text)
         l = [i.strip() for i in l]
         self.assertTrue('200x300' in l and '400x600' in l)
 
@@ -168,7 +171,7 @@ def test_update_object(self):
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertEqual('200 OK', ores.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
 
         testdata = os.path.join(here, '../', 'fixtures/brug.jpg')
         with open(testdata, 'rb') as f:
@@ -203,7 +206,7 @@ def test_copy_object(self):
         ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata)
         self.assertEqual('200 OK', ores.status)
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
-                      ores.body)
+                      ores.text)
         cres2 = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID2')
         self.assertEqual('200 OK', cres2.status)
         json_data = json.dumps({
@@ -278,8 +281,8 @@ def test_delete_object(self):
         self.assertIn(self.storage_location + 'collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300',
                       res.json_body['uri'])
         self.assertEqual('200 OK', res.status)
-        self.assertIn('TEST_CONTAINER_ID', res.body)
-        self.assertIn('200x300', res.body)
+        self.assertIn('TEST_CONTAINER_ID', res.text)
+        self.assertIn('200x300', res.text)
 
         res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', status=404,
                                expect_errors=True)
@@ -305,7 +308,7 @@ def test_create_container_with_id(self):
         self.assertEqual('201 Created', res.status)
         uuid4hex = re.compile('[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z', re.I)
         container_key = res.json_body['container_key']
-        if isinstance(container_key, unicode):
+        if python_version < 3 and isinstance(container_key, unicode):
             container_key = str(container_key)
         self.assertTrue(uuid4hex.match(container_key))
         print(res.json_body['uri'])
@@ -320,7 +323,7 @@ def test_create_object_with_id(self):
         self.assertEqual('201 Created', res.status)
         uuid4hex = re.compile('[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z', re.I)
         object_key = res.json_body['object_key']
-        if isinstance(object_key, unicode):
+        if python_version < 3 and isinstance(object_key, unicode):
             object_key = str(object_key)
         self.assertTrue(uuid4hex.match(object_key))
         print(res.json_body['uri'])

tests/test_stores.py

@@ -22,12 +22,12 @@ def test_usage_scenario(self):
         container_key = 'testing'
         object_key = 'metadata'
         self.store.create_container(container_key)
-        self.store.create_object(container_key, object_key, 'some test data')
+        self.store.create_object(container_key, object_key, b'some test data')
         object_list = self.store.list_object_keys_for_container(container_key)
         self.assertEqual(1, len(object_list))
         self.assertEqual(object_key, object_list[0])
         object_value = self.store.get_object(container_key, object_key)
-        self.assertEqual('some test data', object_value)
+        self.assertEqual(b'some test data', object_value)
         self.store.delete_object(container_key, object_key)
         object_list = self.store.list_object_keys_for_container(container_key)
         self.assertEqual(0, len(object_list))
@@ -70,25 +70,25 @@ def test_update_scenario(self):
         container_key = 'testing'
         object_key = 'metadata'
         self.store.create_container(container_key)
-        self.store.create_object(container_key, object_key, 'some test data')
+        self.store.create_object(container_key, object_key, b'some test data')
         object_value = self.store.get_object(container_key, object_key)
-        self.assertEqual('some test data', object_value)
-        self.store.update_object(container_key, object_key, 'updated data')
+        self.assertEqual(b'some test data', object_value)
+        self.store.update_object(container_key, object_key, b'updated data')
         object_value = self.store.get_object(container_key, object_key)
-        self.assertEqual('updated data', object_value)
+        self.assertEqual(b'updated data', object_value)
 
     def test_delete_nonexisting(self):
         container_key = 'testing'
         object_key = 'metadata'
         self.store.create_container(container_key)
-        self.store.create_object(container_key, object_key, 'some test data')
+        self.store.create_object(container_key, object_key, b'some test data')
         self.assertRaises(NotFoundException, self.store.delete_object, container_key, 'nogo')
 
     def test_add_object_to_nonexisting_container(self):
         error_raised = False
         self.store.create_container('x')
         try:
-            self.store.create_object('xx', '253', 'some test data')
+            self.store.create_object('xx', '253', b'some test data')
         except NotFoundException:
             error_raised = True
         self.assertTrue(error_raised)