rolled to v1 and added iso flag to cutout service

- Cutout service can now get/post data to iso resolution levels in an anisotropic channel
- Due to minor schema change, decided to deprecate v0.8 for simplicity and roll to v1 now that core services are complete. Any additional non-breaking changes will still go in v1
- Rolled “latest” re-write rule to v1

boss_nginx.conf

@@ -34,7 +34,7 @@ server {
     }
 
     location /latest {
-        rewrite ^/latest(.*)$ /v0.8$1 last;
+        rewrite ^/latest(.*)$ /v1$1 last;
     }
 
 }

django/boss/settings/base.py

@@ -146,7 +146,7 @@
 
 }
 # Version that unit tests are being run against
-BOSS_VERSION = 'v0.8'
+BOSS_VERSION = 'v1'
 
 # Maximum number of bytes in an uncompressed matrix supported by the Cutout Service
 CUTOUT_MAX_SIZE = 520 * 1048576

django/boss/urls.py

@@ -50,31 +50,32 @@
     url(r'^v0.5/', views.Unsupported.as_view()),
     url(r'^v0.6/', views.Unsupported.as_view()),
     url(r'^v0.7/', views.Unsupported.as_view()),
+    url(r'^v0.8/', views.Unsupported.as_view()),
 
-    # API version 0.8
-    url(r'^v0.8/meta/', include('bossmeta.urls', namespace='v0.8')),
-    url(r'^v0.8/permissions/?', include('bosscore.urls.permission-urls', namespace='v0.8')),
-    url(r'^v0.8/groups/', include('bosscore.urls.group-urls', namespace='v0.8')),
-    url(r'^v0.8/cutout/', include('bossspatialdb.urls', namespace='v0.8')),
-    url(r'^v0.8/downsample/', include('bossspatialdb.urls_downsample', namespace='v0.8')),
-    url(r'^v0.8/image/', include('bosstiles.image_urls', namespace='v0.8')),
-    url(r'^v0.8/tile/', include('bosstiles.tile_urls', namespace='v0.8')),
-    url(r'^v0.8/ingest/', include('bossingest.urls', namespace='v0.8')),
-    url(r'^v0.8/collection/', include('bosscore.urls.resource_urls', namespace='v0.8')),
-    url(r'^v0.8/coord/', include('bosscore.urls.coord_urls', namespace='v0.8')),
+    # API version 1
+    url(r'^v1/meta/', include('bossmeta.urls', namespace='v1')),
+    url(r'^v1/permissions/?', include('bosscore.urls.permission-urls', namespace='v1')),
+    url(r'^v1/groups/', include('bosscore.urls.group-urls', namespace='v1')),
+    url(r'^v1/cutout/', include('bossspatialdb.urls', namespace='v1')),
+    url(r'^v1/downsample/', include('bossspatialdb.urls_downsample', namespace='v1')),
+    url(r'^v1/image/', include('bosstiles.image_urls', namespace='v1')),
+    url(r'^v1/tile/', include('bosstiles.tile_urls', namespace='v1')),
+    url(r'^v1/ingest/', include('bossingest.urls', namespace='v1')),
+    url(r'^v1/collection/', include('bosscore.urls.resource_urls', namespace='v1')),
+    url(r'^v1/coord/', include('bosscore.urls.coord_urls', namespace='v1')),
 
     # SSO Urls
-    url(r'^v0.8/sso/user/', include('sso.urls.user-urls', namespace='v0.8')),
-    url(r'^v0.8/sso/user-role/', include('sso.urls.user-role-urls', namespace='v0.8')),
+    url(r'^v1/sso/user/', include('sso.urls.user-urls', namespace='v1')),
+    url(r'^v1/sso/user-role/', include('sso.urls.user-role-urls', namespace='v1')),
 
     # Management Console Urls
-    url(r'^v0.8/mgmt/', include('mgmt.urls', namespace='mgmt')),
+    url(r'^v1/mgmt/', include('mgmt.urls', namespace='mgmt')),
     url(r'^/?$', RedirectView.as_view(pattern_name='mgmt:home')),
 
     #Object urls
-    url(r'^v0.8/reserve/', include('bossobject.urls.reserve_urls', namespace='v0.8')),
-    url(r'^v0.8/ids/', include('bossobject.urls.ids_urls', namespace='v0.8')),
-    url(r'^v0.8/boundingbox/', include('bossobject.urls.boundingbox_urls', namespace='v0.8')),
+    url(r'^v1/reserve/', include('bossobject.urls.reserve_urls', namespace='v1')),
+    url(r'^v1/ids/', include('bossobject.urls.ids_urls', namespace='v1')),
+    url(r'^v1/boundingbox/', include('bossobject.urls.boundingbox_urls', namespace='v1')),
 ]
 
 if 'djangooidc' in settings.INSTALLED_APPS:

django/bosscore/test/setup_db.py

@@ -127,14 +127,23 @@ def insert_iso_data(self):
 
     def insert_downsample_data(self):
         """Some resources for small downsample tests"""
-        self.add_coordinate_frame('cf_ds_aniso', 'Description for cf2', 0, 4069, 0, 4069, 0, 128, 4, 4, 35)
-        self.add_experiment('col1', 'exp_ds_aniso', 'cf_ds_aniso', 3, 500, 1)
+        self.add_coordinate_frame('cf_ds_aniso', 'Description for cf2', 0, 4096, 0, 4096, 0, 128, 4, 4, 35)
+        self.add_experiment('col1', 'exp_ds_aniso', 'cf_ds_aniso', 5, 500, 1)
         self.add_channel('col1', 'exp_ds_aniso', 'channel1', 0, 0, 'uint8', 'image')
 
-        self.add_coordinate_frame('cf_ds_iso', 'Description for cf2', 0, 4069, 0, 4069, 0, 128, 6, 6, 6)
+        self.add_coordinate_frame('cf_ds_iso', 'Description for cf2', 0, 4096, 0, 4096, 0, 128, 6, 6, 6)
         self.add_experiment('col1', 'exp_ds_iso', 'cf_ds_iso', 3, 500, 1, hierarchy_method="isotropic")
         self.add_channel('col1', 'exp_ds_iso', 'channel1', 0, 0, 'uint8', 'image')
 
+    def insert_downsample_write_data(self):
+        """Some resources for writing to an off 0 base res (tests write at 4)"""
+        self.add_experiment('col1', 'exp_ds_aniso_4', 'cf_ds_aniso', 5, 500, 1)
+        self.add_channel('col1', 'exp_ds_aniso_4', 'channel1', 0, 4, 'uint8', 'image')
+
+        self.add_experiment('col1', 'exp_ds_iso_4', 'cf_ds_iso', 5, 500, 1, hierarchy_method="isotropic")
+        self.add_channel('col1', 'exp_ds_iso_4', 'channel1', 0, 4, 'uint8', 'image')
+
+
     @staticmethod
     def add_permissions(group, obj):
         # Get the type of model

django/bossspatialdb/test/cutout_view_uint8.py

@@ -914,6 +914,176 @@ def test_channel_uint8_jpeg_invalid_datatype(self):
                                     t_range=None).render()
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
 
+    def test_channel_uint8_downsample_below_iso_fork(self):
+        """ Test writing/reading data to non-base resolution below the iso fork"""
+
+        self.dbsetup.insert_downsample_data()
+
+        test_mat = np.random.randint(1, 254, (17, 300, 500))
+        test_mat = test_mat.astype(np.uint8)
+        h = test_mat.tobytes()
+        bb = blosc.compress(h, typesize=8)
+
+        # Create request
+        factory = APIRequestFactory()
+        request = factory.post('/' + version + '/cutout/col1/exp_ds_aniso/channel1/1/100:600/450:750/20:37/', bb,
+                               content_type='application/blosc')
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1',
+                                    resolution='0', x_range='100:600', y_range='450:750', z_range='20:37', t_range=None)
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        # Create Request to get data you posted
+        request = factory.get('/' + version + '/cutout/col1/exp_ds_aniso/channel1/0/100:600/450:750/20:37/',
+                              HTTP_ACCEPT='application/blosc')
+
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso', channel='channel1',
+                                    resolution='0', x_range='100:600', y_range='450:750',
+                                    z_range='20:37', t_range=None).render()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # Decompress
+        raw_data = blosc.decompress(response.content)
+        data_mat = np.fromstring(raw_data, dtype=np.uint8)
+        data_mat = np.reshape(data_mat, (17, 300, 500), order='C')
+
+        # Test for data equality (what you put in is what you got back!)
+        np.testing.assert_array_equal(data_mat, test_mat)
+
+    def test_channel_uint8_downsample_above_iso_fork(self):
+        """ Test writing/reading data to non-base resolution above the iso fork"""
+        self.dbsetup.insert_downsample_data()
+        self.dbsetup.insert_downsample_write_data()
+
+        test_mat = np.random.randint(1, 254, (2, 256, 256))
+        test_mat = test_mat.astype(np.uint8)
+        h = test_mat.tobytes()
+        bb = blosc.compress(h, typesize=8)
+
+        # Create request
+        factory = APIRequestFactory()
+        request = factory.post('/' + version + '/cutout/col1/exp_ds_aniso_4/channel1/4/0:256/0:256/2:4/?iso=true', bb,
+                               content_type='application/blosc')
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso_4', channel='channel1',
+                                    resolution='4', x_range='0:256', y_range='0:256', z_range='2:4', t_range=None)
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        # Create Request to get data you posted, but ask for aniso data
+        request = factory.get('/' + version + '/cutout/col1/exp_ds_aniso_4/channel1/4/0:256/0:256/2:4/?iso=false',
+                              HTTP_ACCEPT='application/blosc')
+
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso_4', channel='channel1',
+                                    resolution='4', x_range='0:256', y_range='0:256',
+                                    z_range='2:4', t_range=None).render()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # Decompress
+        raw_data = blosc.decompress(response.content)
+        data_mat = np.fromstring(raw_data, dtype=np.uint8)
+        data_mat = np.reshape(data_mat, (2, 256, 256), order='C')
+
+        # Should be 0 at non-iso, since you wrote directly to iso level
+        self.assertEqual(data_mat.sum(), 0)
+
+        # Create Request to get data you posted, but ask for iso data
+        request = factory.get('/' + version + '/cutout/col1/exp_ds_aniso_4/channel1/4/0:256/0:256/2:4/?iso=true',
+                              HTTP_ACCEPT='application/blosc')
+
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_aniso_4', channel='channel1',
+                                    resolution='4', x_range='0:256', y_range='0:256',
+                                    z_range='2:4', t_range=None).render()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # Decompress
+        raw_data = blosc.decompress(response.content)
+        data_mat = np.fromstring(raw_data, dtype=np.uint8)
+        data_mat = np.reshape(data_mat, (2, 256, 256), order='C')
+
+        # Should be 0 at non-iso, since you wrote directly to iso level
+        np.testing.assert_array_equal(data_mat, test_mat)
+
+    def test_channel_uint8_downsample_above_iso_fork_isotropic(self):
+        """ Test writing/reading data to non-base resolution above the iso fork with an isotropic channel"""
+        self.dbsetup.insert_downsample_data()
+        self.dbsetup.insert_downsample_write_data()
+
+        test_mat = np.random.randint(1, 254, (2, 256, 256))
+        test_mat = test_mat.astype(np.uint8)
+        h = test_mat.tobytes()
+        bb = blosc.compress(h, typesize=8)
+
+        # Create request
+        factory = APIRequestFactory()
+        request = factory.post('/' + version + '/cutout/col1/exp_ds_iso_4/channel1/4/0:256/0:256/2:4/?iso=true', bb,
+                               content_type='application/blosc')
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_iso_4', channel='channel1',
+                                    resolution='4', x_range='0:256', y_range='0:256', z_range='2:4', t_range=None)
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        # Create Request to get data you posted, but ask for aniso data
+        request = factory.get('/' + version + '/cutout/col1/exp_ds_iso_4/channel1/4/0:256/0:256/2:4/?iso=false',
+                              HTTP_ACCEPT='application/blosc')
+
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_iso_4', channel='channel1',
+                                    resolution='4', x_range='0:256', y_range='0:256',
+                                    z_range='2:4', t_range=None).render()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # Decompress
+        raw_data = blosc.decompress(response.content)
+        data_mat = np.fromstring(raw_data, dtype=np.uint8)
+        data_mat = np.reshape(data_mat, (2, 256, 256), order='C')
+
+        # Should be equal to what you wrote (data is only isotropic)
+        np.testing.assert_array_equal(data_mat, test_mat)
+
+        # Create Request to get data you posted, but ask for iso data
+        request = factory.get('/' + version + '/cutout/col1/exp_ds_iso_4/channel1/4/0:256/0:256/2:4/?iso=true',
+                              HTTP_ACCEPT='application/blosc')
+
+        # log in user
+        force_authenticate(request, user=self.user)
+
+        # Make request
+        response = Cutout.as_view()(request, collection='col1', experiment='exp_ds_iso_4', channel='channel1',
+                                    resolution='4', x_range='0:256', y_range='0:256',
+                                    z_range='2:4', t_range=None).render()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        # Decompress
+        raw_data = blosc.decompress(response.content)
+        data_mat = np.fromstring(raw_data, dtype=np.uint8)
+        data_mat = np.reshape(data_mat, (2, 256, 256), order='C')
+
+        # Should be equal to what you wrote (data is only isotropic)
+        np.testing.assert_array_equal(data_mat, test_mat)
 
 # @patch('bossutils.configuration.BossConfig', new=MockBossConfig)
 # @patch('redis.StrictRedis', new=mock_strict_redis_client)
@@ -927,23 +1097,24 @@ def setUp(self):
         :return:
         """
         # Create a user
-        dbsetup = SetupTestDB()
-        self.user = dbsetup.create_user('testuser')
+        self.dbsetup = SetupTestDB()
+        self.user = self.dbsetup.create_user('testuser')
 
         # Populate DB
-        dbsetup.insert_spatialdb_test_data()
+        self.dbsetup.insert_spatialdb_test_data()
 
         # Mock config parser so dummy params get loaded (redis is also mocked)
         #self.patcher = patch('bossutils.configuration.BossConfig', MockBossConfig)
         #self.mock_tests = self.patcher.start()
 
         ##self.mock_redis = self.redis_patcher.start()
 
-        self.spdb_patcher = patch('spdb.spatialdb.spatialdb.SpatialDB', MockSpatialDB)
-        self.mock_spdb = self.spdb_patcher.start()
+        #self.spdb_patcher = patch('spdb.spatialdb.spatialdb.SpatialDB', mock_init_)
+        #self.mock_spdb = self.spdb_patcher.start()
 
     def tearDown(self):
         # Stop mocking
         #self.patcher.stop()
-        self.spdb_patcher.stop()
+        #self.spdb_patcher.stop()
         #self.redis_patcher.stop()
+        pass

django/bossspatialdb/test/int_test_cutout_view_uint8.py

@@ -119,6 +119,8 @@ def setUp(self):
                                    port=6379, db=1, decode_responses=False)
         client.flushdb()
 
+        self.dbsetup = self.layer.django_setup_helper
+
     def tearDown(self):
         # Flush cache between tests
         client = redis.StrictRedis(host=self.kvio_config['cache_host'],