RDISCROWD-7283: Access buckets with util endpoints (#922)

* update metadata url path

pybossa/cloud_store_api/s3.py

@@ -125,9 +125,19 @@ def s3_upload_tmp_file(s3_bucket, tmp_file, filename,
         fp = io.BytesIO(content)  # BytesIO accepts bytes string
         url = s3_upload_file(s3_bucket, fp, filename, headers, upload_root_dir,
                              directory, return_key_only, conn_name)
+        bcosv2_prod_util_url = app.config.get('BCOSV2_PROD_UTIL_URL')
+
+        # generate url path to be stored as metadata
+        # which can be different from actual uploaded url
+        # and is based upon the type of uploaded url path
+        meta_url = url
+        if bcosv2_prod_util_url and url.startswith(bcosv2_prod_util_url):
+            meta_url = url.replace("-util", "")
+            app.logger.info("bcosv2 url paths. uploaded path %s, metadata path %s", url, meta_url)
+
     finally:
         os.unlink(tmp_file.name)
-    return url
+    return meta_url
 
 
 def form_upload_directory(directory, filename, upload_root_dir):

test/test_cloud_store_api/test_s3_uploader.py

@@ -38,6 +38,15 @@ class TestS3Uploader(Test):
         }
     }
 
+    util_config = {
+        'BCOSV2_PROD_UTIL_URL': "https://s3.storage.env-util.com",
+        'S3_DEFAULT': {
+            'host': "s3.storage.env-util.com",
+            'port': 443,
+            'auth_headers': [('test', 'name')]
+        }
+    }
+
     def test_check_valid_type(self):
         with NamedTemporaryFile() as fp:
             fp.write(b'hello world')
@@ -60,6 +69,14 @@ def test_upload_from_string(self, set_contents):
             url = s3_upload_from_string('bucket', 'hello world', 'test.txt')
             assert url == 'https://s3.storage.com:443/bucket/test.txt', url
 
+    @with_context
+    @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file')
+    def test_upload_from_string_util(self, set_contents):
+        with patch.dict(self.flask_app.config, self.util_config):
+            """Test -util keyword dropped from meta url returned from s3 upload."""
+            url = s3_upload_from_string('bucket', 'hello world', 'test.txt')
+            assert url == 'https://s3.storage.env.com:443/bucket/test.txt', url
+
     @with_context
     @patch('pybossa.cloud_store_api.s3.io.open')
     def test_upload_from_string_exception(self, open):