storage: allow configuration of storage region for cloud storage (PRO…

…JQUAY-3082)

Boto3 behaves unexpectedly when the resource client is not set to use
the correct region. Boto3 can't seem to correctly set the
X-Amz-Credential header when generating presigned urls if the region
name is not explicitly set, and will always fall back to us-east-1.
To reproduce this:
- Create a bucket in a different region from us-east-1 (e.g
eu-north-1)
- Create a boto3 client/resource without specifying the region
- Generate a presigned url

This seems to be a DNS issue with AWS that only happens shortly after
a bucket has been created, and resolves itself eventually.
Ref:
- boto/boto3#2989
- https://stackoverflow.com/questions/56517156/s3-presigned-url-works-90-minutes-after-bucket-creation

To workaround this, one can specify the bucket endpoint, either
explicitly via endpoint_url, or by setting s3_region, which will be
used to generate the bucket's virtual address.

storage/cloud.py

@@ -731,6 +731,7 @@ def __init__(
         s3_bucket,
         s3_access_key=None,
         s3_secret_key=None,
+        s3_region=None,
         # Boto2 backward compatible options (host excluding scheme or port)
         host=None,
         port=None,
@@ -739,7 +740,12 @@ def __init__(
     ):
         upload_params = {"ServerSideEncryption": "AES256"}
         connect_kwargs = {"config": Config(signature_version="s3v4")}
-        if host or endpoint_url:
+        if s3_region is not None:
+            connect_kwargs["region_name"] = s3_region
+            connect_kwargs["endpoint_url"] = "https://s3.{region}.amazonaws.com".format(
+                region=s3_region
+            )
+        elif host or endpoint_url:
             connect_kwargs["endpoint_url"] = endpoint_url or _build_endpoint_url(
                 host, port=port, is_secure=True
             )

storage/test/test_cloud_storage.py

@@ -21,6 +21,7 @@
 _TEST_BUCKET = "somebucket"
 _TEST_USER = "someuser"
 _TEST_PASSWORD = "somepassword"
+_TEST_REGION = "us-bacon-1"
 _TEST_PATH = "some/cool/path"
 _TEST_UPLOADS_PATH = "uploads/ee160658-9444-4950-8ec6-30faab40529c"
 _TEST_CONTEXT = StorageContext("nyc", None, None, None)
@@ -31,7 +32,8 @@ def storage_engine():
     with mock_s3():
         # Create a test bucket and put some test content.
         boto3.client("s3").create_bucket(Bucket=_TEST_BUCKET)
-        engine = S3Storage(_TEST_CONTEXT, "some/path", _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
+        engine = S3Storage(_TEST_CONTEXT, "some/path", _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD, _TEST_REGION)
+        assert engine._connect_kwargs["endpoint_url"] == "https://s3.{}.amazonaws.com".format(_TEST_REGION)
         engine.put_content(_TEST_PATH, _TEST_CONTENT)
 
         yield engine