Merge pull request #67 from RangelReale/do

DigitalOcean Spaces driver

AUTHORS.rst

@@ -15,3 +15,4 @@ Contributors
 * James Stewart `@JamesStewy <https://github.com/JamesStewy>`_
 * Matt Carr `@matt-carr <https://github.com/matt-carr>`_
 * Sibo Wang `@sibowsb <https://github.com/sibowsb>`_
+* Rangel Reale `@RangelReale <https://github.com/RangelReale>`_

setup.py

@@ -12,6 +12,7 @@
 ]
 EXTRAS_REQUIRE = {
     "amazon": ["boto3>=1.8.00", "boto3-stubs[s3]>==1.12.41.0"],
+    "digitalocean": ["boto3>=1.8.00", "boto3-stubs[s3]>==1.12.41.0"],
     "google": ["google-cloud-storage>=1.18.0", "requests>=2.19.1"],
     "local": [
         "filelock>=3.0.0",

src/cloudstorage/__init__.py

@@ -36,6 +36,7 @@ class DriverName(Enum):
     LOCAL = "LOCAL"
     MINIO = "MINIO"
     S3 = "S3"
+    DIGITALOCEANSPACES = "DIGITALOCEANSPACES"
 
 
 _DRIVER_IMPORTS = {
@@ -45,6 +46,10 @@ class DriverName(Enum):
     DriverName.LOCAL: ("cloudstorage.drivers.local", "LocalDriver"),
     DriverName.MINIO: ("cloudstorage.drivers.minio", "MinioDriver"),
     DriverName.S3: ("cloudstorage.drivers.amazon", "S3Driver"),
+    DriverName.DIGITALOCEANSPACES: (
+        "cloudstorage.drivers.digitalocean",
+        "DigitalOceanSpacesDriver",
+    ),
 }
 
 
@@ -63,7 +68,7 @@ def get_driver(driver: DriverName) -> Drivers:
     :return: DriverName driver class.
     :rtype: :class:`.AzureStorageDriver`, :class:`.CloudFilesDriver`,
       :class:`.GoogleStorageDriver`, :class:`.S3Driver`, :class:`.LocalDriver`,
-      :class:`.MinioDriver`
+      :class:`.MinioDriver`, :class:`.DigitalOceanSpacesDriver`
     """
     if driver in _DRIVER_IMPORTS:
         mod_name, driver_name = _DRIVER_IMPORTS[driver]
@@ -90,12 +95,13 @@ def get_driver_by_name(driver_name: str) -> Drivers:
         * `S3`
         * `LOCAL`
         * `MINIO`
+        * `DIGITALOCEANSPACES`
     :type driver_name: str
 
     :return: DriverName driver class.
     :rtype: :class:`.AzureStorageDriver`, :class:`.CloudFilesDriver`,
       :class:`.GoogleStorageDriver`, :class:`.S3Driver`, :class:`.LocalDriver`,
-      :class:`.MinioDriver`
+      :class:`.MinioDriver`, :class:`.DigitalOceanSpacesDriver`
     """
     driver = DriverName[driver_name]
     return get_driver(driver)

src/cloudstorage/drivers/amazon.py

@@ -224,6 +224,19 @@ def _make_container(self, bucket) -> Container:
             created_at=created_at,
         )
 
+    def _create_bucket_params(self, params: Dict[Any, Any]) -> Dict[Any, Any]:
+        """Process extra create bucket params.
+
+        :param params: Default create bucket parameters.
+        :return: Final create bucket parameters.
+        """
+        # TODO: BUG: Creating S3 bucket in us-east-1
+        if self.region != "us-east-1":
+            params["CreateBucketConfiguration"] = {
+                "LocationConstraint": self.region,
+            }
+        return params
+
     @property
     def session(self) -> boto3.session.Session:
         """Amazon Web Services session.
@@ -267,12 +280,7 @@ def create_container(
         if acl:
             params["ACL"] = acl.lower()
 
-        # TODO: BUG: Creating S3 bucket in us-east-1
-        # See https://github.com/boto/boto3/issues/125
-        if self.region != "us-east-1":
-            params["CreateBucketConfiguration"] = {
-                "LocationConstraint": self.region,
-            }
+        params = self._create_bucket_params(params)
 
         logger.debug("params=%s", params)
 

src/cloudstorage/drivers/digitalocean.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Dict, List, Any
+
+import boto3
+
+from cloudstorage.drivers.amazon import S3Driver
+
+__all__ = ["DigitalOceanSpacesDriver"]
+
+logger = logging.getLogger(__name__)
+
+
+class DigitalOceanSpacesDriver(S3Driver):
+    """Driver for interacting with DigitalOcean Spaces Service.
+    DigitalOcean spaces uses an S3-compatible API, this drivers extends the
+    S3 driver to implement API differences.
+
+    .. code-block:: python
+
+        from cloudstorage.drivers.digitalocean import DigitalOceanSpacesDriver
+
+        storage = DigitalOceanSpacesDriver(key='<my-digitalocean-access-key-id>',
+                   secret='<my-digitalocean-secret-access-key>',
+                   region='sfo2')
+        # <Driver: S3 us-east-1>
+
+    References:
+
+    * `DigitalOcean Spaces API
+      <https://developers.digitalocean.com/documentation/spaces/>`_
+
+    :param key: DigitalOcean Access Key ID.
+    :type key: str
+
+    :param secret: DigitalOcean Secret Access Key.
+    :type secret: str
+
+    :param region: (optional) Region to connect to. Defaults to `sfo2`.
+    :type region: str
+
+    :param kwargs: (optional) Extra driver options.
+    :type kwargs: dict
+    """
+
+    name = "DIGITALOCEANSPACES"
+    hash_type = "md5"
+    url = "https://www.digitalocean.com/products/spaces/"
+
+    def __init__(
+        self, key: str, secret: str = None, region: str = "sfo2", **kwargs: Dict
+    ) -> None:
+        super().__init__(key, secret, region, **kwargs)
+
+    def _create_bucket_params(self, params: Dict[Any, Any]) -> Dict[Any, Any]:
+        return params
+
+    @property
+    def regions(self) -> List[str]:
+        """List of DigitalOcean regions that support Spaces.
+        """
+        return ["nyc3", "ams3", "sfo2", "sgp1", "fra1"]
+
+    # noinspection PyUnresolvedReferences
+    @property
+    def s3(self) -> boto3.resources.base.ServiceResource:
+        """S3 service resource.
+
+        :return: The s3 resource instance.
+        :rtype: :class:`boto3.resources.base.ServiceResource`
+        """
+        return self.session.resource(
+            service_name="s3",
+            region_name=self.region,
+            endpoint_url=f"https://{self.region}.digitaloceanspaces.com",
+        )
+
+    def validate_credentials(self) -> None:
+        # Not available
+        pass

src/cloudstorage/drivers/local.py

@@ -703,7 +703,7 @@ def __setitem__(self, key, value) -> None:
         """
         data = self._load()
         if isinstance(value, bytes):
-            value = value.decode('utf-8')
+            value = value.decode("utf-8")
         data[key] = value
         with open(self.xattr_filename, "w") as outfile:
             json.dump(data, outfile)

tests/conftest.py

@@ -72,6 +72,9 @@ def binary_blob(container, binary_filename):
 
 @pytest.fixture(scope="function")
 def temp_file():
-    _, path = mkstemp(prefix=settings.CONTAINER_PREFIX)
+    fd, path = mkstemp(prefix=settings.CONTAINER_PREFIX)
+    if os.name == "nt":
+        # Must close in Windows, otherwise errors as file being used
+        os.close(fd)
     yield path
     os.remove(path)

tests/settings.py

@@ -43,6 +43,10 @@
 AMAZON_SECRET = config("AMAZON_SECRET", default=None)
 AMAZON_REGION = config("AMAZON_REGION", default="us-east-1")
 
+DIGITALOCEAN_KEY = config("DIGITALOCEAN_KEY", default=None)
+DIGITALOCEAN_SECRET = config("DIGITALOCEAN_SECRET", default=None)
+DIGITALOCEAN_REGION = config("DIGITALOCEAN_REGION", default="sfo2")
+
 AZURE_ACCOUNT_NAME = config("AZURE_ACCOUNT_NAME", default=None)
 AZURE_ACCOUNT_KEY = config("AZURE_ACCOUNT_KEY", default=None)