Add support for Object Storage Gen 2 (#503)

* wip

* Added support for OBJ Gen 2

* Fix lint

* Address PR comments

* More PR comments

Author: ezilber-akamai

linode_api4/groups/object_storage.py

@@ -5,6 +5,11 @@
 
 from deprecated import deprecated
 
+from linode_api4 import (
+    ObjectStorageEndpoint,
+    ObjectStorageEndpointType,
+    PaginatedList,
+)
 from linode_api4.errors import UnexpectedResponseError
 from linode_api4.groups import Group
 from linode_api4.objects import (
@@ -272,6 +277,30 @@ def transfer(self):
 
         return MappedObject(**result)
 
+    def endpoints(self, *filters) -> PaginatedList:
+        """
+        Returns a paginated list of all Object Storage endpoints available in your account.
+
+        This is intended to be called from the :any:`LinodeClient`
+        class, like this::
+
+           endpoints = client.object_storage.endpoints()
+
+        API Documentation: https://techdocs.akamai.com/linode-api/reference/get-object-storage-endpoints
+
+        :param filters: Any number of filters to apply to this query.
+                        See :doc:`Filtering Collections</linode_api4/objects/filtering>`
+                        for more details on filtering.
+
+        :returns: A list of Object Storage Endpoints that matched the query.
+        :rtype: PaginatedList of ObjectStorageEndpoint
+        """
+        return self.client._get_and_filter(
+            ObjectStorageEndpoint,
+            *filters,
+            endpoint="/object-storage/endpoints",
+        )
+
     def buckets(self, *filters):
         """
         Returns a paginated list of all Object Storage Buckets that you own.
@@ -299,6 +328,8 @@ def bucket_create(
         label: str,
         acl: ObjectStorageACL = ObjectStorageACL.PRIVATE,
         cors_enabled=False,
+        s3_endpoint: Optional[str] = None,
+        endpoint_type: Optional[ObjectStorageEndpointType] = None,
     ):
         """
         Creates an Object Storage Bucket in the specified cluster. Accounts with
@@ -320,6 +351,13 @@ def bucket_create(
                         should be created.
         :type cluster: str
 
+        :param endpoint_type: The type of s3_endpoint available to the active user in this region.
+        :type endpoint_type: str
+                   Enum: E0,E1,E2,E3
+
+        :param s3_endpoint: The active user's s3 endpoint URL, based on the endpoint_type and region.
+        :type s3_endpoint: str
+
         :param cors_enabled: If true, the bucket will be created with CORS enabled for
                              all origins. For more fine-grained controls of CORS, use
                              the S3 API directly.
@@ -346,6 +384,8 @@ def bucket_create(
             "label": label,
             "acl": acl,
             "cors_enabled": cors_enabled,
+            "s3_endpoint": s3_endpoint,
+            "endpoint_type": endpoint_type,
         }
 
         if self.is_cluster(cluster_or_region_id):

linode_api4/objects/object_storage.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from typing import Optional
 from urllib import parse
 
@@ -11,7 +12,7 @@
     Property,
     Region,
 )
-from linode_api4.objects.serializable import StrEnum
+from linode_api4.objects.serializable import JSONObject, StrEnum
 from linode_api4.util import drop_null_keys
 
 
@@ -28,6 +29,27 @@ class ObjectStorageKeyPermission(StrEnum):
     READ_WRITE = "read_write"
 
 
+class ObjectStorageEndpointType(StrEnum):
+    E0 = "E0"
+    E1 = "E1"
+    E2 = "E2"
+    E3 = "E3"
+
+
+@dataclass
+class ObjectStorageEndpoint(JSONObject):
+    """
+    ObjectStorageEndpoint contains the core fields of an object storage endpoint object.
+
+    NOTE: This is not implemented as a typical API object (Base) because Object Storage Endpoints
+    cannot be refreshed, as there is no singular GET endpoint.
+    """
+
+    region: str = ""
+    endpoint_type: ObjectStorageEndpointType = ""
+    s3_endpoint: Optional[str] = None
+
+
 class ObjectStorageBucket(DerivedBase):
     """
     A bucket where objects are stored in.
@@ -47,6 +69,8 @@ class ObjectStorageBucket(DerivedBase):
         "label": Property(identifier=True),
         "objects": Property(),
         "size": Property(),
+        "endpoint_type": Property(),
+        "s3_endpoint": Property(),
     }
 
     @classmethod
@@ -63,13 +87,8 @@ def make_instance(cls, id, client, parent_id=None, json=None):
         Override this method to pass in the parent_id from the _raw_json object
         when it's available.
         """
-        if json is None:
-            return None
-
-        cluster_or_region = json.get("region") or json.get("cluster")
-
-        if parent_id is None and cluster_or_region:
-            parent_id = cluster_or_region
+        if json is not None:
+            parent_id = parent_id or json.get("region") or json.get("cluster")
 
         if parent_id:
             return super().make(id, client, cls, parent_id=parent_id, json=json)
@@ -78,6 +97,31 @@ def make_instance(cls, id, client, parent_id=None, json=None):
                 "Unexpected json response when making a new Object Storage Bucket instance."
             )
 
+    def access_get(self):
+        """
+        Returns a result object which wraps the current access config for this ObjectStorageBucket.
+
+        API Documentation: TODO
+
+        :returns: A result object which wraps the access that this ObjectStorageBucket is currently configured with.
+        :rtype: MappedObject
+        """
+        result = self._client.get(
+            "{}/access".format(self.api_endpoint),
+            model=self,
+        )
+
+        if not any(
+            key in result
+            for key in ["acl", "acl_xml", "cors_enabled", "cors_xml"]
+        ):
+            raise UnexpectedResponseError(
+                "Unexpected response when getting the bucket access config of a bucket!",
+                json=result,
+            )
+
+        return MappedObject(**result)
+
     def access_modify(
         self,
         acl: Optional[ObjectStorageACL] = None,

test/fixtures/object-storage_buckets_us-east-1.json

@@ -6,7 +6,9 @@
         "hostname": "example-bucket.us-east-1.linodeobjects.com",
         "label": "example-bucket",
         "objects": 4,
-        "size": 188318981
+        "size": 188318981,
+        "endpoint_type": "E1",
+        "s3_endpoint": "us-east-12.linodeobjects.com"
       }
     ],
     "page": 1,

test/fixtures/object-storage_buckets_us-east-1_example-bucket.json

@@ -5,5 +5,7 @@
   "hostname": "example-bucket.us-east-1.linodeobjects.com",
   "label": "example-bucket",
   "objects": 4,
-  "size": 188318981
+  "size": 188318981,
+  "endpoint_type": "E1",
+  "s3_endpoint": "us-east-12.linodeobjects.com"
 }

test/fixtures/object-storage_buckets_us-east_example-bucket_access.json

@@ -0,0 +1,6 @@
+{
+    "acl": "authenticated-read",
+    "acl_xml": "<AccessControlPolicy...",
+    "cors_enabled": true,
+    "cors_xml": "<CORSConfiguration>..."
+}

test/integration/models/object_storage/test_obj.py

@@ -8,6 +8,7 @@
     ObjectStorageACL,
     ObjectStorageBucket,
     ObjectStorageCluster,
+    ObjectStorageEndpointType,
     ObjectStorageKeyPermission,
     ObjectStorageKeys,
 )
@@ -19,7 +20,14 @@ def region(test_linode_client: LinodeClient):
 
 
 @pytest.fixture(scope="session")
-def bucket(test_linode_client: LinodeClient, region: str):
+def endpoints(test_linode_client: LinodeClient):
+    return test_linode_client.object_storage.endpoints()
+
+
+@pytest.fixture(scope="session")
+def bucket(
+    test_linode_client: LinodeClient, region: str
+) -> ObjectStorageBucket:
     bucket = test_linode_client.object_storage.bucket_create(
         cluster_or_region=region,
         label="bucket-" + str(time.time_ns()),
@@ -31,6 +39,31 @@ def bucket(test_linode_client: LinodeClient, region: str):
     bucket.delete()
 
 
+@pytest.fixture(scope="session")
+def bucket_with_endpoint(
+    test_linode_client: LinodeClient, endpoints
+) -> ObjectStorageBucket:
+    selected_endpoint = next(
+        (
+            e
+            for e in endpoints
+            if e.endpoint_type == ObjectStorageEndpointType.E1
+        ),
+        None,
+    )
+
+    bucket = test_linode_client.object_storage.bucket_create(
+        cluster_or_region=selected_endpoint.region,
+        label="bucket-" + str(time.time_ns()),
+        acl=ObjectStorageACL.PRIVATE,
+        cors_enabled=False,
+        endpoint_type=selected_endpoint.endpoint_type,
+    )
+
+    yield bucket
+    bucket.delete()
+
+
 @pytest.fixture(scope="session")
 def obj_key(test_linode_client: LinodeClient):
     key = test_linode_client.object_storage.keys_create(
@@ -71,19 +104,39 @@ def test_keys(
 
     assert loaded_key.label == obj_key.label
     assert loaded_limited_key.label == obj_limited_key.label
+    assert (
+        loaded_limited_key.regions[0].endpoint_type
+        in ObjectStorageEndpointType.__members__.values()
+    )
 
 
-def test_bucket(
-    test_linode_client: LinodeClient,
-    bucket: ObjectStorageBucket,
-):
-    loaded_bucket = test_linode_client.load(ObjectStorageBucket, bucket.label)
+def test_bucket(test_linode_client: LinodeClient, bucket: ObjectStorageBucket):
+    loaded_bucket = test_linode_client.load(
+        ObjectStorageBucket,
+        target_id=bucket.label,
+        target_parent_id=bucket.region,
+    )
 
     assert loaded_bucket.label == bucket.label
     assert loaded_bucket.region == bucket.region
 
 
-def test_bucket(
+def test_bucket_with_endpoint(
+    test_linode_client: LinodeClient, bucket_with_endpoint: ObjectStorageBucket
+):
+    loaded_bucket = test_linode_client.load(
+        ObjectStorageBucket,
+        target_id=bucket_with_endpoint.label,
+        target_parent_id=bucket_with_endpoint.region,
+    )
+
+    assert loaded_bucket.label == bucket_with_endpoint.label
+    assert loaded_bucket.region == bucket_with_endpoint.region
+    assert loaded_bucket.s3_endpoint is not None
+    assert loaded_bucket.endpoint_type == "E1"
+
+
+def test_buckets_in_region(
     test_linode_client: LinodeClient,
     bucket: ObjectStorageBucket,
     region: str,
@@ -103,6 +156,14 @@ def test_list_obj_storage_bucket(
     assert any(target_bucket_id == b.id for b in buckets)
 
 
+def test_bucket_access_get(bucket: ObjectStorageBucket):
+    access = bucket.access_get()
+
+    assert access.acl is not None
+    assert access.acl_xml is not None
+    assert access.cors_enabled is not None
+
+
 def test_bucket_access_modify(bucket: ObjectStorageBucket):
     bucket.access_modify(ObjectStorageACL.PRIVATE, cors_enabled=True)
 

test/unit/objects/object_storage_test.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from test.unit.base import ClientBaseCase
 
+from linode_api4 import ObjectStorageEndpointType
 from linode_api4.objects import (
     ObjectStorageACL,
     ObjectStorageBucket,
@@ -35,6 +36,14 @@ def test_object_storage_bucket_api_get(self):
             )
             self.assertEqual(object_storage_bucket.objects, 4)
             self.assertEqual(object_storage_bucket.size, 188318981)
+            self.assertEqual(
+                object_storage_bucket.endpoint_type,
+                ObjectStorageEndpointType.E1,
+            )
+            self.assertEqual(
+                object_storage_bucket.s3_endpoint,
+                "us-east-12.linodeobjects.com",
+            )
             self.assertEqual(m.call_url, object_storage_bucket_api_get_url)
 
     def test_object_storage_bucket_delete(self):
@@ -48,6 +57,22 @@ def test_object_storage_bucket_delete(self):
             object_storage_bucket.delete()
             self.assertEqual(m.call_url, object_storage_bucket_delete_url)
 
+    def test_bucket_access_get(self):
+        bucket_access_get_url = (
+            "/object-storage/buckets/us-east/example-bucket/access"
+        )
+        with self.mock_get(bucket_access_get_url) as m:
+            object_storage_bucket = ObjectStorageBucket(
+                self.client, "example-bucket", "us-east"
+            )
+            result = object_storage_bucket.access_get()
+            self.assertIsNotNone(result)
+            self.assertEqual(m.call_url, bucket_access_get_url)
+            self.assertEqual(result.acl, "authenticated-read")
+            self.assertEqual(result.cors_enabled, True)
+            self.assertEqual(result.acl_xml, "<AccessControlPolicy...")
+            self.assertEqual(result.cors_xml, "<CORSConfiguration>...")
+
     def test_bucket_access_modify(self):
         """
         Test that you can modify bucket access settings.
@@ -115,6 +140,8 @@ def test_buckets_in_cluster(self):
             self.assertEqual(bucket.label, "example-bucket")
             self.assertEqual(bucket.objects, 4)
             self.assertEqual(bucket.size, 188318981)
+            self.assertEqual(bucket.endpoint_type, ObjectStorageEndpointType.E1)
+            self.assertEqual(bucket.s3_endpoint, "us-east-12.linodeobjects.com")
 
     def test_ssl_cert_delete(self):
         """